From 606bb717d9c895b0261cb57c914cb362d85b10a7 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 11 Jul 2024 14:17:17 -0400 Subject: [PATCH] add boto s3 sample --- sample/s3z/.gitignore | 2 + sample/s3z/README.md | 178 ++++++++++++++++++++++++++++++++++++ sample/s3z/log-generator.py | 35 +++++++ sample/s3z/requirements.txt | 4 + sample/s3z/s3z.py | 65 +++++++++++++ 5 files changed, 284 insertions(+) create mode 100644 sample/s3z/.gitignore create mode 100644 sample/s3z/README.md create mode 100644 sample/s3z/log-generator.py create mode 100644 sample/s3z/requirements.txt create mode 100644 sample/s3z/s3z.py diff --git a/sample/s3z/.gitignore b/sample/s3z/.gitignore new file mode 100644 index 0000000..a91b9ab --- /dev/null +++ b/sample/s3z/.gitignore @@ -0,0 +1,2 @@ +/logs/* +/ziti_identities/* \ No newline at end of file diff --git a/sample/s3z/README.md b/sample/s3z/README.md new file mode 100644 index 0000000..7c17906 --- /dev/null +++ b/sample/s3z/README.md @@ -0,0 +1,178 @@ +# Ziti S3 Log Uploader + +This example shows how to upload *.log files to a private S3 bucket. + +## Setup :wrench: + +Refer to the [examples README](../README.md) for details on setting up a service and endpoint identities. + +The rest of the example commands assume you are inside this example's directory. + +```bash +cd ./samples/s3z +``` + +### Install Python Requirements + +Install the PyPi modules required by this example. + +```bash +pip install --requirement ./requirements.txt +``` + +## Set Up AWS + +Here are the AWS ingredients. + +1. Choose an AWS region to set everything up +1. An S3 VPC Endpoint (Privatelink Interface) +1. An S3 Bucket +1. A Bucket Policy that requires the VPCE source +1. A Security Group that allows the bucket service host to send 443/tcp to the VPCE +1. Any IAM credential + +### Bucket Policy Example + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Deny-access-if-not-VPCE", + "Effect": "Deny", + "Principal": "*", + "Action": [ + "s3:ListBucket", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::boto-demo-s3z", + "arn:aws:s3:::boto-demo-s3z/*" + ], + "Condition": { + "StringNotEquals": { + "aws:sourceVpce": "vpce-0f3e9a76e6d070f9a" + } + } + }, + { + "Sid": "Allow-access-if-VPCE", + "Effect": "Allow", + "Principal": "*", + "Action": [ + "s3:ListBucket", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::boto-demo-s3z", + "arn:aws:s3:::boto-demo-s3z/*" + ], + "Condition": { + "StringEquals": { + "aws:sourceVpce": "vpce-0f3e9a76e6d070f9a" + } + } + } + ] +} +``` + +## Set Up Ziti + +Here are the Ziti ingredients. + +1. A bucket service to configure the S3 endpoint URL + 1. `intercept.v1` - this configures the Python SDK client to send requests matching the VPC endpoint through the tunnel + + ```json + { + "addresses": [ + "*.vpce-0f3e9a76e6d070f9a.s3.us-west-1.vpce.amazonaws.com" + ], + "portRanges": [ + { + "high": 443, + "low": 443 + } + ], + "protocols": [ + "tcp" + ] + } + ``` + + 1. `host.v1` - this configures the hosting endpoint to send the traffic exiting the Ziti tunnel to the VPC endpoint + + ```json + { + "address": "bucket.vpce-0f3e9a76e6d070f9a.s3.us-west-1.vpce.amazonaws.com", + "allowedPortRanges": [ + { + "high": 443, + "low": 443 + } + ], + "allowedProtocols": [ + "tcp" + ], + "forwardPort": true, + "forwardProtocol": true, + "protocol": "tcp" + } + ``` + +1. Enrolled Ziti identities for each end of the tunnel + 1. client - `s3z.py` will use this identity to "dial" the bucket service + 1. host - a container or VM inside the VPC will provide a privileged exit point to the private endpoint, i.e., hosting tunneler + +1. Service Policies + 1. Dial - the client identity needs dial permission for the bucket service + 1. Bind - the host needs bind permission for the bucket service + +1. Router Policies - ensure your identities and services are granted access to at least one common, online router + +## Generate Some Dummy Files + +If you need some worthless log files you can run this. + +```bash +python ./log-generator.py +``` + +## Understanding the Inputs :brain: + +This example accepts some options and arguments. + +1. `--ziti-identity-file` - The identity file to be used by the SDK tunneler to dial the bucket service +1. `--bucket-name` - where to upload log files +1. `--bucket-endpoint` - the private VPC endpoint URL +1. `--push-log-dir` - local directory where logs should be uploaded from +1. `--object-prefix` - optional directory-like prefix for the uploaded files + +## Running the Example :arrow_forward: + +```bash +python ./s3z/s3z.py \ + --ziti-identity-file=/etc/ziti/client.json \ + --bucket-name=my-private-logs \ + --bucket-endpoint=https://bucket.vpce-0f3e9a76e6d070f9a.s3.us-west-1.vpce.amazonaws.com \ + --push-log-dir=./logs \ + --object-prefix=$(hostname -f)/$(date --utc --iso-8601=s) +``` + +```buttonless title="Output" +Uploaded ./logs/stupefied-ptolemy.log to boto-demo-s3z/loghost.example.com/2024-07-11T18:13:47+00:00 +Uploaded ./logs/modest-feynman.log to boto-demo-s3z/loghost.example.com/2024-07-11T18:13:47+00:00 +Uploaded ./logs/priceless-einstein.log to boto-demo-s3z/loghost.example.com/2024-07-11T18:13:47+00:00 +Uploaded ./logs/gallant-bardeen.log to boto-demo-s3z/loghost.example.com/2024-07-11T18:13:47+00:00 +Uploaded ./logs/epic-heisenberg.log to boto-demo-s3z/loghost.example.com/2024-07-11T18:13:47+00:00 +Uploaded ./logs/vibrant-galileo.log to boto-demo-s3z/loghost.example.com/2024-07-11T18:13:47+00:00 +Uploaded ./logs/hopeful-wilson.log to boto-demo-s3z/loghost.example.com/2024-07-11T18:13:47+00:00 +Uploaded ./logs/distracted-golick.log to boto-demo-s3z/loghost.example.com/2024-07-11T18:13:47+00:00 +Uploaded ./logs/youthful-poitras.log to boto-demo-s3z/loghost.example.com/2024-07-11T18:13:47+00:00 +Uploaded ./logs/agitated-curie.log to boto-demo-s3z/loghost.example.com/2024-07-11T18:13:47+00:00 +``` \ No newline at end of file diff --git a/sample/s3z/log-generator.py b/sample/s3z/log-generator.py new file mode 100644 index 0000000..44315fe --- /dev/null +++ b/sample/s3z/log-generator.py @@ -0,0 +1,35 @@ +import os +import random + +import namesgenerator + + +def generate_friendly_name(): + return namesgenerator.get_random_name(sep='-') + + +def generate_random_data(size_mib): + return os.urandom(size_mib * 1024 * 1024) + + +def create_files(n, min_size_mib, max_size_mib, output_dir): + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + for _ in range(n): + file_name = generate_friendly_name() + ".log" + file_size = random.randint(min_size_mib, max_size_mib) + file_data = generate_random_data(file_size) + + with open(os.path.join(output_dir, file_name), 'wb') as f: + f.write(file_data) + print(f"Created file: {file_name} with size: {file_size} MiB") + + +if __name__ == "__main__": + n = 10 # Number of files to generate + min_size_mib = 1 # Minimum size in MiB + max_size_mib = 5 # Maximum size in MiB + output_dir = "logs" # Directory to save the files + + create_files(n, min_size_mib, max_size_mib, output_dir) diff --git a/sample/s3z/requirements.txt b/sample/s3z/requirements.txt new file mode 100644 index 0000000..2bae547 --- /dev/null +++ b/sample/s3z/requirements.txt @@ -0,0 +1,4 @@ +argparse +boto3 +openziti +namesgenerator \ No newline at end of file diff --git a/sample/s3z/s3z.py b/sample/s3z/s3z.py new file mode 100644 index 0000000..06cd3de --- /dev/null +++ b/sample/s3z/s3z.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +import argparse +import os + +from boto3 import client + +import openziti + + +def configure_openziti(ziti_identity_file): + print("Configuring with", + f"identity file '{ziti_identity_file}'") + return openziti.load(ziti_identity_file) + + +def push_logs_to_s3(bucket_name, bucket_endpoint, + push_log_dir, object_prefix): + s3 = client(service_name='s3', endpoint_url=bucket_endpoint) + + for file_name in os.listdir(push_log_dir): + if file_name.endswith(".log"): + file_path = os.path.join(push_log_dir, file_name) + with openziti.monkeypatch(): + if object_prefix: + s3.upload_file(file_path, bucket_name, + f"{object_prefix}/{file_name}") + print(f"Uploaded {file_path} to", + f"{bucket_name}/{object_prefix}") + else: + s3.upload_file(file_path, bucket_name, file_name) + print(f"Uploaded {file_path} to {bucket_name}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--ziti-identity-file', required=True, + help='Ziti identity file') + parser.add_argument('--bucket-name', required=True, + help='S3 bucket name') + parser.add_argument('--bucket-endpoint', required=True, + help='S3 VPCEndpoint Interface URL') + parser.add_argument('--object-prefix', required=False, default='', + help='Object key prefix in bucket') + parser.add_argument('--push-log-dir', required=False, default='.', + help='Directory containing *.log files to upload') + args = parser.parse_args() + + sts = client('sts') + caller = sts.get_caller_identity() + print("\nAuthenticated to AWS as:", + f"UserId: {caller.get('UserId')}", + f"Account: {caller.get('Account')}", + f"Arn: {caller.get('Arn')}\n", sep="\n\t") + + configure_openziti( + args.ziti_identity_file, + ) + + push_logs_to_s3( + args.bucket_name, + args.bucket_endpoint, + args.push_log_dir, + args.object_prefix + )