# Services : AWS ## CLI #### Install (Linux) ```bash $ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" $ unzip awscliv2.zip $ sudo ./aws/install ``` #### Config ```bash $ aws configure AWS Access Key ID: AKIA25C4RJAKVEFSEHXC AWS Secret Access Key: **************************************** Default region name [None]: us-east-2 Default output format [None]: ``` Will result in... ``` # ~/.aws/config [default] ``` ``` # ~/.aws/credentials [default] aws_access_key_id = AKIA25C4RJAKVEFSEHXC aws_secret_access_key = **************************************** ``` #### Use ```bash $ aws help $ aws s3 help $ aws s3 ls mybucket --recursive --human-readable --summarize $ aws s3 cp s3://mybucket/ may9/ --recursive --exclude "*" --include "logs/2024-05-09*" $ aws s3api head-object --bucket mybucket --key "logs/2024-05-09_22-13-40_thingy3.csv" ``` ``` --include PATTERN --exclude PATTERN *: Matches everything ?: Matches any single character [sequence]: Matches any character in sequence [!sequence]: Matches any character not in sequence Later patterns supercede previous: --exclude "*" --include "*.txt" produces all txt files ``` ## SDK ```python import boto3 session = boto3.Session( aws_access_key_id="...", aws_secret_access_key="..." ) client = session.client( "s3" ) resource = session.resource( "s3" ) ``` #### S3 API ```python bucket = resource.Bucket( "BUCKET" ) bucket.Object( "KEY" ) # Object bucket.objects.all() # iter: ObjectSummary bucket.objects.delete() # iter: ObjectSummary bucket.objects.filter( Prefix="..." ) # iter: ObjectSummary bucket.download_file( key, path ) # download to file bucket.download_fileobj( key, fileobj ) # download to file-like object in bin mode # managed xfer, may use multiple threads obj.content_length obj.key obj.last_modified # datetime.datetime( tzinfo=tzutc() ) obj.metadata # { ... } obj.delete() obj.download_file( path ) # same as bucket.download_file() obj.download_fileobj( fileobj ) # same as bucket.download_fileobj() obj.get() # returns Response (see below) obj.upload_fileobj( fileobj ) # newer than put(), probably preferred objsum.key objsum.last_modified # datetime.datetime( tzinfo=tzlocal() ) objsum.meta # ResourceMeta( "s3", identifiers=[ "bucket_name", "key" ] ) objsum.size objsum.get() # returns Response (see below) ``` ###### Example Response From get() ```json { 'ResponseMetadata': { 'RequestId': 'CY1PWHEJW4GC2795', 'HostId': 'eNE/5PzeRl/DP/5XUl4BlOFxmNYpENC3vF9UpFopKXStZ8rRDU6GP+aGRTkHt0blzoW0CsaTqxQ=', 'HTTPStatusCode': 200, 'HTTPHeaders': { 'x-amz-id-2': 'eNE/5PzeRl/DP/5XUl4BlOFxmNYpENC3vF9UpFopKXStZ8rRDU6GP+aGRTkHt0blzoW0CsaTqxQ=', 'x-amz-request-id': 'CY1PWHEJW4GC2795', 'date': 'Tue, 28 May 2024 19:53:23 GMT', 'last-modified': 'Tue, 02 Jan 2024 15:17:13 GMT', 'etag': '"fdcae8ae8781a45089463d27983facdf"', 'x-amz-server-side-encryption': 'AES256', 'x-amz-meta-startposlat': '40.88555759920422', 'x-amz-meta-droneid': 'thingy36', 'x-amz-meta-startposlng': '-74.04421207917999', 'x-amz-meta-takeoff': '1704184325000', 'x-amz-version-id': 'null', 'x-amz-meta-timezone': 'America%2FNew_York', 'x-amz-meta-droneversion': 'v1.6.0-dev-31c0ba1a0c5f06090bb9427a47bc8b7816fa5d23', 'x-amz-meta-maxalt': '98.081', 'cache-control': 'no-store, max-age=0', 'x-amz-meta-landing': '1704185461000', 'x-amz-meta-battspent': '66', 'x-amz-meta-formatted_address': '18-20%20Lehigh%20St%2C%20Hackensack%2C%20NJ%2007601%2C%20USA', 'x-amz-meta-size': '2023020', 'x-amz-meta-distance': '1883.3069313293008', 'x-amz-meta-maxdist': '890.9045476989817', 'accept-ranges': 'bytes', 'content-type': 'binary/octet-stream', 'server': 'AmazonS3', 'content-length': '2023020' }, 'RetryAttempts': 0 }, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2024, 1, 2, 15, 17, 13, tzinfo=tzutc()), 'ContentLength': 2023020, 'ETag': '"fdcae8ae8781a45089463d27983facdf"', 'VersionId': 'null', 'CacheControl': 'no-store, max-age=0', 'ContentType': 'binary/octet-stream', 'ServerSideEncryption': 'AES256', 'Metadata': { 'startposlat': '40.88555759920422', 'droneid': 'thingy36', 'startposlng': '-74.04421207917999', 'takeoff': '1704184325000', 'timezone': 'America%2FNew_York', 'droneversion': 'v1.6.0-dev-31c0ba1a0c5f06090bb9427a47bc8b7816fa5d23', 'maxalt': '98.081', 'landing': '1704185461000', 'battspent': '66', 'formatted_address': '18-20%20Lehigh%20St%2C%20Hackensack%2C%20NJ%2007601%2C%20USA', 'size': '2023020', 'distance': '1883.3069313293008', 'maxdist': '890.9045476989817' }, 'Body': <botocore.response.StreamingBody at 0x736a9a04c2e0> } ``` #### S3 Cookbook ```python # get object without knowing existence try: response = self.obj.get() except s3.client.exceptions.NoSuchKey as e: return None # check if bucket exists try: self.conn.client.head_bucket( Bucket=self.name ) return True except botocore.client.ClientError as e: if e.response[ "ResponseMetadata"]["HTTPStatusCode" ] == 404: return False raise # re-raise unexpected error # check if object exists try: self.conn.client.head_object( Bucket=self.bucket.name, Key=self.name ) return True except botocore.client.ClientError as e: if e.response[ "ResponseMetadata" ][ "HTTPStatusCode" ] == 404: return False raise # re-raise unexpected error # retrieve object metadata from get() response metadata = response[ "Metadata" ] # where Watchtower stores "metadata" # read object contents from get() response for row in csv.reader( TextIOWrapper( response[ "Body" ] ) ): ... ``` ## References - [https://aws.amazon.com/cli/](https://aws.amazon.com/cli/ "https://aws.amazon.com/cli/") - [https://docs.aws.amazon.com/cli/latest/userguide/](https://docs.aws.amazon.com/cli/latest/userguide/ "https://docs.aws.amazon.com/cli/latest/userguide/") - [https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html "https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html") - [https://docs.aws.amazon.com/cli/latest/reference/](https://docs.aws.amazon.com/cli/latest/reference/ "https://docs.aws.amazon.com/cli/latest/reference/") - [https://docs.aws.amazon.com/lambda/latest/dg/with-s3-example.html](https://docs.aws.amazon.com/lambda/latest/dg/with-s3-example.html "https://docs.aws.amazon.com/lambda/latest/dg/with-s3-example.html") - [https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html "https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html") - [https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/bucket/index.html](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/bucket/index.html "https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/bucket/index.html") - [https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/object/index.html](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/object/index.html "https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/object/index.html") - [https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/objectsummary/index.html](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/objectsummary/index.html "https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/objectsummary/index.html")