torch_snippets

# from torch_snippets.s3_loader2 import S3FileHandler

aws_access_key_id = "AKIAQFXXXXXXXX6CN"
aws_secret_access_key = "AC3XXXXZXXXXXXXXXXXXXXXXXXejfXXXXXXh"

mys3 = S3FileHandler(aws_access_key_id, aws_secret_access_key)

List all Buckets

To lists all the s3 buckets in s3 for given credentials

mys3.list_s3_buckets()

['buckettest0011',
 'candidate-proctoring',
 'sagemaker-ap-south-1-011528263565',
 'sagemaker-studio-011528263565-u1h3juay9nd',
 'sentiment-classification-fastapi']

List all file objects

List all files in an S3 bucket or within a specific prefix of the given bucket along with the file size.

:param bucket_name: str. Name of the S3 bucket.
:param key: str or None. Specific prefix to list files from, defaults to None.

mys3.list_s3_objects(bucket_name="buckettest0011")

{'attendee_db/sumanth.jpg': 170670,
 'attendee_db/test/test/line_profiling_results.txt': 921,
 'attendee_db/test/test/outer_function_profile.txt': 2845,
 'attendee_db/test_2.mp4': 16330195,
 'test/test': 921,
 'test/test/line_profiling_results.txt': 921,
 'test/test/outer_function_profile.txt': 2845}

S3 Folder Download

Download all files from an S3 bucket prefix to a local directory.

:param bucket_name: str. Name of the S3 bucket.
:param local_dir: str. Local directory to which files will be downloaded.
:param prefix: str or None. Prefix path of the folder in the bucket. If None, the whole bucket is downloaded.
:param verbose: bool. Display the download status

mys3.download_s3_folder(
    bucket_name="buckettest0011", local_dir=".", prefix="test/test", verbose=1
)

test/test/
Downloaded test/test/line_profiling_results.txt to ./test/line_profiling_results.txt
Downloaded test/test/outer_function_profile.txt to ./test/outer_function_profile.txt

S3 File Download

Download a specific file from an S3 bucket and optionally return its metadata.

:param bucket_name: str. Name of the S3 bucket.
:param key: str. The key of the file in the S3 bucket.
:param local_dir: str. Local directory to which the file will be downloaded.
:param metadata: bool. If True, return the file’s metadata; otherwise, return None.
:param verbose: bool.
:return: dict or None. Returns metadata of the file if metadata is True, otherwise None.

mys3.download_s3_file(
    bucket_name="buckettest0011",
    key="test/test/outer_function_profile.txt",
    local_dir=".",
    metadata=True,
)

{'ResponseMetadata': {'RequestId': 'D699DNH1XH4995EM',
  'HostId': 'R8MIFqVr0MyVvOwbfM+ZkrgLyxHsPTp8HCqC/x0L5gR+rr9NIQZcVFwJWsmidXJe+VZRclVnONw=',
  'HTTPStatusCode': 200,
  'HTTPHeaders': {'x-amz-id-2': 'R8MIFqVr0MyVvOwbfM+ZkrgLyxHsPTp8HCqC/x0L5gR+rr9NIQZcVFwJWsmidXJe+VZRclVnONw=',
   'x-amz-request-id': 'D699DNH1XH4995EM',
   'date': 'Wed, 16 Oct 2024 05:48:23 GMT',
   'last-modified': 'Tue, 15 Oct 2024 09:40:40 GMT',
   'etag': '"7c49753bd7d2109ce96bd2568ad8fbef"',
   'x-amz-server-side-encryption': 'AES256',
   'x-amz-meta-author': 'XXXXX',
   'accept-ranges': 'bytes',
   'content-type': 'binary/octet-stream',
   'server': 'AmazonS3',
   'content-length': '2845'},
  'RetryAttempts': 0},
 'AcceptRanges': 'bytes',
 'LastModified': datetime.datetime(2024, 10, 15, 9, 40, 40, tzinfo=tzutc()),
 'ContentLength': 2845,
 'ETag': '"7c49753bd7d2109ce96bd2568ad8fbef"',
 'ContentType': 'binary/octet-stream',
 'ServerSideEncryption': 'AES256',
 'Metadata': {'author': 'XXXXX'}}

Uploading file from local to s3 with/without metadata

Upload a file to an S3 bucket with optional metadata.

:param bucket_name: str. Name of the S3 bucket.
:param localfile_path: str. Local path to the file to be uploaded.
:param s3_key: str. S3 key (path within the bucket) where the file will be stored with file name included.
:param metadata: dict or None. Optional metadata for the file. Defaults to None.

mys3.upload_file_to_s3(
    bucket_name="buckettest0011",
    localfile_path="/home/user/Documents/line_profiling_results.txt",
    s3_key="test/test/line_profiling_results.txt",
)

File uploaded successfully to buckettest0011/test/test/line_profiling_results.txt

metadata = {"author": "xxxxx"}
mys3.upload_file_to_s3(
    bucket_name="buckettest0011",
    localfile_path="/home/user/Documents/line_profiling_results.txt",
    s3_key="test/test/line_profiling_results.txt",
    metadata=metadata,
)

File uploaded successfully to buckettest0011/test/test/line_profiling_results.txt

Now lets check by downloading the uploaded file if the metadata is present or not

mys3.download_s3_file(
    bucket_name="buckettest0011",
    key="test/test/line_profiling_results.txt",
    local_dir=".",
    metadata=True,
    verbose=1,
)

Downloaded test/test/line_profiling_results.txt to ./line_profiling_results.txt

{'ResponseMetadata': {'RequestId': 'D69ARVG7KASXKQH1',
  'HostId': 'Je/oIjsM1FAf2psIv4aoclG62HSr9CGpXR/zvagTThcupuCz5FsMdN7ecT243Of+/jH2mOCha30=',
  'HTTPStatusCode': 200,
  'HTTPHeaders': {'x-amz-id-2': 'Je/oIjsM1FAf2psIv4aoclG62HSr9CGpXR/zvagTThcupuCz5FsMdN7ecT243Of+/jH2mOCha30=',
   'x-amz-request-id': 'D69ARVG7KASXKQH1',
   'date': 'Wed, 16 Oct 2024 05:48:23 GMT',
   'last-modified': 'Wed, 16 Oct 2024 05:48:23 GMT',
   'etag': '"5a627cd11fe9a0ec5877b4a4f0f33a62"',
   'x-amz-server-side-encryption': 'AES256',
   'x-amz-meta-author': 'xxxxx',
   'accept-ranges': 'bytes',
   'content-type': 'binary/octet-stream',
   'server': 'AmazonS3',
   'content-length': '921'},
  'RetryAttempts': 0},
 'AcceptRanges': 'bytes',
 'LastModified': datetime.datetime(2024, 10, 16, 5, 48, 23, tzinfo=tzutc()),
 'ContentLength': 921,
 'ETag': '"5a627cd11fe9a0ec5877b4a4f0f33a62"',
 'ContentType': 'binary/octet-stream',
 'ServerSideEncryption': 'AES256',
 'Metadata': {'author': 'xxxxx'}}

Uploading entire folder from local to s3 with/without metadata

Upload all files in a local folder to an S3 bucket with optional metadata.

:param bucket_name: str. Name of the S3 bucket.
:param local_folder_path: str. Local path to the folder to be uploaded.
:param s3_prefix: str. S3 prefix (folder path within the bucket) where the files will be stored.
Defaults to the root of the bucket.
:param metadata: dict or None. Optional metadata for the files. Defaults to None.

mys3.upload_folder_to_s3(
    "buckettest0011", "/home/user/Documents/attendee_db", verbose=1
)

Uploaded attendee_db/sumanth.jpg to buckettest0011/attendee_db/sumanth.jpg
Uploaded attendee_db/test_2.mp4 to buckettest0011/attendee_db/test_2.mp4
Uploaded attendee_db/test/test/outer_function_profile.txt to buckettest0011/attendee_db/test/test/outer_function_profile.txt
Uploaded attendee_db/test/test/line_profiling_results.txt to buckettest0011/attendee_db/test/test/line_profiling_results.txt