How do I download file from S3 bucket by parsing the object key as a parameter?

Question

I managed to list files of.txt extension from an S3 bucket and its specific subfolder prefix. However when I am parsing the object key as a variable to download the file I am getting an error.

from boto3.session import Session
import botocore
import boto3
import os

filepath = os.path.join("./Documents/AWS")

from subprocess import check_output

# Read the access key file

with open(os.path.join(filepath, "accessKeys.txt"), 'r', encoding='utf-8') as f:
line = f.readline().strip()
access_key = line.split(':')[0]
secret_key = line.split(':')[1]

session = Session(aws_access_key_id = access_key,
            aws_secret_access_key = secret_key,
            region_name='eu-west-1')

downloadpath = os.path.join("./Downloads")

# Download file from s3 Bucket

s3 = boto3.resource('s3')

#Bucket

bucket = s3.Bucket('terraform-state-181213')

#list objects within given prefix

objs = list(bucket.objects.filter(Prefix='terraform/'))
obj_key = []
or i in range(0, len(objs)):
print(objs[i].key)

for file in objs:
if file.key.endswith('.txt'):
    obj_key.append(file.key)
    obj_key = str(obj_key).strip('[]')
    print(obj_key)

'terraform/oasis_descriptor.txt'

# Download file parsing the bucket and obj_key as parameters
session.resource('s3').Bucket(bucket).download_file(obj_key,  os.path.join(downloadpath,'test.txt'))

Error:

Traceback (most recent call last):
File "/Users/vinitgohil/Documents/AWS/open_read_s3file.py", line 47, in <module>
s3.Bucket(bucket).download_file(obj_key,  os.path.join(downloadpath,'test.txt'))
File "/Users/vinitgohil/Library/Python/3.8/lib/python/site-packages/boto3/s3/inject.py", line 244, in bucket_download_file
return self.meta.client.download_file(
File "/Users/vinitgohil/Library/Python/3.8/lib/python/site-packages/boto3/s3/inject.py", line 170, in download_file
return transfer.download_file(
File "/Users/vinitgohil/Library/Python/3.8/lib/python/site-packages/boto3/s3/transfer.py", line 307, in download_file
future.result()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/s3transfer/futures.py", line 106, in result
return self._coordinator.result()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/s3transfer/futures.py", line 265, in result
raise self._exception
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/s3transfer/tasks.py", line 255, in _main
self._submit(transfer_future=transfer_future, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/s3transfer/download.py", line 340, in _submit
response = client.head_object(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/botocore/client.py", line 316, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/botocore/client.py", line 598, in _make_api_call
request_dict = self._convert_to_request_dict(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/botocore/client.py", line 644, in _convert_to_request_dict
api_params = self._emit_api_params(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/botocore/client.py", line 673, in _emit_api_params
self.meta.events.emit(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/botocore/hooks.py", line 356, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/botocore/hooks.py", line 228, in emit
return self._emit(event_name, kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/botocore/hooks.py", line 211, in _emit
response = handler(**kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/botocore/handlers.py", line 223, in validate_bucket_name
if not VALID_BUCKET.search(bucket) and not VALID_S3_ARN.search(bucket):
TypeError: expected string or bytes-like object

Answer 1

Here is sample code for downloading an Amazon S3 object.

Resource method

import boto3

s3_resource = boto3.resource('s3')

# Bucket, key, destination
s3_resource.Object('mybucket', 'hello.txt').download_file('/tmp/hello.txt')

Client method

import boto3

s3_client = boto3.client('s3')

# Provide bucket name, key, destination
s3_client.download_file('mybucket', 'hello.txt', '/tmp/hello.txt'))