aws s3 恢复文件夹的所有文件

Question

I have files archived on aws s3 glacier deep archive. I want to initiate the restoration of all objects starting with a prefix.

For that i first try to use de aws cli with this command:

aws s3api list-objects-v2 \
--bucket ${bucket} \
--prefix "${prefix}" \
--query "Contents[?StorageClass=='DEEP_ARCHIVE'].Key" \
--output text \
| sed 's/\t/\n/g' \
| xargs -I %%% \
aws s3api restore-object \
--restore-request Days=${days},GlacierJobParameters={"Tier"=\""${mode}"\"} \
--bucket ${bucket} \
--key "%%%"

I don't know why but some objects have initiated a restoration but others (the majority) have not.

So then i try to use python with the following code:

def restore_object(bucket,prefix,days,tier):
    s3 = boto3.resource('s3')
    client = boto3.client('s3')

    my_bucket = s3.Bucket(bucket)

    logfile = open("restoration.log","w")

    for object in my_bucket.objects.filter(Prefix=prefix):
        if object.storage_class == "DEEP_ARCHIVE":
            try:
                resp = client.restore_object(
                Bucket=bucket,
                Key=object.key,
                RestoreRequest={
                    'Days' : days,
                    'GlacierJobParameters' : {'Tier' : tier}
                    }
                )
                
            except Exception as e:
                logfile.write(f'For the object {object.key}, {e} \n')

But it's very long. 4 hours after the script is still running and many objects have still not initiated the restoration. There are about 70 000 objects in this folder.

Answer 1

As @John suggested i finally used a batch operation. For people interested here the code:

#!/usr/bin/env python3.9

import argparse
from urllib.parse import quote_plus
import os 
import boto3



def get_arguments():
    parser = argparse.ArgumentParser(
        description='''
        Restoration of objects from glacier deep archive on AWS s3

        This script create a manifest file with all object to restore
        and  upload this manifest to s3.

        Then a s3 batch job is created and run

        ''',
        formatter_class=argparse.RawTextHelpFormatter,
        usage='use "%(prog)s --help" for more information',)
    parser.add_argument(
        '--bucket',
        nargs=1,
        help='bucket name (default: %(default)s)',
        default='my-bucket',
        required=False)
    parser.add_argument(
        '--prefix',
        type=str,
        help='<Required> path of the folder to restore (without the name of the bucket)',
        required=True)
    parser.add_argument(
        '--days',
        type=int,
        help='number of days before deletion of the restored object copy (default: %(default)s)',
        default=2,
        required=False)
    parser.add_argument(
        '--mode',
        choices=['STANDARD','BULK'],
        default='STANDARD',
        help='''
        Acces tier option.
        Standard = restoration in 12h 
        Bulk = restoration in 48h 
        (default: %(default)s)
        ''',
        required=False)
    return parser.parse_args()






def create_manifest(bucket,prefix):
    '''Create a manifest file with all object to restore and upload this manifest to s3

    Parameters
    ----------
    bucket : str
        name of the bucket
    prefix : str
        path of the folder to restore (without the name of the bucket)

    Returns
    -------
    manifest_object
        object of the manifest file
    
    '''


    s3 = boto3.resource('s3')
    

    my_bucket = s3.Bucket(bucket)

    prefix_file_name = prefix.replace("/","_") if "/" in prefix else prefix

    manifest = open(prefix_file_name, "w")
    logfile = open("restoration.log","w")

    for object in my_bucket.objects.filter(Prefix=prefix):
        if object.storage_class == "DEEP_ARCHIVE":
            try:
                key_url_encode = quote_plus(object.key, safe = '/')
                manifest.write(f'{bucket},{key_url_encode}\n')
                
            except Exception as e:
                logfile.write(f'For the object {object.key}, {e} \n')

    manifest.close()




    if prefix.endswith('/'):
        manifest_key = prefix + 'Manifest2/Manifest.csv'
    else:
        manifest_key = prefix + '/Manifest2/Manifest.csv'

    my_bucket.upload_file(
        Filename=prefix_file_name,
        Key=manifest_key
        )

    os.remove(prefix_file_name)

    return s3.Object(bucket,manifest_key)


def create_job(manifest_object,bucket,days,tiers,prefix):
    '''Create a s3 batch job

    Parameters
    ----------
    manifest_object : object
        object of the manifest file
    bucket : str
        name of the bucket
    days : int
        number of days before deletion of the restored object copy
    tiers : str
        Acces tier option.
    prefix : str
        path of the folder to restore (without the name of the bucket)

    Returns
    -------
    None

    '''

    clientjobs = boto3.client('s3control')

    manifest_arn = 'arn:aws:s3:::' + bucket + '/' + manifest_object.key

    bucket_arn = 'arn:aws:s3:::' + bucket 

    if prefix.endswith('/'):
        report_key = prefix + 'report_batch_jobs'
    else:
        report_key = prefix + '/report_batch_jobs'

    response = clientjobs.create_job(
        AccountId='myaccountid',
        ConfirmationRequired=False,
        Operation={
            'S3InitiateRestoreObject': {
                'ExpirationInDays': days,
                'GlacierJobTier': tiers
                }
            },
        Report={
        'Bucket': bucket_arn,
        'Format': 'Report_CSV_20180820',
        'Enabled': True,
        'Prefix': report_key,
        'ReportScope': 'FailedTasksOnly'
        },
        Manifest={
            'Spec': {
                'Format': 'S3BatchOperations_CSV_20180820',
                'Fields': [
                    'Bucket','Key'
                    ]
        },
            'Location': {
                'ObjectArn': manifest_arn,
                'ETag': manifest_object.e_tag
        }
            },
        Priority=1,
        RoleArn='arnofiamrole'
        )




def main():
    bucket = get_arguments().bucket
    prefix = get_arguments().prefix

    manifest_object = create_manifest(bucket,prefix)

    days = get_arguments().days
    tiers = get_arguments().mode

    create_job(manifest_object,bucket,days,tiers,prefix)

main()

i just remove personal information (account id and iam role ARN)