aws copy dir recursively from one location to another location in same s3 bucket

Question

As the title describe, how to copy dir recursively from one location to another location in same s3 bucket? I am using the java api, I noticed that the class TransferMnager could help to copy from s3 to s3, but I can't find a api that allow to copy dir recursively. Could anyone helps me? Thanks

Answer 1

There is not any Java API for copying recursively. You can use AWS CLI with --recursive flag but it traverses and copies all files under the hood.

Moreover, you can implement something like;

public void run() throws ExecutionException {
    String bucketName = "someBucket";
    String folderPath = "folder/path";
    String sse = "AES256";
    setEncryptionForBucket(bucketName, folderPath, sse);
}

private void setEncryptionForBucket(String bucketName, String folderPath, String sse) throws ExecutionException {
    LOGGER.info("Setting encryption for: [" + bucketName + "] with encryption type: [" + sse + "] and " + (folderPath != null ? "with folder path: /" + folderPath : ""));
    AmazonS3 s3Client = factory.create(AmazonS3Client.class);
    TransferManager transferManager = TransferManagerBuilder.standard().build();
    List<String> failedKeys = new ArrayList<>();

    StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    ParallelTaskExecutor parallelTaskExecutor = new ParallelTaskExecutor(THREAD_COUNT);
    s3Client.listObjectsV2(bucketName, folderPath)
            .getObjectSummaries().stream().map(S3ObjectSummary::getKey).forEach(key ->
            parallelTaskExecutor.submit(() -> setEncryptionForKey(transferManager, bucketName, key, failedKeys))
    );

    parallelTaskExecutor.waitForTasks();
    transferManager.shutdownNow();
    LOGGER.info("Task executed in " + stopWatch.getTime() / 1000 + " sec.");

    if (!failedKeys.isEmpty()) {
        LOGGER.error("Some of the keys cannot be encrypted: " + failedKeys);
        throw new RuntimeException("Some of the keys cannot be encrypted: " + failedKeys);
    }

    LOGGER.info("All objects encrypted successfully.");
}

private void setEncryptionForKey(TransferManager transferManager, String bucketName, String key, List<String> failedKeys) {
    CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName, key);
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    copyObjectRequest.setNewObjectMetadata(objectMetadata);

    StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    try {
        Copy xfer = transferManager.copy(copyObjectRequest);
        xfer.waitForCompletion();
        LOGGER.info("Xfer setting encryption successfully done for: " + key + " in " + stopWatch.getTime() + " ms.");
    } catch (Exception e) {
        LOGGER.error("Xfer failed for: " + key, e);
        failedKeys.add(key);
    }
}