Amazon S3 AWS upload progress listener

Question

有谁知道如何在Amazon S3的分段上传中查看上传进度（百分比）？

Answer 1

I would do it like this:

MultipleFileUpload transfer = transferManager.uploadDirectory(mybucket, null, new File(localSourceDataFilesPath), false);

// blocks the thread until the upload is completed
showTransferProgress(transfer);

Then in showTransferProgress , I would create a block the upload using a sleep, and do the math every X seconds:

private void showTransferProgress(MultipleFileUpload xfer) {
        while (!xfer.isDone()) {
            // some logic to wait so you don't do the math every second like a Thread.sleep

            TransferProgress progress = xfer.getProgress();
            long bytesTransferred = progress.getBytesTransferred();
            long total = progress.getTotalBytesToTransfer();
            Double percentDone = progress.getPercentTransferred();
            LOG.debug("S3 xml upload progress...{}%", percentDone.intValue());
            LOG.debug("{} bytes transferred to S3 out of {}", bytesTransferred, total);
        }

        // print the final state of the transfer.
        TransferState xferState = xfer.getState();
        LOG.debug("Final transfer state: " + xferState);
    }

this line is what you are looking for:

Double percentDone = progress.getPercentTransferred();

Answer 2

this works, but may be a better way to do this

TransferManager tm = TransferManagerBuilder
                .standard()
                .withS3Client(s3Client)
                .build();

        PutObjectRequest request = new PutObjectRequest(bucketName, file.getName(), file);

        ProgressListener progressListener = new ProgressListener() {
            @Override
            public void progressChanged(com.amazonaws.event.ProgressEvent progressEvent) {
                bytesUploaded += progressEvent.getBytesTransferred();// add counter
                if (bytesUploaded > byteTrigger) {
                    if ((bytesUploaded + sizeRatio) < fileSize) {
                        byteTrigger = bytesUploaded + sizeRatio ;
                    } else {
                        byteTrigger = bytesUploaded + (sizeRatio / 6);// increase precision approaching the end
                    }
                    String percent = new DecimalFormat("###.##").format(bytesUploaded * 100.0 / fileSize);
                    log.info("Uploaded: " + FileUtils.byteCountToDisplaySize(bytesUploaded) + " - " + percent + "%");
                }
            }
        };

        request.setGeneralProgressListener(progressListener);
        Upload upload = tm.upload(request);

        log.info("starting upload");

        upload.waitForUploadResult();

        log.info("Upload completed");

Answer 3

Hi guys here is is my final version

private void awsHoldUntilCompletedAndShowTransferProgress(Upload upload) throws InterruptedException, AmazonClientException {
        TransferProgress tProgress = upload.getProgress();
        long totalSize = tProgress.getTotalBytesToTransfer();
        long bPrevious = 0;
        int timerSec = Math.toIntExact(Math.round(Math.sqrt(totalSize / 1024 / 1024) / 4));// calculate based on file size
        if (timerSec > 60) {// no longer than 60 second per loop
            timerSec = 60;
        }
        while (!upload.isDone()) {
            long bTransferred = tProgress.getBytesTransferred();
            String strMbps = Double.valueOf((((bTransferred - bPrevious) / timerSec) / 1024) / 1024).toString() + " MBps";
            String strTransfered = bTransferred + " bytes";
            if (bTransferred > 1024) {
                strTransfered = Double.valueOf((bTransferred / 1024) / 1024).toString() + " MB";
            }
            log.info("Upload progress: "
                    + strTransfered
                    + " / "
                    + FileUtils.byteCountToDisplaySize(totalSize) + " - "
                    + Math.round(tProgress.getPercentTransferred()) + "% "
                    + strMbps);

            bPrevious = bTransferred;
            TimeUnit.SECONDS.sleep(timerSec);
        }

        Transfer.TransferState transferState = upload.getState();
        log.info("Final transfer state: " + transferState);

        if (transferState == Transfer.TransferState.Failed) {
            throw upload.waitForException();
        }
}

and here is where I call the code above from

..stuff...

TransferManager tm = TransferManagerBuilder
                        .standard()
                        .withS3Client(s3Client)                    
                        .build();

LocalDateTime uploadStartedAt = LocalDateTime.now();
log.info("Starting to upload " + FileUtils.byteCountToDisplaySize(fileSize));

Upload up = tm.upload(bucketName, file.getName(), file);

awsHoldUntilCompletedAndShowTransferProgress(up);

log.info("Time consumed: " + DurationFormatUtils.formatDuration(Duration.between(uploadStartedAt, LocalDateTime.now()).toMillis(), "dd HH:mm:ss"));