如何从Node.js中的Amazon S3存储桶同步下载文件

Question

I have to download multiple files from S3 bucket using node.js. For that I have to write a for loop & call the s3.getObject(param) method to download. After the files are downloaded I have to merge their contents.

I have written like this:

var fileContentList = new ArrayList();

for(i=0; i<fileNameList.length i++){
    s3.getObject({ Bucket: "my-bucket", Key: fileNameList.get(i) }, function (error, data) {
    if (error != null) {
      alert("Failed to retrieve an object: " + error);
    } else {
      alert("Loaded " + data.ContentLength + " bytes");
      fileContentList.add(data.Body.toString());
    }
  }
);
}

//Do merging with the fileContentList.

But as s3.getObject is an asynchronous call the current thread moves on & nothing gets added to the fileContentList while I am doing the merging.

How can I solve the problem? Any idea?
Is their any synchronous method in aws-sdk to download file?

Answer 1

Promises is better way,

var getObject = function(keyFile) {
    return new Promise(function(success, reject) {
        s3.getObject(
            { Bucket: "my-bucket", Key: keyFile },
            function (error, data) {
                if(error) {
                    reject(error);
                } else {
                    success(data);
                }
            }
        );
    });
}

var promises = [];
var fileContentList = new ArrayList();

for(i=0; i<fileNameList.length i++){
    promises.push(getObject(fileNameList.get(i)));
}

Promise.all(promises)
.then(function(results) {
    for(var index in results) {
        var data = results[index];
        fileContentList.add(data.Body.toString());
    }
    // continue your process here
})
.catch(function(err) {
    alert(err);
});

Answer 2

You can use here async each, it will download all the files in parallel. In this example the download will continue if some of the files failed, if you want to stop downloading the files once an error occurred, call the callback with the error, this will call to the final callback immediately.

async documentation

var async = require('async');
var fileContentList = new ArrayList();
function downloadS3Multiple(done){
    async.each([
            function (callback) {
                s3.getObject({Bucket: "my-bucket", Key: fileNameList.get(i)}, function (err, res) {
                    if (err) {
                        alert("Failed to retrieve an object: " + error);
                        callback();
                    }
                    else {
                        alert("Loaded " + data.ContentLength + " bytes");
                        fileContentList.add(data.Body.toString());
                        callback();

                    }
                })
            }
        ], function (err, results) {
            done(err, fileContentList)
        });
}

Answer 3

Keep track of downloads you initiated in a separate list and as each download completes check to see if they are all done.

var fileContentList = new ArrayList();
var completedList = new ArrayList();
// function to setDone and initiate merge if all download attempts completed.
function setDone(i) {
    completedList[i]=true;
    var allDone= true;
    for(i=0; i<completedList.length && allDone=completedList[i] && allDone; i++);
    if(allDone) {
       mergeFiles();
    }
}

// fill completedList with a false value for each fill to be downloaded
for(i=0; i<fileNameList.length;i++) completedList.add(false);

// initiate the downloads
for(i=0; i<fileNameList.length; i++){
    s3.getObject({ Bucket: "my-bucket", Key: fileNameList.get(i) }, function (error, data) {
    if (error != null) {
      alert("Failed to retrieve an object: " + error);
    } else {
      alert("Loaded " + data.ContentLength + " bytes");
      fileContentList.add(data.Body.toString());
    }
    setDone(i);
  }
);
}

A more elegant solution, if you only want to merge the files if all downloads complete successfully:

var fileContentList = new ArrayList();

for(i=0; i<fileNameList.length i++){
    s3.getObject({ Bucket: "my-bucket", Key: fileNameList.get(i) }, function (error, data) {
    if (error != null) {
      alert("Failed to retrieve an object: " + error);
    } else {
      alert("Loaded " + data.ContentLength + " bytes");
      fileContentList.add(data.Body.toString());
    }
    if(fileContentList.length==fileNameList.length) combineFiles();
  }
);
}

Answer 4

I have solved using this. Though I have not tried the answers of Alexander,Lena & Sébastian I believe each of the answers mentioned by them would also work in this case. Many many thanks to them for their quick reply:

Async.eachSeries(casCustomersList, function (customerName, next){
        if(casCustomersList.length>0 && customerName != customerId) {

            var paramToAws = {
                Bucket: bucketName,
                Key: folderPath +'applicationContext-security-' + customerName + '.xml'      //file name
            };
            AWSFileAccessManager.downloadFile(paramToAws, function (error, result) {
                if (error) {
                    next(error);
                } else {
                    customerApplicationContext.add(result.Body.toString());
                    next();
                }

            });
        } else{
            next();
        }

    }, function(err) {
       //Write the rest of your logic here to process synchronously as it is the callback function

    }