Multiple file upload to s3 node.js

Question

I am working on MEAN stack application and I am using AWS SDK to upload multiple files to S3. I am using busboy and AWS SDK.

Code:

var inputObj = {};
 var busboy = new Busboy({ headers: req.headers });
    // The file upload has completed
    busboy.on('finish', function() {
     console.log('Upload finished.....');
     var file = [];
     const file1 = req.files.clogo;
     const file2 = req.files.cbanner1;
     const file3 = req.files.cbanner2;
     const file4 = req.files.cbanner3;
     const file5 = req.files.cbanner4;
     const file6 = req.files.clongHeader;
     file.push(file1);
     file.push(file2);
     file.push(file3);
     file.push(file4);
     file.push(file5);
     file.push(file6);
     multipleUploadToS3(req.body.cname, file, function(fileName) {
            console.log("client file upload finished.....");
             if(fileName.length == 6){
                inputObj.clogo = fileName[0];
                inputObj.cbanner1 = fileName[1];
                inputObj.cbanner2 = fileName[2];
                inputObj.cbanner3 = fileName[3];
                inputObj.cbanner4 = fileName[4];
                inputObj.clongHeader = fileName[5];

                console.log(inputObj);
                var clientObj = new client(inputObj);
                clientObj.save(function(err, client) {
                    console.log("Client Saved.....");
                    if (err) {
                        return res.status(400).send({
                            message: errorHandler.getErrorMessage(err)
                        });
                    } else {
                        res.json(client);
                    }
                });    
             }
     });
    });
    req.pipe(busboy);

File Upload Method:

function multipleUploadToS3(client, file, callback) {
    console.log("multipleUpload to S3"); 
    console.log(client);
    console.log(file);

    let s3bucket = new AWS.S3({
        accessKeyId: IAM_USER_KEY,
        secretAccessKey: IAM_USER_SECRET,
        Bucket: BUCKET_NAME,
    });

    var fileNames = [];

    for(var i=0; i<file.length; i++){
        s3bucket.createBucket(function () {
            var params = {
            Bucket: BUCKET_NAME,
            Key: client+ '/' + file[i].name,
            Body: file[i].data,
            };
            s3bucket.upload(params, function (err, data) {
                if (err) {
                    console.log('error in callback');
                    console.log(err);
                }
                console.log('success');
                //console.log(data.key);
                fileNames.push(data.key);
                if(i == file.length){ callback(fileNames);}
            });
        });
    }

};

The issue: file upload is asynchronous so for example if file1 I am uploading is honest.jpg then I want multipleUploadToS3 method to return file name after its done uploading to S3. I am binding this in inputObj keys which will be saved to mongo db . so inputObj.logo should have logo.png in it not the banner image which is happening due to asynchronous call.

This is working for a single file but failing for multiple files.

Answer 1

The problem is because for loop is synchronous and file upload is asynchronous.

Take a look at this example below,

for(var i = 0; i<5;i++) {
  setTimeout(function(){ console.log(i); }, 100);
}

The above loop will print 5 for 5 times ie 55555.

This behaviour is because for loop gets executed immediately making i=5 and when timeout gets executed it prints "i" value 5 for 5 times. 5 times because setTimeout is pushed in the queue for five times.

There are two ways to solve the problem you are facing

1. You can use Recursion.
2. Use neo-async (async-parallel) lib to control the async flow of javascript(Nodejs). click here for neo-async lib

Hope this clears your doubt. Please comment for more info.