Async issues for multiple POST request

Question

I use a function _getLogFileUrls that does a POST request and returns the data to a callback function. The returned data is being added to an array.

Function _getLogFileUrls is called in a loop.

How do i return the resultant array after all the requests have been completed?

I understand setTimeout is a wrong way for this approach and the way i am writing the code is creating a callback hell. How do i get rid of the setTimeout function without getting into all sorts of async issues.

// app.post('/api/getLogs', auth, loggingAPI.getLogs); // Make API request

function getLogs(req, res) {
    if ( ! req.body.id) {
        return res.status(500).send('Please check the params!');
    }

    var date;

    if (req.body.date) {
        date = req.body.date;
    } else {
        date = new Date().toISOString().slice(0,10);
    }

    var sqlQuery = "SELECT `LogFileID` FROM `logs_data` WHERE `EmpID` = '" + req.body.id + "' AND DATE(`Timestamp`) = '" + date + "'",
        resArray= [];

    hitThisQueryForMe(sqlQuery, res, function(rows) {
        if ( ! rows.length) res.json(rows);

        _.each(rows, function(item) {
            console.log('item: ' + item.LogFileID);
            _getLogFileUrls(item.LogFileID, function(response) {
                resArray.push(response);
            });
        });

        setTimeout(function() {
            console.log('final urls: ' + JSON.stringify(resArray));   
            res.send(resArray);
            resArray = [];
        }, 4000);

    });
}

function _getLogFileUrls(logFileId, callback) {
  var request = require('request'),
    config = require('../../config.js');    

    var fileParams = {
        fileName: 'LogFiles/' + logFileId       
    };

    request.post({
        url: config.filesServiceUrl + 'get-logfile-urls',
        json: fileParams
    }, function(error, response, body) {
        if (!error && response.statusCode === 200) {
            callback(body);
        } else {
            console.log('err requesting  logs: ' + JSON.stringify(error));
            res.status(400).send('Err requesting  logs:');
        }
    }).on('error', function(err) {
        console.log('File service error for Logs: ' + err);
    });
}

Answer 1

Use async.eachOf to iterate over your array and apply an async function on each element:

async.eachOf(myArray, function(myElementInArray, it, callback){

    // call async function on element
    myAsyncFunction(myElementInArray, function(err){
       if(err)
         return callback(err); // abort async flow
       else
         return callback(); 
    });

}, function(err){
  // final callback called when the flow is finished

});

//// EDIT

async module documentation : https://caolan.github.io/async/docs.html

Replace this code :

        _.each(rows, function(item){            
            console.log('item: ' + item.LogFileID);
            _getLogFileUrls(item.LogFileID, function(response){
                resArray.push(response);            
            });         
        });

        setTimeout(function(){
            console.log('final urls: ' + JSON.stringify(resArray)); 
            res.send(resArray);
            resArray = [];
        }, 4000);

With

 var async = require("async");
 var resArray = [];
 async.eachOf(rows, function(item, it, callback){

    console.log('item: ' + item.LogFileID);
    // call async function on element
     _getLogFileUrls(item.LogFileID, function(response){
         resArray.push(response);
         return callback(); 
    });

}, function(err){
  // final callback called when the flow is finished
   console.log('final urls: ' + JSON.stringify(resArray)); 
   res.send(resArray);
});