将多个 http.requests 中的 arrays 组合成一个数组

Question

I'm trying to wrap my head around why this loop of https.requests isn't working.

I'm trying to make 5 http requests every minute. Every http GET request calls the function processRequest() as a callback, and every processRequest() returns an array from the request, let's call it resultsArray. I'd like to be able to produce one array or object every minute that is the concatenation of all 5 "resultsArray"s from the 5 http requests, but I'm having trouble figuring out how to declare what variables where. Here is what I have tried:

The CronJob:

var CronJob = require('cron').CronJob;
var tripDataFinal= new Object();

function makeRequest (){
    var urlCounter = 0 ; 
    var feedsList = ["urlA", "urlB", "urlC", "urlD", "urlE"];
    feedsList.forEach((feedToCheck,i)=>{
        options = {
            host: 'host',
            path: feedToCheck 
        }   
        https.request(options, function(response){
            processRequest(response, i, urlCounter, resultsArrayAll);}).on('error', (e) => {
        console.log("Error!"); console.error(e);}).end();   
    })

}
new CronJob('* * * * * *', makeRequest
, null, true, 'America/New_York'); 

The function processRequest() defined in a separated require() file:

module.exports = {


processRequest : function(response, feedNumber, urlCounter, tripDataFinal) {
        urlCounter++;
        //do stuff to 'response' to produce a result called resultsArray
        resultsArrayAll["array"+urlCounter] = resultsArray;
        console.log(urlCounter);
        if (urlCounter == 5){
            urlCounter = 0 ; 
            return concat(resultsArrayAll.array0
                        , resultsArrayAll.array1
                        , resultsArrayAll.array2
                        , resultsArrayAll.array3
                        , resultsArrayAll.array4);
        }
        else return resultsArrayAll;

    }}

The problem is that 'urlCounter' (which is how I count which request of the 5 I'm on) doesn't seem to be advancing past 1. What (maybe multiple things) am I doing wrong?

Thanks

Answer 1

You can wrap your http.request in a Promise and then do Promise.all() , like this:

function makeRequest (){
    var urlCounter = 0 ; 
    var feedsList = ["urlA", "urlB", "urlC", "urlD", "urlE"];
    var requestPromises = feedsList.map(function(url){
        return new Promise(function (resolve, reject){
            options = {
                host: 'host',
                path: feedToCheck 
            }   
            https.request(options, function(response){
                resolve(response);
            }).on('error', (e) => {
                reject(e);
            }).end();   
        });
    });

    Promise.all(requestPromises)
        .then(function(resultsArray){
            console.log('Results:', resultsArray);
        })
        .catch(function(e) { console.error(e); });
}

So here, in resultsArray you will have 5 responses from 5 requests (in the order they were made), and you can do whatever you want with them here, ex passing the array to the processRequest() function or something.

Answer 2

You can write your logic here no need to rest the urlCounter inside processRequest

var CronJob = require('cron').CronJob

function makeRequest () {
  var urlCounter = 0
  var feedsList = ['urlA', 'urlB', 'urlC', 'urlD', 'urlE']
  var resultArray = []
  feedsList.forEach((feedToCheck, i) => {
    options = {
      host: 'host',
      path: feedToCheck
    }
    https.request(options, function (response) {
      const result = processRequest(response) // do processing of response and returns a array
      resultArray.concat(result) // concat all results here
    })
  })
}
new CronJob('* * * * * *', makeRequest, null, true, 'America/New_York')

If you want to use concated resultArray you need to use promises here once you done with the all the request you can resolve that promise with resultArray and can use it after it.