Iteration of http requests in nodejs

Question

I am trying to make multiple http get requests in nodeJs. I need to continue getting records until there are no more available. I am having problems trying to do the iteration of a chunk of code. I had been reading about the async module but I am not understanding how to apply it. Please help.

Here is the New code 11/25/2014 : The step I want to repeat is the GetUrl(newBlock) is the step I want to repeat until there is no more blocks. I determine that there is no more blocks when the result.lenght of the request response is less than 5,000.

Is there a way of me getting this parameter pack from newBlock function and then stop the loop if I use the whilst async module?

// JS Script:        GPSAPIClient
// Code Description: API client code to retrieve GPS data for customer with account number: 47631
//                   This script requires Node.Js environment installed with Request() and Node-crontab modules (through NPM).
//                   It will run (as a backend script) every minute and retrieve the GPS available.
//
// Author :          Vanessa Torres, Professional Services
var request = require("request");
var fs = require('fs');
var async = require('async');
var crontab = require('node-crontab');
var csvFile = "";


var debug = false;
process.argv.forEach(function(val, idx, array) {
    if (val.toLowerCase() === 'debug') {
        debug = true;
        console.log('\n\n\n******************DEBUG MODE ACTIVE(Do not run in prod)******************\n\n\n')
    }
});

console.log('waiting to start running');


var jobId = crontab.scheduleJob('* * * * *', function() {
    //Gettting  system date and adding leading zeros when are single digits.  I need this to build the get request with date filters.


    var d = new Date();
    var nday = d.getDate();
    var nmonth = d.getMonth();
    var nhour = d.getHours();
    var nmin = d.getMinutes();

    var nfullyear = d.getFullYear();
    if (nday < 10) {
        nday = '0' + nday;
    };
    //1 minute substraction except for when is zero (the value will negative).  
    if (nmin != 0) {
        var nmin = nmin - 1;
    };
    if (nmin < 10) {
        nmin = '0' + nmin;
    };
    // added because TLC Plumbing is 2 hours behind us. (MST)
    nhour = nhour - 2;
    if (nhour < 10) {
        nhour = '0' + nhour;
    };

    var nmonth = nmonth + 1;
    if (nmonth < 10) {
        nmonth = '0' + nmonth;
    };

    var options = {
        //url: 'https://credentials@api.comettracker.com/v1/gpsdata' + '?fromdate=' + nfullyear + '-' + nmonth + '-' + nday + 'T' + nhour + '%3a' + nmin + '%3a' + '00',
        url: 'https://credentials@api.comettracker.com/v1/gpsdata?fromdate=2015-11-23T17%3a00%3a00&todate=2015-11-24T10%3a00%3a00',
        method: 'GET',
        rejectUnauthorized: !debug
    };

    function GetUrl(callback) {

        return request(options, function(error, response, body) {
            console.log('request for links');
            if (error) throw new Error(error);
            var result = JSON.parse(body)['links'];
            var urlnext = result[2].href;
            console.log('UrlNext: ' + urlnext);
            console.log(result[2].rel);
            //moreBlocks = newBlock(urlnext);

            moreBlocks = callback(urlnext);
        });
        // request to get the next block of records (maximun 5,000)
    };


    // HTTP get request - 1st request
    request(options, function(error, response, body) {
        if (error) throw new Error(error);
        var result = JSON.parse(body)['gps-recs'];
        console.log(result.length);
        //console.log(result);


        //create .csv file if result.length is > = 1 (If we received GPS records)
        if (result.length >= 1 && result.length < 5000) {
            console.log('entered first if, result.length: ' + result.length);
            buildCSV(result);
        };

        //add the subsequent request when result.length = 5000

        if (result.length == 5000) {
            console.log('entered second if, result.length: ' + result.length);
            buildCSV(result);
            console.log('I came back from buildCSV.  result.length is : ' + result.length);
            **GetUrl(newBlock)**;
            //console.log('moreblocks back from newblock: ' + moreBlocks);
        };
    });
});

function newBlock(urlnext) {
    console.log('URL passed to newBlock: ' + urlnext);
    // option 2 - variables needed to built the url (like the first request)
    var n = urlnext.length;
    var rest = urlnext.substr(40, n);
    console.log('Rest: ' + rest);

    var options = {
        url: 'https://credentials@api.comettracker.com/v1/gpsdata/' + rest,
        method: 'GET',
        rejectUnauthorized: !debug
    };

    request(options, function(error, response, body) {
        console.log('nextblock request');
        if (error) throw new Error(error);
        var result = JSON.parse(body)['gps-recs'];
        if (result.length == 0) {
            //no records for filter
            console.log('first if' + result.length);
            moreBlocks = false;
        };

        if (result.length < 5000 && result.length > 0) {
            //last block appends record and ends
            appendCSV(result);
            moreBlocks = false;
            console.log('second if' + result.length);
        };
        if (result.length == 5000) {
            //appends records but has to keep running
            appendCSV(result);
            console.log('third if- moreBlocks' + result.length);
        };
        console.log('moreBlocks: ' + moreBlocks);
    });

};

function buildCSV(result) {

    var csvFile = " ";
    console.log('before csvFile: ' + csvFile);
    //adding headers
    csvFile = csvFile.concat('UserNumber' + ',' + 'UserTimeTag' + ',' + 'Latitude' + ',' + 'Longitude' + ',' + 'SpeedMph' + ',' + 'Heading' + ',' + 'Status' + '\r\n');
    // loop runs result.length times
    for (var i = 0; i < result.length; i++) {
        csvFile = csvFile.concat(result[i].UserInfo.UserNumber + ',' + result[i].UserTimeTag + ',' + result[i].Latitude + ',' + result[i].Longitude + ',' + result[i].SpeedMph + ',' + result[i].Heading + ',' + result[i].Status + '\r\n');

    };
    //console.log(csvFile);  
    fs.writeFile('file.csv', csvFile, function(err) {

        if (err) throw err;
        console.log('file saved');

    });
};



//appending additional blocks 
function appendCSV(result) {
    var csvString = " ";
    // loop runs result.length times
    for (var i = 0; i < result.length; i++) {
        csvString = csvString.concat(result[i].UserInfo.UserNumber + ',' + result[i].UserTimeTag + ',' + result[i].Latitude + ',' + result[i].Longitude + ',' + result[i].SpeedMph + ',' + result[i].Heading + ',' + result[i].Status + '\r\n');

    };
    // console.log(csvString);  
    fs.appendFile('file.csv', csvString, function(err) {

        if (err) throw err;
        console.log('Data appended');

    });
};

Answer 1

It is difficult to see if moreBlocks is in the same scope as the while . On the other hand, the while statement will start making calls indefinetilly until that flags changes, and that might be too late. You need to make sure you do each subsequent request at a time.

You could use async's ´whilst´ function:

var moreBlocks = true;

whilst(function(){
  return moreBlocks === true;
},function(callback){
  return request(options, function(err, res, body){
    console.log('request for links');
    if (err) throw new Error(err);
    var result = JSON.parse(body)['links'];
    var urlnext = result[2].href;
    console.log('UrlNext: ' + urlnext);
    console.log(result[2].rel);

    //You NEED to make sure you update moreBlocks
    moreBlocks = newBlock(urlnext);

    // Go to next step.
    return callback(res);

  })
},{
  //You are done!
  //Call any function you'd like.
});

Check the docs here !

Edit, if newBlock is asynchronous:

var moreBlocks = true;

whilst(function(){
  return moreBlocks === true;
},function(callback){
  return request(options, function(err, res, body){
    console.log('request for links');
    if (err) throw new Error(err);
    var result = JSON.parse(body)['links'];
    var urlnext = result[2].href;
    console.log('UrlNext: ' + urlnext);
    console.log(result[2].rel);

    //Now status has the info about moreBlocks.
    return newBlock(urlnext, function(status){
      moreBlocks = status;
      // Go to next step.
      return callback(res);
    });       

  })
},{
  //You are done!
  //Call any function you'd like.
});

Somewhere on newBlock:

function newBlock(urlnext, callback) {
      // some code..
      return request(options, function(){
        //here status decides if we are done.
        return callback(status);
      });
    }