How to asynchronously do multiple REST API requests in node.js?

Question

What I want to do is do multiple remote REST API requests, where I want each request to be done sequentially, one after another. I read that async.js is the way to do this.

As I do not know how many times I have to do the request, I am using async.whilst() . The idea is that I will stop requesting once the API returns zero posts. Here is my code so far (for testing purposes I limited the loop to run for 5 times only).

var request = require('request');
var async   = require('async');

var APIKEY  = 'api-key-here';

var i = 0;
var continueWhilst = true;
async.whilst(
  function test() { return continueWhilst; },
  doThisEveryTime,
  function (err) {
    // Done
  }
);

function doThisEveryTime(next) {

  reqURL = 'http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=' + APIKEY + '&offset=' + i*20;  

  request(
    reqURL,
    function (err, resp, body) {
      if(!err && resp.statusCode == 200) {
        var resultAsJSON = JSON.parse(body);      
        console.log(reqURL);        
        console.log("Request #" + i + " done");
    }
  });

  i++;
  if (i===5) {
    continueWhilst = false;
  }
  console.log("Iterating, i= " + i);
  next();
}

The test output is like this:

Iterating, i= 1
Iterating, i= 2
Iterating, i= 3
Iterating, i= 4
Iterating, i= 5
http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=api-key-here&offset=80
Request #5 done
http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=api-key-here&offset=80
Request #5 done
http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=api-key-here&offset=80
Request #5 done
http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=api-key-here&offset=80
Request #5 done
http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=api-key-here&offset=80
Request #5 done

the request method is somehow only taking into account the final i value of 80. What did I do wrong, and how do I fix this?

Answer 1

The problem is occurring because your reqURL variable has no var , and does not belong to the doThisEveryTime scope. You are also making all your requests at once, since you are not waiting for a request to complete before calling next . These two things are leading the request to happen five times at the same time with the same URL. Try the following:

function doThisEveryTime(next) {
    var reqURL = 'http://api.tumblr.com/v2/blog/testsite.com/posts/text?api_key=' + APIKEY + '&offset=' + i*20;  

    request(reqURL, function (err, resp, body) {
        if (!err && resp.statusCode === 200) {
            var resultAsJSON = JSON.parse(body);      
            console.log(reqURL);        
            console.log("Request #" + i + " done");
        }

        i += 1;

        if (i === 5) {
            continueWhilst = false;
        }

        console.log("Iterating, i= " + i);

        // This is in the callback for request now.
        next();
    });
}