Node.js + request + for loop : Runs twice

Question

I created a simple scraper using cheerio and request client but it doesn't work the way I want.

First I see all the "null returned, do nothing" messages on the terminal and then see the names, so I think it first checks all the urls that returns a null, then non-nulls.

I want it to run in the right order, from 1 to 100.

app.get('/back', function (req, res) {
  for (var y = 1; y < 100; y++) {
    (function () {
      var url = "example.com/person/" + y +;
      var options2 = {
        url: url,
        headers: {
          'User-Agent': req.headers['user-agent'],
          'Content-Type': 'application/json; charset=utf-8'
        }
      };
      request(options2, function (err, resp, body) {
        if (err) {
          console.log(err);
        } else {
          if ($ = cheerio.load(body)) {
            var links = $('#container');
            var name = links.find('span[itemprop="name"]').html(); // name
            if (name == null) {
              console.log("null returned, do nothing");
            } else {
              name = entities.decodeHTML(name);
              console.log(name);
            }
          }
          else {
            console.log("can't open");
          }
        }
      });
    }());
  }
});

Answer 1

If you are not using promises and you want to run the requests sequentially, then this is a common design pattern for running a sequential async loop:

app.get('/back', function (req, res) {
    var cntr = 1;

    function next() {
        if (cntr < 100) {
            var url = "example.com/person/" + cntr++;
            var options2 = {
                url: url,
                headers: {
                    'User-Agent': req.headers['user-agent'],
                    'Content-Type': 'application/json; charset=utf-8'
                }
            };
            request(options2, function (err, resp, body) {
                if (err) {
                    console.log(err);
                } else {
                    if ($ = cheerio.load(body)) {
                        var links = $('#container');
                        var name = links.find('span[itemprop="name"]').html(); // name
                        if (name == null) {
                            console.log("null returned, do nothing");
                        } else {
                            name = entities.decodeHTML(name);
                            console.log(name);
                        }
                    } else {
                        console.log("can't open");
                    }
                    // do the next iteration
                    next();
                }
            });
        }
    }
    // start the first iteration
    next();
});

If you want to make all the requests in parallel (multiple requests in flight at the same time) which will be a faster end result and then accumulate all the results in order at the end, you can do this:

// create promisified version of request()
function requestPromise(options) {
    return new Promise(function(resolve, reject) {
        request(options2, function (err, resp, body) {
            if (err) return reject(err);
            resolve(body);
        });
    });
}

app.get('/back', function (req, res) {
    var promises = [];
    var headers = {
        'User-Agent': req.headers['user-agent'],
        'Content-Type': 'application/json; charset=utf-8'
    };
    for (var i = 1; i < 100; i++) {
        promises.push(requestPromise({url: "example.com/person/" + i, headers: headers}));
    }
    Promise.all(promises).then(function(data) {
        // iterate through all the data here
        for (var i = 0; i < data.length; i++) {
            if ($ = cheerio.load(data[i])) {
                var links = $('#container');
                var name = links.find('span[itemprop="name"]').html(); // name
                if (name == null) {
                    console.log("null returned, do nothing");
                } else {
                    name = entities.decodeHTML(name);
                    console.log(name);
                }
            } else {
                console.log("can't open");
            }
        }
    }, function(err) {
        // error occurred here
    });

});