Async double callback in NodeJS loop

Question

I'm going to the next step of my webscraper today !

I'm already looping on an url array with async and I would loop again in this callback and wait for its exectution before restart.

I can not figure out how use two callback.

This is my code :

   var getWebData = function(url) {
   var data = [];
   async.eachSeries(url, function(urlSingle, cb) {
      request(urlSingle, function(err, resp, body) {
        if (!err) {
          var $ = cheerio.load(body);
          var categoriesURL = [];
            $('.ombre_menu li').each(function(i, element) {
              $(this).find('.nav_sous-menu_bloc li a').each(function(i, element) {
                categoriesURL.push('https://blabla' + $(this).attr('href'));
              })

              // I WANT TO LOOP on the categoriesURL array HERE

                var jsObject = { name : "", description : "", price: "", categorie: "", liter: "", kilo: "", pricePer: "", quantity: "", capacity: "", promotion: "", scrapingDate : "", url: "" };
                data.push(jsObject);
            })
        }
       cb();
      })
   }, function() {
    // this will rum when loop is done
    var json = JSON.stringify(data);
    fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err) {
        console.log('File successfully written!');
     });
   });
}

getWebData(url);
app.listen('8080');

Does anyone know how can I do ?

Thanks

Answer 1

You can use nested eachSeries. Like this:

var getWebData = function(url) {
   var data = [];
   async.eachSeries(url, function(urlSingle, cb) {
      request(urlSingle, function(err, resp, body) {
        if (!err) {
          var $ = cheerio.load(body);
          var categoriesURL = [];
            $('.ombre_menu li').each(function(i, element) {
              $(this).find('.nav_sous-menu_bloc li a').each(function(i, element) {
                categoriesURL.push('https://blablablac' + $(this).attr('href'));
              })
              async.eachSeries(caturl, function(categoriesURL, cb2) {
                      //Do whatever you want to do here
                      cb2();
              },  function() {
                 //You can apply if and else for err an according to that you can set your callback responce here
                 cb();
              };
            })
        }
      })
   }, function() {
    // this will rum when loop is done
    var json = JSON.stringify(data);
    fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err) {
        console.log('File successfully written!');
     });
   });
}

getWebData(url);
app.listen('8080');

Answer 2

Made couple of changes in your code:

1. Used .mapSeries in place of .eachSeries . This way you can get data from iterator function in same order as the input array. Means you'll get [4,9] for input [2,3] to a square function, never [9,4]
2. Broke code into functions so that each function does one specific task
3. Moved categoriesURL processing out of loop 1
4. Returning early. It improves code readability. if (err) return callback(err);

---

function getWebData(url) {

  // Using .mapSeries in place of .eachSeries as you seem to want to get data from iterator function
  async.mapSeries(url, processUrl, function(err, results) {
    // this will rum when loop is done
    var json = JSON.stringify(results);
    fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err) {
      console.error('Error', err);
      console.log('File successfully written!');
    });
  });
}

function processUrl(url, callback) {
  request(url, function(err, resp, body) {
    if (err) // Return simple cases early; Improves code readability
      return callback(err); // or return callback(); -- if you don't want to send error upwards

    var $ = cheerio.load(body);
    var categoriesURL = [];

    $('.ombre_menu li')
      .each(function(i, element) { // loop 1
        $(this)
          .find('.nav_sous-menu_bloc li a')
          .each(function(i, element) { // loop 2
            categoriesURL.push('https://blablablac' + $(this)
              .attr('href'));
          }) // loop 2 end

      }) // loop 1 end
    // I WANT TO LOOP ON THE categoriesURL ARRAY HERE
    // Using .mapSeries in place of .eachSeries for same above reason
    async.mapSeries(categoriesURL, processCategoryUrl, function(err, results) {
      if (err)
        return callback(err);
      // This function is called after process array categoriesURL

      // Do what you want here then call callback provided to this method
      return callback(null, results);
    })
  })
}

function processCategoryUrl(categoryUrl, callback) {
  // Just process categoryUrl here and call callback with error or results
  return callback();
}

getWebData(url);
app.listen('8080');