Wait for all callback to complete before returning objects

Question

I have a function that needs to scrape a website and returns a list of addresses. In the callback from scrape, for each address returned I need to do another scraping operation then process the data, then I want to return the entire processed collection. I don't mind blocking if I have to. Ultimately I have to end up with a JSON object with the entire collection. Is this possible and how can I do this?

function doSomething(req, res){

    var collection = [];

    scrape1(params, function(error, addresses){
        if(!error){
            for(var i in addresses){   
                //do some stuff with addresses here

                scrape2(otherparams, function(error, address, data){
                    //manipulate the data here

                    collection.push({ 'address' : address, 'data' : data})  
                });
            }
            //this just returns an empty set obviously
            res.json(collection);

            //how can I return the entire collection from this function?
        }
    }); 
}

Answer 1

Here's one solution using the async module:

function doSomething(req, res){

  var collection = [];

  scrape1(params, function(error, addresses){
    if (error)
      return console.error(err); // handle error better

    async.each(addresses, function(address, callback) {
      scrape2(otherparams, function(err, address, data){
        // manipulate the data here
        if (err)
          return callback(err);
        collection.push({ 'address' : address, 'data' : data});
        callback();
      });
    }, function(err) {
      if (err)
        return console.error(err);  // handle error better
      res.json(collection);
    });
  }); 
}