nodejs recursively fetch all pages from different REST end points

Question

I am kind of struggling with my JavaScript. Still getting my head around the callbacks. I have a function which recursively fetches the next link page from an REST endpoint.which works fine for a single url

Now i need to use the same recrusive function to do same for other end points. thats where iam struggling now. It is obvious that iam missing a point or i badly understood callback concept.

please excuse the formatting What i have done so far ..

function getFullLoad(rest_url,callback){
    mongoose.connect(url, function(err) {
    console.log(url);
    if (err) throw err;
    console.log("successfully connected");
    getToken(function(err, token) {
        if (err) throw console.error(err);
         console.log('using access token to retrieve data :', token[0]
          ['access_token']);
  var options = {
    'auth': {
      'bearer': token[0]['access_token']
    }
  } 
   request.get(rest_url, options, function(error, response, body) {
       if (!error && response.statusCode === 200) {
          var data = JSON.parse(body);
          //save data in mongo
         });
       }
      var next_page = JSON.parse(body)["_links"]["next"]
      if (next_page) {
        var next_page_url = JSON.parse(body)["_links"]["next"]["href"];
        console.log("Fetching data from ", next_page);
        //recrusively call back with next url
         getFullLoad(next_page_url,callback);
        } else {
           callback();
      } else {
         if (error) throw error;
      }
   });
rest_url= "http://myrest_url"
//this bit is working if there is a single url
getFullLoad(rest_url, function() {
console.log('done , got all data');
 });

this works great ,now i need to call the same function with different urls . It may be some urls may not have not pagination.

so far this is my effort , it only takes the first url and then does nothing. This bit is not working . what iam doing wrong? i tried creating another callback function. i guess my knowledge is quite limited on callbacks. any help very much appreciated

    api_endpoints =[ {"url1": "http://url1"},{"url2": "http://url2"}]
    api_endpoints.forEach(function(Item, endpoint) { 
    var endpoint_name = Object.keys(api_endpoints[endpoint]).toString());
    var rest_url = api_config.BASE_URL + api_endpoints[endpoint]   
                   [endpoint_name];
    getFullLoad(rest_url, function) {
     console.log('done');
         });
           });

Thank you for looking my question

Answer 1

With a bit of refactoring I think you can get something to work.

I would change your getFullLoad method to return a promise (the request.get) then in your foreach loop I would build up an array of promises that call the getFullLoad with the different urls.

Then use promises.all(arrayOfPromises) to execute them and manage the results.

It will be easier to understand and maintain long term.

Check out the promise documentation here:

Promise.all