Collecting paginated data of unknown size using promises

Question

I'm querying a REST-API to get all groups. Those groups come in batches of 50. I would like to collect all of them before continuing to process them.

Up until now I relied on callbacks but I'd like to use promises to chain the retrieval of all groups and then process the result-array further.

I just don't quite get how to replace the recursive functional call using promises.

How would I use A+ promises to escape the callback hell I create with this code?

function addToGroups() {
   var results = []

   collectGroups(0)

   function collectGroups(offset){
     //async API call
     sc.get('/tracks/'+ CURRENT_TRACK_ID +'/groups?limit=50&offset=' + offset , OAUTH_TOKEN, function(error, data){

       if (data.length > 0){
         results.push(data)
         // keep requesting new groups
         collectGroups(offset + 50)
       }
       // finished
       else {
         //finish promise
       }
     })
   }
 }

Answer 1

Using standard promises, wrap all of your existing code as shown here:

function addToGroups() {
    return new Promise(function(resolve, reject) {
        ...  // your code, mostly as above
    });
}

Within your code, call resolve(data) when you're finished, or reject() if for some reason the chain of calls fails.

To make the whole thing more "promise like", first make a function collectGroups return a promise:

function promiseGet(url) {
    return new Promise(function(resolve, reject) {
        sc.get(url, function(error, data) {
            if (error) {
                reject(error);
            } else {
                resolve(data);
            }
        });
    }
}
// NB: promisify-node can do the above for you

function collectGroups(offset, stride) {
    return promiseGet('/tracks/'+ CURRENT_TRACK_ID +'/groups?limit=' + stride + '&offset=' + offset , OAUTH_TOKEN);
}

and then use this Promise in your code:

function addToGroups() {
    var results = [], stride = 50;
    return new Promise(function(resolve, reject) {
        (function loop(offset) {
            collectGroups(offset, stride).then(function(data) {
                if (data.length) {
                    results.push(data);
                    loop(offset + stride);
                } else {
                    resolve(data);
                }
            }).catch(reject);
        )(0);
    });
}

Answer 2

This could work. I am using https://github.com/kriskowal/q promises.

  var Q = require('q');

  function addToGroups() {
     var results = []
     //offsets hardcoded for example
     var a = [0, 51, 101];
     var promises = [], results;
     a.forEach(function(offset){
        promises.push(collectGroups(offset));
     })
     Q.allSettled(promises).then(function(){
        promises.forEach(function(promise, index){
          if(promise.state === 'fulfilled') {
            /* you can use results.concatenate if you know promise.value (data returned by the api)
               is an array */
            //you also could check offset.length > 0 (as per your code)
            results.concatenate(promise.value); 
            /*
              ... do your thing with results ...
            */
          }
          else {
            console.log('offset',index, 'failed', promise.reason);
          }
        });
     });
   }

  function collectGroups(offset){
    var def = Q.defer();
    //async API call
    sc.get('/tracks/'+ CURRENT_TRACK_ID +'/groups?limit=50&offset=' + offset , OAUTH_TOKEN, function(error, data){
      if(!err) {
        def.resolve(data);
      }
      else {
        def.reject(err);
      }
    });
    return def.promise;
  }

Let me know if it works.

Answer 3

Here's complete example, using spex.sequence :

var spex = require("spex")(Promise);

function source(index) {
    return new Promise(function (resolve) {
        sc.get('/tracks/' + CURRENT_TRACK_ID + '/groups?limit=50&offset=' + index * 50, OAUTH_TOKEN, function (error, data) {
            resolve(data.length ? data : undefined);
        });
    });
}

spex.sequence(source, {track: true})
    .then(function (data) {
        // data = all the pages returned by the sequence;
    });

I don't think it can get simpler than this ;)