Q promises and mongo db q.all

Question

I'm making calls to a mongodb database - pulling data out... reading it, and then making further requests based on that data. Once all the data has been received, I wish to process it.

I've been using Q.promises library but don't know what I'm doing. I thought that q.all would only trigger once everything has completed? However, my processPlaylist function runs twice. I've commented the code below:

Thanks, Rob

var PlaylistCollection = require('./models/playlist');
var AssetCollection = require('./models/asset');
var screenID = '############';
var playerData = [];
// array continaing playlistys which have been synced
var alreadySynced = [];

// Process our playlist once downloaded
    var processPlaylist = function (playerData) {
    console.log('----Processing Playerlist-----')
    console.log(playerData);
    // DO STUFF
}


// Get playlist by id. Return playlist Data
var getSubLists = function (id) {
    return PlaylistCollection.findById(id);
}



// Get sub-playlist function

function getSubListRecursive(id) {

return getSubLists(id).then(function (playlist) {

    // store all our returned playlist data into a playlist array
    playerData.push(playlist)

    // an Array to keep tabs on what we've already pulled down
    alreadySynced.push(playlist.id)

    // get all our playlist.resources, and only return those which are unique
    var playlistResources = _.uniq(playlist.resources, 'rid');
    //  console.log('Playlist Resources: ', playlistResources)
    //  console.log(alreadySynced);
    var sublists = _.pluck(_.filter(playlistResources, { 'type': 'playlist' }), 'rid');
    // remove playlists which have already been synced. We don't want to pull them down twice
    sublists = _.difference(sublists, alreadySynced);
    //  console.log('sublists: ', sublists)

    // Get the next playlist and so on...
    var dbops = sublists.map(function (sublist) {
        // console.log(sublist)
        return getSubListRecursive(sublist)
    });

    q.all(dbops).then(function () {
        console.log('All Done - so process the playlist')
        return processPlaylist(playerData);
    });

})

}


// Trigger the whole process..... grab our first playlist / ScreenID
getSubListRecursive(screenID);

and I get the following output:

    ----Processing Playerlist-----
[ { _id: 554d1df16ce4c438f8e2225b,
    title: 'list 1',
    __v: 29,
    daily: true,
    endTime: '',
    startTime: '',
    resources:
     [ { rid: '55650cebef204ab70302a4d9',
         title: 'list 4',
         type: 'playlist' },
       { rid: '554d1df16ce4c438f8e2225b',
         title: 'list 1',
         type: 'playlist' } ] },
  { _id: 55650cebef204ab70302a4d9,
    title: 'list 4',
    __v: 1,
    daily: false,
    endTime: '',
    startTime: '',
    resources:
     [ { rid: '55650647ef204ab70302a4d8',
         title: 'list 3',
         type: 'playlist' } ] } ]
All Done - so process the playlist
----Processing Playerlist-----
[ { _id: 554d1df16ce4c438f8e2225b,
    title: 'list 1',
    __v: 29,
    daily: true,
    endTime: '',
    startTime: '',
    resources:
     [ { rid: '55650cebef204ab70302a4d9',
         title: 'list 4',
         type: 'playlist' },
       { rid: '554d1df16ce4c438f8e2225b',
         title: 'list 1',
         type: 'playlist' } ] },
  { _id: 55650cebef204ab70302a4d9,
    title: 'list 4',
    __v: 1,
    daily: false,
    endTime: '',
    startTime: '',
    resources:
     [ { rid: '55650647ef204ab70302a4d8',
         title: 'list 3',
         type: 'playlist' } ] },
  { _id: 55650647ef204ab70302a4d8,
    title: 'list 3',
    __v: 5,
    daily: false,
    endTime: '',
    startTime: '',
    resources:
     [ { rid: '55650637ef204ab70302a4d7',
         title: 'list 2',
         type: 'playlist' },
       { rid: '554d1df16ce4c438f8e2225b',
         title: 'list 1',
         type: 'playlist' },
       { rid: '55650cebef204ab70302a4d9',
         title: 'list 4',
         type: 'playlist' } ] } ]    

EDIT

There were a number of things wrong with what I wrote. I discussed it with a buddy of mine - who pointed out that getSubListRecursive is being invoked recursively several times so the q.all statement is being executed several times...

So I refactored...

// Get sub-playlist function

function getSubListRecursive(id) {
    console.log(id)
    return getSubLists(id).then(function (playlist) {
        if (playlist) {
            // store all our returned playlist data into a playlist array
            playerData.push(playlist)
            // an Array to keep tabs on what we've already pulled down
            alreadySynced.push(playlist.id)
            // get all our playlist.resources, and only return those which are unique
            var playlistResources = _.uniq(playlist.resources, 'rid');
            //  console.log('Playlist Resources: ', playlistResources)
            //  console.log(alreadySynced);
            var sublists = _.pluck(_.filter(playlistResources, { 'type': 'playlist' }), 'rid');
            // remove playlists which have already been synced. We don't want to pull them down twice
            sublists = _.difference(sublists, alreadySynced);
            //  console.log('sublists: ', sublists)
            return sublists.map(function (sublist) {
                // console.log(sublist)
                if (sublists.length > 0) {
                    return getSubListRecursive(sublist)
                } else {
                    return processPlaylist(playerData);
                }
            });
        } else {
            return processPlaylist(playerData);
        }
    });
}

this works. I'm basically using promises to control the flow here - which probably isn't the best way of doing it? I no longer use an all statement, and ultimately end up with array populated with all the playlist data - which I can manipulate in my processPlaylist function.

However, I've not marked the question as solved, as I'd really like to know how I can do this with Q.all (properly use promises)

Thanks, Rob

Answer 1

I think you were just confused about when the entire process was finished. You need to wait until the entire recursive promise chain has resolved. I think you could use the original code with a slight change to where processPlaylist() is called:

var PlaylistCollection = require('./models/playlist');
var AssetCollection = require('./models/asset');
var screenID = '############';
var playerData = [];
// array continaing playlistys which have been synced
var alreadySynced = [];

// Process our playlist once downloaded
var processPlaylist = function (playerData) {
    console.log('----Processing Playerlist-----')
    console.log(playerData);
    // DO STUFF
}


// Get playlist by id. Return playlist Data
var getSubLists = function (id) {
    return PlaylistCollection.findById(id);
}

// Get sub-playlist function
function getSubListRecursive(id) {
    return getSubLists(id).then(function (playlist) {
        // store all our returned playlist data into a playlist array
        playerData.push(playlist)

        // an Array to keep tabs on what we've already pulled down
        alreadySynced.push(playlist.id)

        // get all our playlist.resources, and only return those which are unique
        var playlistResources = _.uniq(playlist.resources, 'rid');
        //  console.log('Playlist Resources: ', playlistResources)
        //  console.log(alreadySynced);
        var sublists = _.pluck(_.filter(playlistResources, { 'type': 'playlist' }), 'rid');
        // remove playlists which have already been synced. We don't want to pull them down twice
        sublists = _.difference(sublists, alreadySynced);
        //  console.log('sublists: ', sublists)

        // Get the next playlist and so on...
        var dbops = sublists.map(function (sublist) {
            // console.log(sublist)
            return getSubListRecursive(sublist)
        });

        return q.all(dbops);
    });
}

// Trigger the whole process..... grab our first playlist / ScreenID
getSubListRecursive(screenID).then(function() {
    console.log('All Done - so process the playlist')
    return processPlaylist(playerData);
});