Running serial operations in node.js

Question

I have an async function that executes some shell commands in parallel

require("fs").readdir("./", function (error, folders) { // asynched
    require("underscore")._(folders).each(function (folder, folderKey, folderList) { // asynched
    r("child_process").exec("ls ./" + folder, function(error, stdout, stderr) {
           console.log("Cant put it here") // Will be run after the first execution is completed
        })
        console.log("Cant put it here either") // Will be run immediately before any execution is completed
    })
    console.log("Cant put it here either") // Will be run immediately before any execution is completed
})

I want to do something after those shell commands are executed but I cant figure out how to do this with async library. Those shell commands are executed in parallel, so there is no way to register a handler that is executed after all of them are executed.

Any ideas?

Answer 1

Using the async.js library: https://github.com/caolan/async

var fs = require('fs');
var async = require('async');
var exec = require('child_process').exec;

fs.readdir("./", function (error, folders) {
    async.forEach(folders, function (folder, callback) {
        exec("ls ./" + folder, function (error, stdout, stderr) {
            callback();
        });
    },
    function (error) {
        // this is called after all shell commands are complete
    })
});

Answer 2

Update

After you edited your question to show your actual code, this should work:

var ps = require('child_process');
var fs = require('fs');
var _ = require("underscore");

function listDirectory(dir, callback) {
    fs.readdir(dir, function (error, folders) {

        // simply keep track on how many of the ls's have finished
        var count = folders.length, done = 0;
        _(folders).each(function (folder, folderKey, folderList) {
            ps.exec("ls ./" + folder, function(error, stdout, stderr) {
                console.log('one more done');
                done++;
                if (done === count) {
                    callback();
                } 
            });  
        });
    })
}

listDirectory('./', function() {
    console.log('all done!');
});

When run:

one more done
one more done
one more done
one more done
one more done
one more done
one more done
one more done
one more done
one more done
one more done
one more done
all done!

Old answer

I suppose your using the child_process module.

You need to create a wrapper around your commands.

var spawn = require('child_process').spawn;


// this function will execute all the things in the cmds array
// after all the processes have exited, it will call the callback
function multiSpawn(cmds, callback) {
    var count = cmds.length;
    var done = 0;
    for(var i = 0; i < count; i++) {
        // spawn the process, modify this if you need to hook up further events
        // you could also pass in further functions to setup each spawn

        var ps = spawn(cmds[i].shift(), cmds[i]);
        ps.on('exit', function() {
            done++; // keep track on how many commands have finished
            if (done === count) {
                callback();
            } 
        });
    }
}

multiSpawn([
    ['ls', '/usr/lib', '-l'],
    ['sleep', '1'], // this will sleep 1 seconds
    ['ls', '/usr/share', '-a']

], function() {
    console.log('all done');
});

This will run all the 3 commands, the two ls will finish instantly, sleep will delay everything for 1 second, after that the callback gets called.

I guess this would also be possible with Futures , but that's most likely overkill in this situation.