How can I get node.js to return data once all operations are complete

Question

I am just learning server-side JavaScript so please bear with any glaring mistakes I've made.

I am trying to write a file parser that operates on HTML files in a directory and returns a JSON string once all files have been parsed. I started it with a single file and it works fine. it loads the resource from Apache running on the same machine, injects jquery, does the parsing and returns my JSON.

var request = require('request'),
    jsdom = require('jsdom'),
    sys = require('sys'),
    http = require('http');

http.createServer(function (req, res) {
    request({uri:'http://localhost/tfrohe/Car3E.html'}, function (error, response, body) {
        if (!error && response.statusCode == 200) {
            var window = jsdom.jsdom(body).createWindow();
            jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
              // jQuery is now loaded on the jsdom window created from 'body'
                var emps = {};
                jquery("tr td img").parent().parent().each(function(){
                    var step = 0;
                    jquery(this).children().each(function(index){
                        if (jquery(this).children('img').attr('src') !== undefined) {
                            step++;
                            var name = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
                            var name_parts = name.split(",");
                            var last = name_parts[0];
                            var name_parts = name_parts[1].split(/\u00a0/g);
                            var first = name_parts[2];
                            emps[last + ",_" + first] = jquery(this).children('img').attr('src');
                        }
                    });
                });
                emps = JSON.stringify(emps);
                //console.log(emps);
                res.writeHead(200, {'Content-Type': 'text/plain'});
                res.end(emps);


            });
        } else {
            res.writeHead(200, {"Content-Type": "text/plain"});
            res.end("empty");
            //console.log(response.statusCode);
        }
    });
}).listen(8124);

Now I am trying to extend this to using the regular file system (fs) and get all HTML files in the directory and parse them the same way and return a single combined JSON object once all files have been parsed. Here is what I have so far but it does not work.

var sys = require("sys"),
    fs = require("fs"),
    jsdom = require("jsdom"),
    emps = {};
    //path = '/home/inet/www/media/employees/';

readDirectory = function(path) {
    fs.readdir(path, function(err, files) {
        var htmlfiles = [];
        files.forEach(function(name) {
            if(name.substr(-4) === "html") {
                htmlfiles.push(name);
            }
        });
        var count = htmlfiles.length;
        htmlfiles.forEach(function(filename) {
            fs.readFile(path + filename, "binary", function(err, data) {
                if(err) throw err;
                window = jsdom.jsdom(data).createWindow();
                jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
                    jquery("tr td img").parent().parent().each(function(){
                        var step = 0;
                        jquery(this).children().each(function(index){
                            if (jquery(this).children('img').attr('src') !== undefined) {
                                step++;
                                var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
                                var name_parts = empname.split(",");
                                var last = name_parts[0];
                                var name_parts = name_parts[1].split(/\u00a0/g);
                                var first = name_parts[2]
                                emps[last + ",_" + first] = jquery(this).children('img').attr('src');
                            }
                        });
                    });
                });
            });
        });
    });
}

readDirectory('/home/inet/www/media/employees/', function() {
    console.log(emps);
});

In this particular case, there are 2 html files in the directory. If i console.log(emps) during the htmlfiles.forEach() it shows me the results from the first file then the results for both files together the way I expect. how do I get emps to be returned to readDirectory so i can output it as desired?

Completed Script

After the answers below, here is the completed script with a httpServer to serve up the detail.

var sys = require('sys'),
    fs = require("fs"),
    http = require('http'),
    jsdom = require('jsdom'),
    emps = {};



    var timed = setInterval(function() {
        emps = {};
        readDirectory('/home/inet/www/media/employees/', function(emps) {
        });
    }, 3600000);

readDirectory = function(path, callback) {
    fs.readdir(path, function(err, files) {
        var htmlfiles = [];
        files.forEach(function(name) {
            if(name.substr(-4) === "html") {
                htmlfiles.push(name);
            }
        });
        var count = htmlfiles.length;
        htmlfiles.forEach(function(filename) {
            fs.readFile(path + filename, "binary", function(err, data) {
                if(err) throw err;
                window = jsdom.jsdom(data).createWindow();
                jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
                    var imagecount = jquery("tr td img").length;
                    jquery("tr td img").parent().parent().each(function(){
                        var step = 0;
                        jquery(this).children().each(function(index){
                            if (jquery(this).children('img').attr('src') !== undefined) {
                                step += 1;
                                var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
                                var name_parts = empname.split(",");
                                var last = name_parts[0];
                                var name_parts = name_parts[1].split(/\u00a0/g);
                                var first = name_parts[2]
                                emps[last + ",_" + first] = jquery(this).children('img').attr('src');
                            }
                        });
                    });
                    count -= 1;
                    if (count <= 0) {
                        callback(JSON.stringify(emps));
                    }
                });
            });

        });
    });
}

var init = readDirectory('/home/inet/www/media/employees/', function(emps) {

        });


http.createServer(function (req, res) {
    res.writeHead(200, {'Content-Type': 'text/plain'});
    res.end(JSON.stringify(emps));
}).listen(8124);

Answer 1

That sure is a lot of code a couple of mistakes.

1. You're never calling the callback function you supply to readDirectory
2. You need to keep track of the files you have parsed, when you parsed all of them, call the callback and supply the emps

This should work:

var sys = require("sys"),
    fs = require("fs"),
    jsdom = require("jsdom"),
    //path = '/home/inet/www/media/employees/';

// This is a nicer way
function readDirectory(path, callback) {
    fs.readdir(path, function(err, files) {

        // make this local
        var emps = {};
        var htmlfiles = [];
        files.forEach(function(name) {
            if(name.substr(-4) === "html") {
                htmlfiles.push(name);
            }
        });

        // Keep track of the number of files we have parsed
        var count = htmlfiles.length;
        var done = 0;
        htmlfiles.forEach(function(filename) {
            fs.readFile(path + filename, "binary", function(err, data) {
                if(err) throw err;
                window = jsdom.jsdom(data).createWindow();
                jsdom.jQueryify(window, 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js', function (window, jquery) {
                    jquery("tr td img").parent().parent().each(function(){
                        var step = 0;
                        jquery(this).children().each(function(index){
                            if (jquery(this).children('img').attr('src') !== undefined) {
                                step++;
                                var empname = jquery(this).parent().next().next().children('td:nth-child('+step+')').children().children().text();
                                var name_parts = empname.split(",");
                                var last = name_parts[0];
                                var name_parts = name_parts[1].split(/\u00a0/g);
                                var first = name_parts[2]
                                emps[last + ",_" + first] = jquery(this).children('img').attr('src');
                            }
                        });
                    });
                    // As soon as all have finished call the callback and supply emps
                    done++;
                    if (done === count) {
                        callback(emps);
                    }   
                });
            });
        });
    });
}

readDirectory('/home/inet/www/media/employees/', function(emps) {
    console.log(emps);
});

Answer 2

You seem to be doing this a tad wrong

readDirectory('/home/inet/www/media/employees/', function() {
    console.log(emps);
});

But you've defined your function as:

readDirectory = function(path) {

Where is the callback argument? Try this:

readDirectory = function(path, callback) {

then under emps[last + ",_" + first] = jquery(this).children('img').attr('src'); put

callback.call(null, emps);

Your callback function will be called however many times your loop goes on for. If you want it to return all of them at once, you'll need to get a count of how many times the loop is going to run for, count up until that number then call your callback when the emps array is full of the data you need.