node.js Async URL Retrieval to JSON

Question

The below code goes out and grabs a JavaScript variable from a URL for each of the keys in codes and then once all the variables are retrieved successfully it creates a JSON object from the cumulative results. This works great on the client side, but I want to do this processing (for many more key codes) on the server side every 15 minutes. I certainly could patch something together in Python to do this, but it seems a bit absurd given that the code is so easy in JS. I am not too familiar with node.js, but after reading about it I think it may be the best solution - although I'm sure there are other options. How would I go about queuing up the script retrievals so that I get a message when they are all done and can POST the JSON file to a storage location on the server? Can this be done on the server side with something like node exampleFile.js?

var codes = {'C': {}, 'S': {}, 'W': {}};
var keys = [];
for (var k in codes) keys.push(k);
var queue = keys.map(function (d) {
    var url = "http://www.agricharts.com/marketdata/jsquote.php?user=&pass=&allsymbols=" + d;
    return $.getScript(url)
        .done(function (e, textStatus) {
            codes[d] = qb; // qb is the name of the JS variable found in the URL
        });
    });

$.when.apply(null, queue).done(function () {
    var output = JSON.stringify(codes); // save this JSON file to the server for    processing on the client side
});

Answer 1

You could do an http get:

require('http');

var codes = {'C': {}, 'S': {}, 'W': {}};
var keys = [];
for (var k in codes) keys.push(k);
var counter = 0;
keys.map(function(d){
    var url = "http://www.agricharts.com/marketdata/jsquote.php?user=&pass=&allsymbols=" + d;
    http.get(url, function(res) {
        var scriptData = '';
        res.on('data', function (chunk) {
            scriptData+=chunk;
        });
        res.on('end',function(){
            counter++;
            eval(scriptData); 
            codes[d] = qb;
            if(counter==keys.length){
                var output = JSON.stringify(codes); // save this JSON file to the server for    processing on the client side
            }
        });
    });
});

after that you can queue and post the results to a server. check out: http://nodejs.org/api/http.html#http_http_get_options_callback

the V8 VM implements ECMAScript specification so a lot of the things you can do in the browser,except DOM manipulation, can be done in nodejs like the eval function