NodeJS Order Of Operations

Question

I'm trying to create a NodeJS application to pull SQL records and insert them into MongoDB. The tables I'm interested in are somewhat large (1million+ records). For small datasets (< 200,000) my app works great, but running against the full table starts to eat up RAM and bring the server to a crawl.

It looks like Node is running through my "for" loop, branching off processes for each SQL sub select, and then running the MongoDB updates.

I never see "Mongo Connected!" until the last "Getting Responses for Activity #" is written to the screen.

#!/var/node/bin/node
var odbc = require("odbc");
var db = new odbc.Database();

var MongoClient = require('mongodb').MongoClient;
var format = require('util').format;

db.open("DSN=<DSN>;SERVER=<SERVER>;DATABASE=<DB>;UID=<UID>;PWD=<PWD>", function (err) {
   if(err) throw err;
   console.log("SQL Connected!");

      var sqlstr = "SELECT TOP 1000 * FROM tbl_A NOLOCK";
      console.log("Executing '" + sqlstr + "' against SQL Server");

      db.query(sqlstr, function (sql1err, rows, moreResults) {
         if (sql1err) throw sql1err;

         for (var i = 0; i < rows.length; i++) {
            InsertActivity(db, rows[i], i, rows.length, function () {});
         }
      });
});

function InsertActivity(sql, activity, cur, total, callback) {
   console.log("Getting Responses for Activity #" + activity.ActivityID);
   var rsql = "SELECT * FROM tbl_Responses NOLOCK WHERE ActivityID = " + activity.ActivityID;
   sql.query(rsql, function (sqlerr, rows, moreResults) {
      if (sqlerr) console.log(sqlerr);
      activity.resonses = rows;

      MongoClient.connect('mongodb://localhost:27017/m', function (merr, mdb) {
         console.log("Mongo Connected!");
         mdb.collection("activity").insert(activity, function () {
            console.log("Inserted Activity #" + activity.ActivityID + " inserted into Mongo");
            mdb.close(function () { console.log("Mongo Disconnected!"); });
            callback();
         });
      });

      if (cur == total - 1) sql.close(function () { console.log("SQL Disconnected!"); });
   });

   console.log(rsql);
}

Answer 1

What you need is unfortunately an undocumented function (I'll fix that). The function is db.queryResult which returns the result object that allows you to fetch rows individually. That will avoid buffering the entire result set into memory.

https://github.com/wankdanker/node-odbc/blob/master/test/test-query-select-fetch.js

var db = require('odbc')();

db.open(connectionString, function (err) {
    db.queryResult('select * from tbl_A NOLOCK', function (err, result) {
        fetchMore();

        function fetchMore() {
            result.fetch(function (err, data) {
                if (!data) {
                    //we're all done, clean up
                }

                doStuffWithData(data, function (err) {
                    fetchMore();
                });
            });
        }
    });
});

function doStuffWithData(data, cb) {
    //do stuff
    cb(null);
}