Node.js Mongodb-native driver connection sharing

Question

main.js

var http = require('http');
var UserModel = require('./models/user.js');
var server = http.createServer(function(req, res){
  UserModel.create({
  }), function(e, o){
    if(e) { console.log(e); } else {
    } console.log(o); }
  });
}).listen(3000);

connections.js

var mongo = require('mongodb');

module.exports = {
    dbMain: new mongo.Db('main', new mongo.Server('127.0.0.1', 27017, { auto_reconnect: true }, {})),
    dbLog: new mongo.Db('log', new mongo.Server('127.0.0.1', 27017, { auto_reconnect: true }, {}))
};

/models/user.js

var mongodb = require('mongodb');
var db = require('./connections.js').dbMain;

module.exports = {
  create: function(newData, callback){
    db.open(function(e, db){
      db.collection('users', function(e, collection){
        collection.insert(newData, callback);
      });
    });
  }
}

When I use the code above, the server crashes with the problem that, the SECOND time a request comes in, we still have the database connection opened, so lets add db.close to our Users.create function.

  create: function(newData, callback){
    db.open(function(e, db){
      db.collection('users', function(e, collection){
        collection.insert(newData, function(e, o){
          db.close(); // Voila.
          callback(e, o);
        });
      });
    });
  }

At this stage the server CAN still crash, because of multiple connections open, I don't understand why or how this can happen but it does.

How do I organize my project into models (I don't want to use Mongoose, my validation is done in a different layer not the model, so Mongoose would be an overkill for me)? Also how do I handle connections in the project?

Answer 1

you could have a library that wraps all this up nicely - it means that only one connection to the database will be opened and the same (open) connection will be returned for the second request - if you are getting 1000+ per second, this is a make-or-break issue (ie not re-opening the connection each request)...

users.js :

var connections = require('./connections.js');

var serverCache = connections('127.0.0.1', 27017); 

module.exports = {
  create: function(newData, callback){
    serverCache('main', 'users', function(e, collection){
      collection.insert(newData, callback);
    })
  }
}

connections.js

var mongo = require('mongodb');

// a mongo connection cache
// pass in host & port
// it returns a function accepting dbName, collectionName & callback
var mongoCache = function(host, port){

  // keep our open connections
  var mongoDatabases = {};

  var ensureDatabase = function(dbName, readyCallback){
    // check if we already have this db connection open
    if(mongoDatabases[dbName]){
      readyCallback(null, mongoDatabases[dbName]);
      return;
    }

    // get the connection
    var server = new mongo.Server(host, port, {auto_reconnect: true});

    // get a handle on the database
    var db = new mongo.Db(dbName, server);
    db.open(function(error, databaseConnection){
      if(error) throw error;

      // add the database to the cache
      mongoDatabases[dbName] = databaseConnection;

      // remove the database from the cache if it closes
      databaseConnection.on('close', function(){
        delete(mongoDatabases[dbName]);
      })

      // return the database connection
      readyCallback(error, databaseConnection);
    })
  }

  var ensureCollection = function(dbName, collectionName, readyCallback){

    ensureDatabase(dbName, function(error, databaseConnection){
      if(error) throw error;

      databaseConnection.createCollection(collectionName, function(error, collection) {
        if(error) throw error;

        // return the collection finally
        readyCallback(error, collection);
      })

    })
  }

  return ensureCollection;
}

module.exports = mongoCache;

Answer 2

I'm currently using a global connection with multiple http requests. In the past I created a complex library that was creating several connections to MongoDB and picking up one randomly for each connection.

Later I found that the native driver can do that for me, which is pretty neat. Currently I'm using a single object, and the driver chooses to which connection send the query.

var srvOpts = {
    auto_reconnect: true,
    poolSize: 10,
};

var conn = new Mongo.Server("localhost", 27017, srvOpts),
    db = new Mongo.Db("dbname", conn, {});

db.open(function (){});

As you can this is a great idea, I'm thinking to copy that idea into the Redis driver that I'm using, but I'm short on time so I doubt that I will do any time soon.