Complex addition to subdocument array with mongoose

Question

My data model has Accounts, and an accounts has some credit transactions for it. I have designed these transactions as subdocuments:

var TransactionSchema = new Schema({
        amount: Number,
        added: Date
    }),
    AccountSchema = new Schema({
        owner: ObjectId,
        balance: Number,
        transactions: [TransactionSchema]
    });

When a Transaction is added to an Account , the following should happen:

• transactions has the new one pushed to it
• transactions is sorted by date (for later display)
• balance is set to the sum of all transactions

I have put that in a Schema.methods -function for now, doing the above in JavaScript, before a save. However, I am not sure that is secure for multiple inserts at once.

How can this better be solved in Mongoose to use atomic or some kind of transactional update? In SQL, I would just do a transaction, but I cannot in MongoDB, so how to make sure that transactions and balance are always correct?

Answer 1

You can do all of that with a single update call that combines all three of those operations (which is the only way to make a combination of updates atomic). You don't sum the transactions during the update, instead you update balance with the amount of the change:

var transaction = {
    amount: 500,
    added: new Date()
};

Account.update({owner: owner}, {
        // Adjust the balance by the amount in the transaction.
        $inc: {balance: transaction.amount},
        // Add the transaction to transactions while sorting by added.
        $push: {transactions: {
            $each: [transaction],
            $sort: {added: 1}
        }}
    }, callback);

Note that this does utilize the $sort modifier for the $push which was added in 2.4 and updated in 2.6 so you need to be using a recent build.

Answer 2

Pretty much going along the same lines of the answer that just beat me to the punch, but I did have a longer explanation, so it takes a while.

So very much the same thing with one more tweak. Doing the transactions using the $inc operator on your balance is what you want rather than recalculating, so essentially this block of code:

  bucket.find({
    "account": accId, "owner": owner, "day": day
  }).upsert().updateOne(
    {
      "$inc": { "balance": amount },
      "$push": {
        "transactions": {
          "$each": [{ "amount": amount, "added": date }],
          "$sort": { "added": -1 }
        }
      }
    }
  );

The other "tweak" part is the buckets concept. While basically doing this with an array is a good idea and the use of $inc makes that part of the transaction atomic, the problem is you don't want lots of items in the array. And this will grow considerably over time.

The best way to do this is to only keep so many items in that array and limit these to "bucketed" results. I also have added a little more handling in here for at least "Attempting" to keep a singular "balance" point in sync, but in reality you probably want to verify that periodically, as the resulting multiple updates are not bound by on a transaction.

But the update on the "bucket" is atomic. Mileage may vary to actual implementation, but here is the essentinal demonstration code:

var async = require('async'),
    mongoose = require('mongoose'),
    Schema = mongoose.Schema;

mongoose.connect('mongodb://localhost/shop');

var ownerSchema = new Schema({
  name: String,
  email: String,
  accounts: [{ type: Schema.Types.ObjectId, ref: "Account" }]
});

var transactionSchema = new Schema({
  amount: Number,
  added: Date
});

var recentBucketSchema = new Schema({
  _id: { type: Schema.Types.ObjectId, ref: "AccountBucket" },
  day: Date
});

var accountSchema = new Schema({
  owner: { type: Schema.Types.ObjectId, ref: "Owner" },
  balance: { type: Number, default: 0 },
  recent: [recentBucketSchema]
});

var accountBucketSchema = new Schema({
  day: Date,
  account: { type: Schema.Types.ObjectId, ref: "Account" },
  owner: { type: Schema.Types.ObjectId, ref: "Owner" },
  balance: { type: Number, default: 0 },
  transactions: [transactionSchema]
});

var Owner = mongoose.model( "Owner", ownerSchema );
var Account = mongoose.model( "Account", accountSchema );
var AccountBucket = mongoose.model( "AccountBucket", accountBucketSchema );

var owner = new Owner({ name: "bill", emal: "bill@test.com" });
var account = new Account({ owner: owner });
owner.accounts.push(account);

var transact = function(accId,owner,amount,date,callback) {

  var day = new Date(
    date.valueOf() - (date.valueOf() % (1000 * 60 * 60 * 24)) );

  var bucket = AccountBucket.collection.initializeOrderedBulkOp();
  var account = Account.collection.initializeOrderedBulkOp();

  bucket.find({
    "account": accId, "owner": owner, "day": day
  }).upsert().updateOne(
    {
      "$inc": { "balance": amount },
      "$push": {
        "transactions": {
          "$each": [{ "amount": amount, "added": date }],
          "$sort": { "added": -1 }
        }
      }
    }
  );

  bucket.execute(function(err,response) {
      if (err) throw err;

      var upObj = {
        "$inc": { "balance": amount }
      };

      if ( response.nUpserted > 0 ) {
        var id = response.getUpsertedIds()[0]._id;
        upObj["$push"] = {
          "recent": {
            "$each": [{ "_id": id, "day": day }],
            "$sort": { "day": -1 },
            "$slice": 30
          }
        };
      }

      console.log( JSON.stringify( upObj, undefined, 4 ) );

      account.find({ "_id": accId }).updateOne(upObj);
      account.execute(function(err,response) {
        callback(err,response);
      });
    }
  );
};

mongoose.connection.on("open",function(err,conn) {

  async.series([

    function(callback) {
      async.each([Owner,Account,AccountBucket],function(model,complete) {
        model.remove(function(err) {
          if (err) throw err;
          complete();
        });
      },function(err) {
        if (err) throw err;
        callback();
      });
    },

    function(callback) {
      async.each([account,owner],function(model,complete) {
        model.save(function(err) {
          if (err) throw err;
          complete();
        });
      },function(err) {
        if (err) throw err;
        callback();
      });
    },

    function(callback) {
      var trandate = new Date();
      transact(account._id,owner._id,10,trandate,function(err,response) {
        if (err) throw err;

        console.log( JSON.stringify( response, undefined, 4 ) );
        callback();
      });
    },

    function(callback) {
      var trandate = new Date();
      trandate = new Date( trandate.valueOf() + ( 1000 * 60 * 60 * 1 ) );
      transact(account._id,owner._id,-5,trandate,function(err,response) {
        if (err) throw err;

        console.log( JSON.stringify( response, undefined, 4 ) );
        callback();
      });
    },

    function(callback) {
      var trandate = new Date();
      trandate = new Date( trandate.valueOf() - ( 1000 * 60 * 60 * 1 ) );
      transact(account._id,owner._id,15,trandate,function(err,response) {
        if (err) throw err;

        console.log( JSON.stringify( response, undefined, 4 ) );
        callback();
      });
    },

    function(callback) {
      var trandate = new Date();
      trandate = new Date( trandate.valueOf() - ( 1000 * 60 * 60 * 24 ) );
      transact(account._id,owner._id,-5,trandate,function(err,response) {
        if (err) throw err;

        console.log( JSON.stringify( response, undefined, 4 ) );
        callback();
      });
    },

    function(callback) {
      var trandate = new Date("2014-07-02");
      transact(account._id,owner._id,10,trandate,function(err,response) {
        if (err) throw err;

        console.log( JSON.stringify( response, undefined, 4 ) );
        callback();
      });
    },

  ],function(err) {

    String.prototype.repeat = function( num ) {
      return new Array( num + 1 ).join( this );
    };

    console.log( "Outputs\n%s\n", "=".repeat(80) );

    async.series([

      function(callback) {
        Account.findById(account._id,function(err,account) {
          if (err) throw err;

          console.log(
            "Raw Account\n%s\n%s\n",
            "=".repeat(80),
            JSON.stringify( account, undefined, 4 )
          );
          callback();
        });
      },

      function(callback) {
        AccountBucket.find({},function(err,buckets) {
          if (err) throw err;

          console.log(
            "Buckets\n%s\n%s\n",
            "=".repeat(80),
            JSON.stringify( buckets, undefined, 4 )
          );
          callback();
        });
      },

      function(callback) {
        Account.findById(account._id)
          .populate("owner recent._id")
          .exec(function(err,account) {
            if (err) throw err;

            var raw = account.toObject();

            raw.transactions = [];
            raw.recent.forEach(function(recent) {
              recent._id.transactions.forEach(function(transaction) {
                raw.transactions.push( transaction );
              });
            });

            delete raw.recent;

            console.log(
              "Merged Pretty\n%s\n%s\n",
              "=".repeat(80),
              JSON.stringify( raw, undefined, 4 )
            );
            callback();
          });
      }

    ],function(err) {
      process.exit();
    });

  });

});

This listing is using the "bulk" updates API functionality available to MongoDB 2.6, but you don't have to use it. It's just here to dump out the more meaningful responses from the updates.

The general case when "bucketing" the transactions is that you are going to split them up somehow. The base example here is "day", but possibly something else is more practical.

In order to ensure you create a new bucket when that identifier changes, the "upsert" functionality of MongoDB updates is used. This should be generally okay by itself as you could later get a running balance across all "buckets", but in this case we are going to at least "try" to keep an "Account" master in sync, if only for a little more demonstration.

After the update on the current bucket is complete, the response is checked to see if an "upsert" occurred. Under a legacy or mongoose API .update() this will just return the _id of the "upserted" document in a third argument in the callback.

Where an "upsert" occurs and a new bucket is created, we are also going to add that to the master "Account" as a list of recent buckets, in fact the 30 most recent. So this time the $push operation uses an additional $slice modifier to the other $each and $sort operations.

The last two need to be used together even though there is only one array element to add. MongoDB 2.4 versions actually require the $slice at all times with these modifiers, so if you if you do not really want to limit, set $slice to a large number, but it is good practice to restrict the length of the array.

In each case, the dates are sorted with the most recent first despite the order in which all the sample code inserts them. The output will show you in this form all of what actually happens in the write operations, but a summary of the general end results is here for reading purposes:

Outputs
========================================================================

Raw Account
========================================================================
{
    "_id": "53bf504ac0716cbc113fbac5",
    "owner": "53bf504ac0716cbc113fbac4",
    "__v": 0,
    "recent": [
        {
            "_id": "53bf504a79b21601f0c00d1d",
            "day": "2014-07-11T00:00:00.000Z"
        },
        {
            "_id": "53bf504a79b21601f0c00d1e",
            "day": "2014-07-10T00:00:00.000Z"
        },
        {
            "_id": "53bf504a79b21601f0c00d1f",
            "day": "2014-07-02T00:00:00.000Z"
        }
    ],
    "balance": 25
}

Buckets
========================================================================
[
    {
        "_id": "53bf504a79b21601f0c00d1d",
        "account": "53bf504ac0716cbc113fbac5",
        "day": "2014-07-11T00:00:00.000Z",
        "owner": "53bf504ac0716cbc113fbac4",
        "transactions": [
            {
                "amount": -5,
                "added": "2014-07-11T03:47:38.170Z"
            },
            {
                "amount": 10,
                "added": "2014-07-11T02:47:38.153Z"
            },
            {
                "amount": 15,
                "added": "2014-07-11T01:47:38.176Z"
            }
        ],
        "balance": 20
    },
    {
        "_id": "53bf504a79b21601f0c00d1e",
        "account": "53bf504ac0716cbc113fbac5",
        "day": "2014-07-10T00:00:00.000Z",
        "owner": "53bf504ac0716cbc113fbac4",
        "transactions": [
            {
                "amount": -5,
                "added": "2014-07-10T02:47:38.182Z"
            }
        ],
        "balance": -5
    },
    {
        "_id": "53bf504a79b21601f0c00d1f",
        "account": "53bf504ac0716cbc113fbac5",
        "day": "2014-07-02T00:00:00.000Z",
        "owner": "53bf504ac0716cbc113fbac4",
        "transactions": [
            {
                "amount": 10,
                "added": "2014-07-02T00:00:00.000Z"
            }
        ],
        "balance": 10
    }
]

Merged Pretty
========================================================================
{
    "_id": "53bf504ac0716cbc113fbac5",
    "owner": {
        "_id": "53bf504ac0716cbc113fbac4",
        "name": "bill",
        "__v": 0,
        "accounts": [
            "53bf504ac0716cbc113fbac5"
        ]
    },
    "__v": 0,
    "balance": 25,
    "transactions": [
        {
            "amount": -5,
            "added": "2014-07-11T03:47:38.170Z"
        },
        {
            "amount": 10,
            "added": "2014-07-11T02:47:38.153Z"
        },
        {
            "amount": 15,
            "added": "2014-07-11T01:47:38.176Z"
        },
        {
            "amount": -5,
            "added": "2014-07-10T02:47:38.182Z"
        },
        {
            "amount": 10,
            "added": "2014-07-02T00:00:00.000Z"
        }
    ]
}