Node.js: “process run out of memory”

Question

I have the following code which leads to the error: FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory

It doesn't make any difference whether I set --max_old_space_size=4096 (or bigger numbers) or not. I have 16 GB RAM on my PC.

System: win10/node.js 6.9.2/mongodb 3.4

I have over 16 000 000 messages in the "chatModel". With a smaller amount of messages the code works.

Do you have any suggestions how to solve the problem/optimize the code?

function sortOutMessages(){

var atSymbol = "@";
var rgx = new RegExp("^\\" +atSymbol);


chatModel.find({messageContent: rgx}, function (err, doc){

    var docs = doc;
    var docsLength = docs.length;

    for (var i =0; i<docsLength;i++) {
        var directedMessagesObj = new directedMessagesModel
        ({
            timeStamp: docs[i].timeStamp,
            channelName: docs[i].channelName,
            userName: docs[i].userName,
            userID: docs[i].userID,
            messageContent: docs[i].messageContent,
            messageLength: docs[i].messageLength,
            subscriber: docs[i].subscriber,
            turbo: docs[i].turbo,
            moderator: docs[i].moderator
        });

        directedMessagesObj.save({upsert:true}, function (err) {
            var fs = require('fs');
            if (err) {
                    fs.appendFile("undefinedLog.txt", "error at " + new Date().toLocaleString() + " directedMessagesObj.save " + "\r\n")
                    loggerWinston.warn("error at " + new Date().toLocaleString() + " directedMessagesObj.save " + "\r\n");
                    return console.log(err);
            }

        });
    }

});



}

Answer 1

You are creating docs.length amount of new Promise with this code. What you should be doing instead is to limit the amount of Promise that can be running.

Since what you have been trying to write is a synchronous code, I would advise calling the next dbSave in the callback of the previous one.

If you are after a parallel system, I would create a simple semaphore system.

Answer 2

I advise

 module.exports.asyncEach = function(iterableList, callback, done) {
  var i = -1,
    length = iterableList.length;

function loop() {
    i++;
    if (i === length) {
        done();
        return;
    }
    callback(iterableList[i], loop);
}
loop();
   };

then you call asyncEach with async save model is very beter

Answer 3

Thank you all for your answers! I decided to go a different way and it works for now:

var cursor = chatModel.find({messageContent: rgx}).cursor();

cursor.on('data',function (doc) {
    var directedMessagesObj = new directedMessagesModel
            ({
                timeStamp: doc.timeStamp,
                channelName: doc.channelName,
                userName: doc.userName,
                userID: doc.userID,
                messageContent: doc.messageContent,
                messageLength: doc.messageLength,
                subscriber: doc.subscriber,
                turbo: doc.turbo,
                moderator: doc.moderator
            });

    directedMessagesObj.save({upsert:true},function (err) {
        var fs = require('fs');
                    if (err) {
                            fs.appendFile("undefinedLog.txt", "error at " + new Date().toLocaleString() + " directedMessagesObj.save " + "\r\n");
                            loggerWinston.warn("error at " + new Date().toLocaleString() + " directedMessagesObj.save " + "\r\n");
                            return console.log(err);
                    }
    });
});