NODE.JS：致命错误 - JS分配失败 - 在解析大型json对象时处理内存不足

Question

I am trying to parse and add into mongodb database some data that am getting from an API. I want to get all the data for every user from specific time till today. So what I am doing is, I am retrieving for each user data for 5 days for each iteration, so its like 2-3 month data seperating into 5 days.

For some reason I am getting this error with the Allocation Failer - Process out of memory.

Seems like I get this error at the time when I arrive at a particular user, cz he seems having more data than others.

I did tried this command when running the script: node --max-old-space-size=4028 worksnap.js.

My code looks like this:

var currentMonth = new Date();
    var startDate = new Date("February 1, 2016 00:00:00");  //Start from February

var counter = 1;
while (startDate.getMonth() <= currentMonth.getMonth()) {
    //todo:: look if u have to increaze the start time, due the previous end time becomes start time it can take the same time time entries (have to be reviewd and make sure)....
    var from = new Date(startDate).getTime() / 1000;
    startDate.setDate(startDate.getDate() + 5);
    var to = new Date(startDate).getTime() / 1000;
    iterateThruAllStudents(from, to);
} 

function getTimeEntriesFromWorksnap(error, response, body) {
        //console.log(response.statusCode);
        if (!error && response.statusCode == 200) {
            parser.parseString(body, function (err, results) {
                var json_string = JSON.stringify(results.time_entries);
                var timeEntries = JSON.parse(json_string);
                _.forEach(timeEntries, function (timeEntry) {
                    _.forEach(timeEntry, function (item) {
                        saveTimeEntry(item);
                    });
                });
            });
        }
    }

    function saveTimeEntry(item) {
        Student.findOne({
                'worksnap.user.user_id': item.user_id[0]
            })
            .populate('user')
            .exec(function (err, student) {
                if (err) {
                    throw err;
                }
                student.timeEntries.push(item);
                student.save(function (err) {
                    if (err) {
                        console.log(err);
                    } else {
                        console.log('item inserted...');
                    }
                });

            });
    }

    function iterateThruAllStudents(from, to) {
        Student.find({status: 'student'})
            .populate('user')
            .exec(function (err, students) {
                if (err) {
                    throw err;
                }

                _.forEach(students, function (student, i) {
                    if (student.worksnap.user != null) {
                        setTimeout(function () {
                            var options = {
                                url: 'https://api.worksnaps.com/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
                                headers: {
                                    'Authorization': 'Basic bGhNSVwJkVUFasSxx2loOFVyZkFyOENEZEsdxxxCdUlHdElWMHo0czo='
                                }
                            };
                            request(options, getTimeEntriesFromWorksnap);
                        }, 5000 * i);
                    }
                });
            });
    }

Anyone knows what I am doing wrong here?

Answer 1

This is more a comment, as it does not contain a solution.

There are two things that looks fishy :

One problem is with:

while (startDate.getMonth() <= currentMonth.getMonth()) {
  //todo:: look if u have to increaze the start time, due the previous end time becomes start time it can take the same time time entries (have to be reviewd and make sure)....
  var from = new Date(startDate).getTime() / 1000;
  startDate.setDate(startDate.getDate() + 5);
  var to = new Date(startDate).getTime() / 1000;
  iterateThruAllStudents(from, to);
} 

You don't wait until you process the data of one student, but you request the data of all students in parallel.

A similar problem is the setTimeout , because depending on the execution time your code needs to hold the data of multiple requests in memory.

You should use something like async or Promise to solve asynchrone loops.