NodeJS-内存不足：大数据处理时会杀死进程错误

Question

I have a couple of .csv files that I need to compare against another large .csv file (over 300,000 rows) and I am running into an Out of Memory error on my server. I am running this on a server with 4GB RAM so I am not sure why this is happening but my code looks like this. I am using the ya-csv to read in the csv lines:

var csv = require('ya-csv');
var fs = require('graceful-fs');
var async = require('async');


var first_silo = [];
var second_Silo = [];
var combined = [];

var reader = csv.createCsvFileReader('december_raw.csv', {columnsFromHeader:true,'separator': ','});
var first = csv.createCsvFileReader('first_data.csv', {columnsFromHeader:false,'separator': ','});
var second = csv.createCsvFileReader('second_data.csv', {columnsFromHeader:false,'separator': ','})


async.series([
 //push data from other .csv files into arrays
function(callback){
   first.addListener('data', function(data){
      first_silo.push(data[0]);
   })
   first.addListener('end', function(){
      callback();
   })
},

function(callback){
   second.addListener('data', function(data){
       second_silo.push(data[0]);
   });
   second.addListener('end', function(data){
       callback();
   });
},

function(callback){
    reader.addListener('data', function(data){
       //compare the data from reader to each item in the first array and append the items that get a match to a .csv.
       for(var i=0;i<first_silo.length;i++){
           if(data[0] === first_silo[i]){
               fs.appendFileSync('results.csv', data[0]+","+first_silo[i])
               break;
           }
       } 
    });
},

function(callback){
    reader.addListener('data', function(data){
        //do the same with the first array as the second.
        for(var i=0;i<second_silo.length;i++){
            if(data[0] === second_silo[i]){
               fs.appendFileSync('results.csv', data[0]+","+second_silo[i]);
               break;
            }
        }
    })
}
])

When I do this I dont get the past first_silo comparison. The node app will just stop and I can see an out of memory error when I dmesg.

I have tried to run this program with this flag as well:

--max-old-space-size=3000

I still get the same error.

Is there a smarter way to do this? Any help would be greatly appreciated.

Answer 1

Your algorithm is running pretty inefficient for a few reasons. Please forgive me, but I'm going to do this without using the async.series call you're using. Hopefully it will still be useful.

First thing's first: I'm making an assumption. I'm assuming that the data size of your first file december_raw.csv is smaller than your second and third files. Even if this isn't the case, this should still work without running out of memory as long as the file's contents aren't over your memory limitation.

Second, you're loading up two arrays at the same time instead of doing one at a time. This is basically doubling your memory usage.

Third, my hunch is that when you're running csv.createCsvFileReader, you're beginning the stream on all of them at the same time. You likely don't want this.

Because you're comparing two files to the contents of december_raw.csv , it might be better to load the contents of that file in memory completely, and then stream-compare the other two files to this in series using a callBack and a universal comparison function.

var csv = require('ya-csv');
var fs = require('graceful-fs');

var reader_silo = []; // a variable that holds the rows of the main csv.

var reader = csv.createCsvFileReader('december_raw.csv', {columnsFromHeader:true,'separator': ','});
reader.addListener('data', function(data){
  reader_silo.push(data[0]); // load each read in row into the array
});

reader.addListener('end', function(){
  //start comparing with first csv file.
  compareRows('first_data.csv', function(){
    // compare with second data
    compareRows('second_data.csv');
  });
});

// the comparison function, takes in the filename, and a callBack if there is one.    
function compareRows(csvFileName, callBack){

  var csvStream = csv.createCsvFileReader(csvFileName, {columnsFromHeader:false,'separator': ','}); // begin stream

  csvStream.addListener('data', function(data){
    for (var i = 0; i < reader_silo.length; i++) {
      if(data[0] === reader_silo[i]){
        fs.appendFileSync('results.csv', data[0]+","+reader_silo[i]);
        break;
      }
    }
  });

  csvStream.addListener('end', function(data){
    // if there's a callBack then we can execute it.
    // in this case the first time it is executed there is a callBack which executes this function again with the next file.
    if(callBack && typeof callBack === "function") callBack();
  });
}

PS. If your script continues beyond this, you might also want to consider zeroing out the reader_silo when you're done your comparisons. So your 'end' listener callBack would look like this:

reader.addListener('end', function(){
  compareRows('first_data.csv', function(){
    compareRows('second_data.csv', function(){
      reader_silo = [];
    });
  });
});

Answer 2

Here's an even more memory efficient answer, without any assumptions. In it, you make sure you pass in the smallest CSV file as the first argument to a compareRows function.

This really makes sure that you're being as memory efficient as possible, by keeping only the smallest set possible stored in memory.

var csv = require('ya-csv');
var fs = require('graceful-fs');

var smallFileName = ""; // used to see if we need to really reload the file again.
var smaller_silo = [];

compareRows('smaller.csv', 'larger.csv', function(){
  compareRows('smaller.csv', 'anotherLarger.csv', function(){
    smaller_silo = []; }); // done
});

function compareRows(smallerFileName, largerFileName, callBack){

  var reader;
  if(smallerFileName !== smallFileName){
    smallFileName = smallerFileName;
    reader = csv.createCsvFileReader(smallerFileName, { columnsFromHeader: true, separator: ','});
    reader.addListener('data', function(data){
      smaller_silo.push(data[0]);
    });

    reader.addListener('end', function(){
      compareSmallerToLarger(largerFileName, callBack);
    });
  }
  else{
    compareSmallerToLarger(largerFileName, callBack);
  }
}

function compareSmallerToLarger(largerFileName, callBack){

  var csvStream = csv.createCsvFileReader( largerFileName, { columnsFromHeader: false, 'separator':','});
  csvStream.addListener('data', function(data){
    for (var i = 0; i < smaller_silo.length; i++) {
      if(data[0] === smaller_silo[i]){
        fs.appendFileSync('results.csv', data[0]+","+smaller_silo[i]);
        break;
      }
    }
  });
  csvStream.addListener('end', function(data){
    if(callBack && typeof callBack === "function") callBack();
  });
}

Anyway, I shouldn't obsess over things...