Async await in loop stops after 370th iteration

Question

I'm trying to load a large csv file (size in GB's) in chunks. Code below:

lineReader.open(filename,async function(err, reader) {
  if (err) throw err;
  var dataArr=[]; 
  while (reader.hasNextLine()) {
    reader.nextLine(function(err, line) {
      if(err) throw err;
      console.log(line);
      dataArr.push(csv_parse(line,headers));
    });
    console.log(dataArr.length);
    if(dataArr.length == 3000){
      console.log(JSON.stringify(dataArr));
      await timeout(6000);
      console.log("timeout");
      dataArr = [];
    }
  }
    reader.close(function(err) {
      if (err) throw err;
    });
});

reader.nextline() stops working after line 370 even though the while loop is executing. however when I move the await outside of the if the code seems to work fine. Why is this happening.

Answer 1

You're mixing promises ( async/await ) with callbacks ( reader.nextLine() ), which is causing your problems.

Specifically, you're calling reader.close() too early, because it gets called before all lines have been read; the 370 lines that do get read is probably what fits into the buffer that gets read from the file before it gets closed.

A solution would be to also make reading the next line promise-based, for instance like this:

const getNextLine = async reader => {
  return new Promise((resolve, reject) => {
    reader.nextLine(function(err, line) {
      if (err) return reject(err);
      resolve(line);
    });
  });
}

lineReader.open(filename, async function(err, reader) {
  if (err) throw err;
  var dataArr = [];
  while (reader.hasNextLine()) {
    let line = await getNextLine(reader);
    dataArr.push(csv_parse(line, headers));
    console.log(dataArr.length);
    if (dataArr.length == 3000) {
      console.log(JSON.stringify(dataArr));
      await timeout(6000);
      console.log("timeout");
      dataArr = [];
    }
  }
  reader.close(function(err) {
    if (err) throw err;
  });
});