How do I read a large csv file using exceljs

Question

I'm using exceljs with csv and excel files in my project. Recently the files become really large and I am having trouble to read them as normal. The following code works good for small files however it can't read Million rows .

I realized I need to use stream but I don't understand how it works. I would like to use it for the same case just to be able to read large. Can someone help please?

var Excel = require('exceljs');
var workbook = new Excel.Workbook();
readSecurityCSV(workbook);
var header = {}
var adSecurities = []



function readSecurityCSV(workbook){
workbook.csv.readFile('./csv/clientsOrders.csv')
.then(worksheet => {

   worksheet.eachRow({ includeEmpty: true }, function(row, rowNumber) {

     if(rowNumber == 1){
       header = {}
       row.eachCell({ includeEmpty: true }, function(cell, cellNumber) {
         header[cellNumber] = cell

       });
     }else{
       var currentSecurity = {}
       row.eachCell({ includeEmpty: true }, function(cell, cellNumber) {

        currentSecurity[header[cellNumber].value] = cell.value
       });

       currentSecurity.rowNumber = rowNumber

       adSecurities.push(currentSecurity)


     }

   });
   console.log(color.blue("adSecurity"), adSecurities[0]);

 })

}

Answer 1

you can split csv file then loop through it.

the splitting can be done like this:

var chunkSize = 1024 * 1024;
var fileSize = file.size;
var chunks = Math.ceil(file.size/chunkSize,chunkSize);
var chunk = 0;
while (chunk <= chunks) {
  var offset = chunk*chunkSize;
  console.log(file.slice(offset,offset + chunkSize));
  chunk++;
}

if you want to work with multiple promises in paralel, check this link [ 1 ]