NodeJS CSVReader - createReadStream with csv-parse returns empty array

Question

I have the CSV reader class that takes two parameters (filePath, and model). The file path is the path to the.csv file, and the model is the model that the.csv will be built on. The data will be stored in the output array. It as has a problem as when I return the array, it is empty. But when I console log the array, it has the data in there. Can someone help me?

Index.js

const parse = require('csv-parse')
const fs = require('fs');
const output = [];

class CSVReader{
    static GetRecord(filePath, model){
        fs.createReadStream(filePath)
            .pipe(parse({columns: false, delimiter: ',', trim: true, skip_empty_lines: true}))
            .on('readable', function (){
                let record
                while (record = this.read()){
                    let city = model.create(record)
                    output.push(record)
                }
            })
            .on('end', function (){
                //console.log(output);
            })
        return output;
    }
}
module.exports = CSVReader;

I used Jest to test the file but it has a problem as expected is 6, but I received []

Index.test.js

const CSVReader = require('../src/Index');
const City = require('../src/Models/City')
test('Can Read CSV File', () => {
    let filePath  = 'data/worldcities.csv';
    let records = CSVReader.GetRecord(filePath, City);
    expect(records.length).toBe(6);
});

Answer 1

tl;dr:

This is an async call, so you cannot just return the response and expect it to work. You need to use the Promise API and an async function.

What makes this async?

All of the Node.js fs API is async (excluding the fs.*Sync functions).

How can I use Promises?

You can return a Promise at the top of your function, then pass a callback:

return new Promise((resolve, reject) => { /* callback */ });

Fixing the code

// all of this is fine
const parse = require('csv-parse')
const fs = require('fs');
// remove the const output = []; as this will cause problems

class CSVReader{
    // this needs to be async
    static async GetRecord(filePath, model){
        // return a promise
        return new Promise((resolve, reject) => {
            // assign output here (https://stackoverflow.com/a/66402114/14133230)
            const output = [];
            fs.createReadStream(filePath)
                .pipe(parse({columns: false, delimiter: ',', trim: true, skip_empty_lines: true}))
                .on('readable', function (){
                    let record
                    while (record = this.read()){
                        let city = model.create(record)
                        output.push(record)
                    }
                    // you may need to call WriteStream#close() here
                })
                .on('end', function (){
                    // output is available here
                    // resolve the promise
                    resolve(output);
                })
        });
    }
}
module.exports = CSVReader;

Using the new Promise-based function

const CSVReader = require('../src/Index');
const City = require('../src/Models/City')
test('Can Read CSV File', () => {
    let filePath  = 'data/worldcities.csv';
    let records = CSVReader.GetRecord(filePath, City); // type Promise
    records.then((response) => { // when promise fills
        expect(response.length).toBe(6);
    });
});

Answer 2

First recommendation: get rid of the globally defined "const output = []" in your library class. This will cause so much confusion to whoever uses your static method. You'll end up accumulating results of previous calls on there.

Also I recommend: if you don't need it to handle huge csv files you can definitely go for the synchronous version and get rid of the complexity of the async loading.

const parse = require('csv-parse/lib/sync');
const fs = require('fs');

class CSVReader{
    static GetRecord(filePath, model) {
        const csvContents = fs.readFileSync(filePath);
        const output = parse(csvContents, {columns: false, delimiter: ',', trim: true,     skip_empty_lines: true})
            .map(function (record) {
                return model.create(record);
            });
        return output;
    }
}
module.exports = CSVReader;
    

Answer 3

Another potentially helpful restructuring could come from attaching the factory pattern (which you currently aim to implement with CSVReader) directly to the base implementation of your Models. This will make it clearer which Models are usable with the CSVReader and how the implementation of a Model is structured to be usable with the pattern.

The base CSVModel provides the general listFromCSV functionality and modelFromRecord instantiation.

src/Models/CSVModel.js

const parse = require('csv-parse/lib/sync');
const fs = require('fs');

class CSVModel {
    static modelFromRecord(record) {
        var model = new this();
        model.record = record;
        return model;
    }
    static listFromCSV (filePath) {
        const csvContents = fs.readFileSync(filePath);
        return parse(csvContents, {columns: false, delimiter: ',', trim: true, skip_empty_lines: true})
            .map((record) => {
                return this.modelFromRecord(record);
            });
    }
}
module.exports = CSVModel;

City model inherits the listFromCSV factory from the CSVModel and specifies its data association to local model properties

src/Models/City.js

const CSVModel = require('./CSVModel');

class City extends CSVModel {
    static modelFromRecord(record) {
        var model = super.modelFromRecord(record);
        // assign properties of the CSV row to dedicated model properties
        [model.name, model.id, model.asd] = [record[0], record[1], record[2]];
        // or classical
        model.name = record[0]; 
        // ...
        return model;
    }
}
module.exports = City;

This allows the usage of the concrete Model Class to instantiate a list from a CSV.

test/csvmodel.test.js

const City = require('../src/Models/City');
test('Can Read City list', () => {
    let filePath  = 'data/worldcities.csv';
    let records = City.listFromCSV(filePath);
    expect(records.length).toBe(6);
});