Counter in async function

Question

I am a novice in JS/nodeJS world and I have some difficulties with the asynchronous way of thinking...

app.post("/upload", upload.array("files"), uploadFiles);
function uploadFiles(req, res) {
    req.files.forEach(function(file) {
        var linesInserted = 0;
        var linesInError = 0;
        fs.createReadStream(file.destination + file.filename)
        .pipe(parse({ delimiter: ",", columns: false, fromLine: 2 }))
        .on("data", function (row) {
            Model.findOneAndUpdate(
                { code: row[0] },
                { 
                    $set: { 
                        a: row[1],
                        b: row[2],
                        c: row[4],
                        d: row[5],
                        e: row[6],
                        f: moment(row[7], "DD/MM/YYYY").toISOString(),
                        g: moment(row[8], "DD/MM/YYYY").toISOString(),
                        h: row[12].split(/\s/)[0].replace(',','.')
                    },
                    $setOnInsert: { 
                        i: row[0], 
                        j: "airbnb",
                        k: row[3],
                        l: moment(row[10], "DD/MM/YYYY").toISOString()
                    },
                    $push: { 
                        connectedTo: [{ m : "xxx" }, { service: "n", serviceId: "o" }] 
                    }
                },
                { 
                    upsert: true,
                    runValidators: true
                },
                function(err, res) {
                    if (err) {
                        console.log(err);
                        linesInError++;
                    } else if (res) {
                        console.log(linesInserted);
                        linesInserted++;
                    }
                }
            );
        })
        .on("end", function () {
            File.create({
                file: file.destination + file.filename,
                originalName: file.originalname,
                linesInserted: linesInserted,
                linesInError: linesInError
            });

            console.log(`File ${file.originalname} - ${file.destination + file.filename} imported`);
        })
        .on("error", function (error) {
            console.log(error.message);
        });
    });
    res.json({ counterFilesImported: req.files.length });
}

With this code, the problem is that, at the end the values for File are 0 for linesInserted and linesInError. These counters increment during the reading of the file but at the "end" their value is 0. I know that this is a problem of asynchronous calls but I can't find any solution.

I would like to retrieve the "real" value of the counters.

Thank you for your help !

Answer 1

Your issue is that the stream pushes data/rows as fast as it can, and when it's done it calls end but it doesn't wait for the rows to be processed before checking the lineCount.

I have mixed feelings about streams, but according to the docs they are async iterable; that I know how to work with.

I've changed the code to Promises.

app.post("/upload", upload.array("files"), function (req, res) {
  processFiles(req.files);

  res.json({ counterFilesImported: req.files.length });
});

async function processFiles(files) {
  for (const file of files) {
    try { // to catch stream errors

      let linesInserted = 0;
      let linesInError = 0;

      const stream = fs.createReadStream(file.destination + file.filename)
        .pipe(parse({ delimiter: ",", columns: false, fromLine: 2 }));

      for await (const row of stream) {
        try {
          await Model.findOneAndUpdate(
            { code: row[0] },
            {
              $set: {
                a: row[1],
                b: row[2],
                c: row[4],
                d: row[5],
                e: row[6],
                f: moment(row[7], "DD/MM/YYYY").toISOString(),
                g: moment(row[8], "DD/MM/YYYY").toISOString(),
                h: row[12].split(/\s/)[0].replace(',', '.')
              },
              $setOnInsert: {
                i: row[0],
                j: "airbnb",
                k: row[3],
                l: moment(row[10], "DD/MM/YYYY").toISOString()
              },
              $push: {
                connectedTo: [{ m: "xxx" }, { service: "n", serviceId: "o" }]
              }
            },
            {
              upsert: true,
              runValidators: true
            }
          );

          ++linesInserted;

        } catch (err) {

          console.log(err);
          ++linesInError;
        }
      }

      File.create({
        file: file.destination + file.filename,
        originalName: file.originalname,
        linesInserted: linesInserted,
        linesInError: linesInError
      });

      console.log(`File ${file.originalname} - ${file.destination + file.filename} imported`);
    } catch (error) {
      // error with the stream/file
      console.log(error.message);
    }
  }
}