当一个 stream 依赖于另一个时，如何处理 node.js 流的情况？

Question

I am developing a functionality for bulk uploading and I came with this issue.

I want to archive files and that will be uploaded to my server. Also the archive will contain a manifest file - which will describe each file with various properties / meta data / etc.

The issue occurs when I want to send back the response. The stream which is reading the manifest file is closed which leads to immediate callback execution. Bellow I will show the examples.

const csv = require("fast-csv");
const fs = require("fs");
const path = require("path");

async function proccesUpload() {
  const manifestReadStream = fs.createReadStream(
    path.join(__dirname, "manifest.txt")
  );

  manifestReadStream
    .pipe(
      csv.parse({
        delimiter: ";",
      })
    )
    .on("data", async (row) => {
        // do processing for each file described in manifest file
      const hash = crypto.createHash("sha1");
      const rs = fs.createReadStream(targetFile, {
        flags: "r",
        autoClose: true,
      });
      rs.on("data", (data) => hash.update(data, "utf-8"));
      rs.on("close", function onReadStreamClose() {
        // do proccessing for file
      });
    })
    .on("end", async () => {
      // return response when all formating was performed
    });
}

By using nest read stream, the on "end" is executed before all the files are processed. How can I solve this?

Answer 1

I recommend using async iterators will make the code easier and callback free

async function proccesUpload() {
  const manifestReadStream = fs.createReadStream(
    path.join(__dirname, "manifest.txt")
  );

  const parserStream = manifestReadStream.pipe(
    csv.parse({
      delimiter: ";",
    })
  );

  for await (const row of parserStream) {
    // do processing for each file described in manifest file
    const hash = crypto.createHash("sha1");
    const rs = fs.createReadStream(targetFile, {
      flags: "r",
      autoClose: true,
    });
    for await (const data of rs) {
      hash.update(data, "utf-8");
    }
    // DONE PROCESSING THE ROW
  }

  // DONE PROCESSING ALL FILES
  // return response when all formating was performed
}