Express res.download extremely weird behaviour

Question

I have a NodeJS backend that listens for a POST request with a json package. Using the information in the json package it builds a csv file with around 15,000 lines and then uses res.download to send the csv file back to the client.

Reading from a cloud database into the csv file is no problem. I have checked the file in the server and the lines are all there and they are accurate. However, the file that is downloaded on the client side has maybe a few hundred lines cut off. It seems like res.download() is running too soon even though I've explicitly set the stream to end once the for loop is over or it is running when it is supposed to but the csv file is still buffering or something

Here is my code:

Server side:

app.post('/dashboard/download_data', function (req, res) {
    let payload = req.body;
    ref.orderByKey().once("value", function (snapshot) {
        let data = snapshot.val();
        writer.pipe(fs.createWriteStream('C:\\user\\EVCS_portal\\out.csv'));
        for (let key in data) {
            if (data.hasOwnProperty(key)) {
                test_time = data[key]['time'];
                writer.write({
                    time: data[key]['time'],
                    ac2p: data[key]['ac2p'],
                    dcp: data[key]['dctp']
                })
            }
        }
        writer.end('This is the end of writing\n');
        writer.on('finish', () => {
            console.log(test_time);
            res.download('C:\\user\\EVCS_portal\\out.csv');
            console.log('file sent out!')
        });
    })

Client side js:

firebase.auth().currentUser.getIdToken(true).then(function (idToken) {
        let xhr = new XMLHttpRequest();

        xhr.onreadystatechange = function () {
            if (xhr.readyState === 4 && xhr.status === 200) {
                let a = document.createElement('a');
                a.href = window.URL.createObjectURL(xhr.response);
                a.download = download_date + '.csv';
                a.style.display = 'none';
                document.body.appendChild(a);
                a.click();
            }
        };

        let url = "/dashboard/download_data";
        xhr.open("POST", url, true);
        xhr.setRequestHeader("Content-Type", "application/json");
        xhr.responseType = 'blob';

        // Package our payload including the idToken and the date
        let data = JSON.stringify({"idToken": idToken, 'date': download_date});
        xhr.send(data);

Answer 1

You're not listening to the correct stream for the 'finish' event which leads to you sending a response before the csv will have finished writing. You're calling res.download() when writer has finished piping data to the stream, but thats not when the csv WriteableStream via fs.createReadStream() has finished writing all of the piped data in its stream to the file system.

Instead of creating the WriteableStream for the csv within the pipe() , save it in a variable and add a listener for the 'finish' event. Listening for 'finish' on writer resulted in your server responding before the csv was ready.

app.post('/dashboard/download_data', function (req, res) {
  let payload = req.body;
  ref.orderByKey().once("value", function (snapshot) {
      let data = snapshot.val();

      let csvWriter = fs.createWriteStream('C:\\user\\EVCS_portal\\out.csv')

      writer.pipe(csvWriter);
      for (let key in data) {
          if (data.hasOwnProperty(key)) {
              test_time = data[key]['time'];
              writer.write({
                  time: data[key]['time'],
                  ac2p: data[key]['ac2p'],
                  dcp: data[key]['dctp']
              })
          }
      }
      writer.end('This is the end of writing\n');
      csvWriter.on('finish', () => {
          console.log(test_time);
          res.download('C:\\user\\EVCS_portal\\out.csv');
          console.log('file sent out!')
      });
  })
})

Answer 2

The server code seems ok, but for the client code seems it is a getting the download too hurry.

Pleaes have a look to see this thread and try xhr.onload instead of xhr.onreadystatechange.