Node.js脚本静默失败？

Question

I've written a Node.js script that uses the download , axios , and fs modules to extract urls from JSON provided by the Federal Register , and download the associated PDF files. However, the script routinely fails to download all of the PDFs.

For whatever reason, my script "stalls" before downloading all of the PDF files. Meaning, it starts off great (downloads maybe 70, 80 files) but then stalls. It doesn't fire my catch block, or fail in any way. It just stops downloading.

The number of files varies based on what wifi connection I'm on. However, I've never been able to get the code to finish, and fire the .then block in my code. Ideally, I would like to use the .then block to process the files once they are downloaded.

Here is the code:

// The callback function that writes the file...
function writeFile(path, contents, cb){
  mkdirp(getDirName(path), function(err){
    if (err) return cb(err)
      fs.writeFile(path, contents, cb)
  })
};

// The function that gets the JSON...
axios.get(`http://federalregister.gov/api/v1/public-inspection-documents.json?conditions%5Bavailable_on%5D=${today}`)
  .then(downloadPDFS)
  .catch((err) => {
    console.log("COULD NOT DOWNLOAD FILES: \n", err);
  });

// The function that downloads the data and triggers my write callback...
function downloadPDFS(res) {
  const downloadPromises = res.data.results.map(item => (
    download(item.pdf_url)
      .then(data => new Promise((resolve, reject) => {
        writeFile(`${__dirname}/${today}/${item.pdf_file_name}`, data, (err) => {
          if(err) reject(err);
          else resolve(console.log("FILE WRITTEN: ", item.pdf_file_name));
        });
      }))
  ))
  return Promise.all(downloadPromises).then((res) => console.log("DONE"))
}

My project is on Github here , in case you'd like to install it and try for yourself. Here's a summary of what's going on, in plain English:

The script fetches JSON from a server, which contains the urls to all 126 PDFs. It then passes an array of these urls to the synchronous map function. Each of the urls is transformed into a promise, with the download module. That promise is implicitly returned, and stored in the Promise.all wrapper. When the download promise resolves (the document is done downloading) my custom writeFile function will trigger, writing the PDF file with the downloaded data. When all of the files have downloaded, the Promise.all wrapper should resolve. But that doesn't happen.

What is going wrong?

EDIT --

As you can see below, the script runs for a while, but then it just stalls and doesn't download any more files...

Answer 1

If it really is a rate issue then there's a few ways you can solve it (depending on how the API is rate limited)

Below there are 3 solutions in one

rateLimited ... this fires off requests limited to a given number of requests per second

singleQueue ... one request at a time, no rate limit, just all requests in series

multiQueue ... at most a given number of requests "in flight" at a time

const rateLimited = perSecond => {
    perSecond = isNaN(perSecond) || perSecond < 0.0001 ? 0.0001 : perSecond;
    const milliSeconds = Math.floor(1000 / perSecond);
    let promise = Promise.resolve(Date.now);
    const add = fn => promise.then(lastRun => {
        const wait = Math.max(0, milliSeconds + lastRun - Date.now);
        promise = promise.thenWait(wait).then(() => Date.now);
        return promise.then(fn);
    });
    return add;
};
const singleQueue = () => {
    let q = Promise.resolve();
    return fn => q = q.then(fn);
};
const multiQueue = length => {
    length = isNaN(length) || length < 1 ? 1 : length;
    const q = Array.from({ length }, () => Promise.resolve());
    let index = 0;
    const add = fn => {
        index = (index + 1) % length;
        return q[index] = q[index].then(fn);
    };
    return add;
};

// uncomment one, and only one, of the three "fixup" lines below
let fixup = rateLimited(10); // 10 per second for example
//let fixup = singleQueue;   // one at a time
//let fixup = multiQueue(6); // at most 6 at a time for example

const writeFile = (path, contents) => new Promise((resolve, reject) => {
    mkdirp(getDirName(path), err => {
        if (err) return reject(err);
        fs.writeFile(path, contents, err => {
            if (err) return reject(err);
            resolve();
        })
    })
});


axios.get(`http://federalregister.gov/api/v1/public-inspection-documents.json?conditions%5Bavailable_on%5D=${today}`)
    .then(downloadPDFS)
    .catch((err) => {
        console.log("COULD NOT DOWNLOAD FILES: \n", err);
    });

function downloadPDFS(res) {
    const downloadPromises = res.data.results.map(item => fixup(() => 
        download(item.pdf_url)
        .then(data => writeFile(`${__dirname}/${today}/${item.pdf_file_name}`, data))
        .then(() => console.log("FILE WRITTEN: ", item.pdf_file_name))
    ));
    return Promise.all(downloadPromises).then(() => console.log("DONE"));
}

I've also refactored the code a bit so downloadPDFS uses promises only - all the node-callback style code is put into writeFile

Answer 2

As Jaromanda pointed out, this is likely to do with the API limiting my access, not with an error in the script.

I added a filter to the script, to select less data, and it works. As follows:

axios.get(`http://federalregister.gov/api/v1/public-inspection-documents.json?conditions%5Bavailable_on%5D=${today}`)
  .then(downloadPDFS)
  .then(() => {
    console.log("DONE")
  })
  .catch((err) => {
    console.log("COULD NOT DOWNLOAD FILES: \n", err);
  });

function downloadPDFS(res) {
  const EPA = res.data.results.filter((item) => {
    return item.agencies[0].raw_name === "ENVIRONMENTAL PROTECTION AGENCY"; //// THIS FILTER
  });

  const downloadPromises = EPA.map(item => ( //// ONLY DOWNLOADING SOME OF THE DATA
    download(item.pdf_url)
      .then(data => new Promise((resolve, reject) => {
        writeFile(`${__dirname}/${today}/${item.pdf_file_name}`, data, (err) => {
          if(err) reject(err);
          else resolve(console.log("FILE WRITTEN: ", item.pdf_file_name));
        });
      }))
  ))
  return Promise.all(downloadPromises)
}