JavaScript / Promise-定义Promise链接之间的超时

Question

I am using cheeriojs to scrap a site, i need to emit a lot of requests on several url parameters.

minimal code:

const rp = require('request-promise');
const cheerio = require('cheerio');

[1, 2, 3].forEach(element => {
  url = `https://stackoverflow.com/q=${element}`
  rp(url)
    .then((html) => {
      // Logic code
   })
})

I would like to set a timeout between each request, how can we define it?

Answer 1

I think the most readable approach would be to use an async function and promise-ified timeout.

function sleep(millis) {
  return new Promise(resolve => setTimeout(resolve, millis));
}

async function process(list) {
  for (const item of list) {
    const html = await rp(`https://stackoverflow.com/q=${item}`);
    ... do stuff
    await sleep(1000);
  }
}

Answer 2

Currently all the requests are essentially made in parallel. Before you can add a delay between them you have to execute them in sequence. You can do that by chaining promises. This is easy to do with .reduce :

const rp = require('request-promise');
const cheerio = require('cheerio');

[1, 2, 3].reduce((p, element) => {
  url = `https://stackoverflow.com/q=${element}`
  return p
    .then(() => rp(url))
    .then((html) => {
      // Logic code
    });
}, Promise.resolve())

This builds a chain that is equivalent to

rp(url1)
  .then(html => ...)
  .then(() => rp(url1))
  .then(html => ...)
  .then(() => rp(url2))
  .then(html => ...)

To add a delay, we define a function that returns a function that returns a promises that resolves after x milliseconds via setTimeout :

function wait(x) {
  return () => new Promise(resolve => setTimeout(resolve, x));
}

Now we can add that to our chain (I'm replacing rp with something runnable here):

 function wait(x) { return () => new Promise(resolve => setTimeout(resolve, x)); } [1, 2, 3].reduce((p, element) => { const url = `https://stackoverflow.com/q=${element}` return p .then(() => Promise.resolve(url)) .then((html) => { console.log(`Fetched ${html}`); }) .then(wait(2000)); }, Promise.resolve()) 

Answer 3

You can use the index argument of forEach as multiplier for timeout delay

const delay = 1000

[1, 2, 3].forEach((element, i) => {
    url = `https://stackoverflow.com/q=${element}`
    setTimeout(() => {
       rp(url)
           .then((html) => {
            // Logic code
           })
    }, i * delay);

})

Answer 4

If you want to use a forEach statement, use my first code. If it doesn't matter to you, see my second (simpler) working example, based on @JFord's answer.

RunKit demo with forEach

RunKit demo with for item of list

Note: the code has been fixed to work properly

forEach example

const rp = require('request-promise')
const cheerio = require('cheerio')

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms))
}

async function forEachAsync(arr, fn) {
  for (var i = 0; i < arr.length; i++) {
    await fn(arr[i])
  }
}

async function fetchUrls() {
  await forEachAsync([55505362, 55505363, 55505364], async element => {
    await sleep(2000)
    console.log('been 2000 seconds')
    var url = `https://stackoverflow.com/questions/${element}`
    await rp(url)
      .then(html => {
        console.log(html)
      })
      .catch(function(e) {
        console.log(e.message) // "oh, no!"
      })
  })
}

fetchUrls()

for item of list example

This is a working example, based on the answer by @JFord but additionally handling errors.

const rp = require('request-promise')
const cheerio = require('cheerio')

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms))
}

async function fetchUrls(list) {
  for (const item of list) {
    const html = await rp(`https://stackoverflow.com/q=${item}`).catch(function(e) {
        console.log(e.message) // There's an error
    })
    console.log("html: " + html)
    await sleep(2000);
  }
}

fetchUrls([1,2,3])