如何处理依赖于另一个承诺的承诺循环

Question

I'm new to JavaScript and I'm having trouble with promises. I'm using cloudscraper to retrieve a webpage's html to scrape data from. I have a simple function - getData() - which calls cloudscraper.get() and passes the html to the extract() function, which is responsible for scraping data. This is the working code:

const getData = function(pageUrl) {
  var data;
  return cloudscraper.get(pageUrl)
    .then(function(html) {
      data = extract(html);
      return data;  
    })
    .catch(function(err) {
      // handle error
    })
}

The "data" object returned contains an array of URLs I want to connect to, in order to retrieve other information. That information has to be stored in the same data object. So I want to call cloudscraper.get() method again for each URL contained in the array. I've tried the code below:

const getData = function(pageUrl) {
  var data;
  // first cloudscraper call:
  // retrieve main html
  return cloudscraper.get(pageUrl)
    .then(function(html) {
      // scrape data from it
      data = extract(html);
      for (let i = 0; i < data.array.length; ++i) {
        // for each URL scraped, call cloudscraper
        // to retrieve other data
        return cloudscraper.get(data.array[i])
          .then(function(newHtml) {
            // get other data with cheerio
            // and stores it in the same array
            data.array[i] = getNewData(newHtml);
          })
          .catch(function(err) {
            // handle error
          }) 
        }
        return data;  
      })
    .catch(function(err) {
      // handle error
    })
}

but it doesn't work, because the data object is returned before the promises in the loop are resolved. I know that probably there is a simple solution, but I couldn't figure it out, so could you please help me? Thanks in advance.

Answer 1

The best way to avoid these kinds of problems is to use async/await , as suggested in the comments. Here's an example based on your code:

const getData = async function(pageUrl) {
  var data;
  // first cloudscraper call:
  // retrieve main html
  try {
    const html = await cloudscraper.get(pageUrl);
    // scrape data from it
    data = extract(html);
    for (let i = 0; i < data.array.length; ++i) {
      // for each URL scraped, call cloudscraper
      // to retrieve other data
      const newHtml = await cloudscraper.get(data.array[i]);
      // get other data with cheerio
      // and stores it in the same array
      data.array[i] = getNewData(newHtml); // if getNewData is also async, you need to add await
    }
  } catch (error) {
    // handle error
  }
  return data;
}
// You can call getData with .then().catch() outside of async functions 
// and with await inside async functions

Answer 2

This can be significantly simplified by using Promise.all , and await / async

If my understanding is correct, you are trying to execute the below steps:

1. Get original HTML
2. Extract some HTML (looks like you're after some more urls)
3. For each url extracted, you want to re-call cloudscraper
4. Put the results of each call back into the original data object.

 const getData = async (pageUrl) => { const html = await cloudscraper.get(pageUrl); const data = extractHtml(html); const promises = data.array.map( d => cloudscraper.get(d)); const results = await Promise.all(promises); // If you wanted to map the results back into the originaly data object data.array.forEach( (a, idx) => a = results[idx] ); return data; };