如何使用异步、等待和承诺？

Question

I am building a web scraper to get all of user's submissions on codeforces. I don't know much about async, await, promises. I have used axios (promise based) to request codeforces and cheerio to parse HTML.

app.post("/", (req, res) => {
  const usernameorhandle = req.body.userName;
  getstatus(usernameorhandle).then ( ()=> {
      var output = fs.createWriteStream(__dirname + '/Data/solutions.zip');
      var archive = archiver('zip', {
        zlib: { level: 9 } // Sets the compression level.
      });
      output.on('close', function() {
        console.log(archive.pointer() + ' total bytes');
        console.log('archiver has been finalized and the output file descriptor has closed.');
      });
      output.on('end', function() {
        console.log('Data has been drained');
      });
      res.attachment(__dirname + "/Data/Problems", 'Codeforces-Solutions');
      archive.pipe(res);
      archive.directory(__dirname + "/Data/Problems", 'Codeforces-Solutions');
      archive.finalize();
    }) })

I am using to accept post request. I am putting all the solutions into a folder and creating zip folder and then send to res.

Below is my getstatus function.

    async function getstatus(handle){
  return new Promise(async (resolve, reject)=> {
    console.log("HELLLLLLLOOOOOOOO");
    await axios.get("https://codeforces.com/api/user.status?handle=" + handle + "&from=1")
      .then(response => {
        if(response.data.status === 'OK'){
          let results = response.data.result;
          console.log("AAAAAAAAAAAAAAAAAAAAAAAa");
          scrape(results).then( () =>{
            console.log("DONE");
            resolve();
          })
          .catch(err => console.log(err));
          // resolve();
        }
        else console.log(submissions.comment);
      })
  })

}

I use scrape function to obtain HTML data and put to folder named Problems.

async function scrape (results){
  console.log("inside scrape");
  //  console.log("HELLO");
  return new Promise( async (resolve, reject) => {
    await results.forEach(async (result)=> {
      if(result.verdict === 'OK'){
        await axios.get("https://codeforces.com/contest/" + result.contestId + "/submission/" + result.id)
        .then(solutionPage => {
          const $ = cheerio.load(solutionPage.data);
          const path = "/home/srujan/Desktop/crawlerapp/Data/Problems/" + result.problem.name + ".cpp";
           fs.writeFile(path, $('#program-source-text').text(), function(err){
            if(err){
              console.log(err);
            }
            else{
              console.log("Saved file");
            }
          })
        })
        .catch( error => {
          console.log("HTML PARSE ERROR" + error);
        })
     }
    })
    console.log("hey");
    resolve();

  })

The problem is I am getting

HELLLLLLLOOOOOOOO
AAAAAAAAAAAAAAAAAAAAAAAa
inside scrape
hey
DONE
saved file
saved file
...

Browser downloads after DONE and then files are saved. I am new to js and don't know why I am getting this.

PS: I know this is very long question. I tried reading a lot about this. Didn't understand properly how to do that. I copy pasted some code which I didn't understand like how to zip a folder.

Answer 1

The problem is to use result.forEach Try to use a simple for(let i = 0; i < result.length; i++) without async.

If that doesn't work, try to return anything inside the then.

Answer 2

This is how I would construct getstatus function with await async

 async function getstatus(handle) {

    const response = await axios.get("https://codeforces.com/api/user.status?handle=" + handle + "&from=1")

    if(response.data.status === 'OK') {

      let results = response.data.result;

      try {
        await scrape(results);
        console.log("DONE");
      }
      catch(error) {
      }

    }
}

and scrape function accordingly...

const fs = require('fs').promises;

async function scrape (results) {
  results.forEach(async (result)=> {

   if(result.verdict === 'OK') {
    const solutionPage = await axios.get("https://codeforces.com/contest/" + result.contestId + "/submission/" + result.id)

    const $ = cheerio.load(solutionPage.data);
    const path = "/home/srujan/Desktop/crawlerapp/Data/Problems/" + result.problem.name + ".cpp";

    try {
      await fs.writeFile(path, $('#program-source-text').text())
      console.log("Saved file");
    }
    catch(error) {
    }
  }
 }
}             

Answer 3

forEach(callback) executes callback . If callback returns a promise (ie, it's an async function), the promise won't be resolved before calling the callback on the next element of the array.

So, basically, you can't use async functions inside forEach... But you can use for-loops or Promise.all instead!

Also, fs.writeFile works with sync + callback, but there exists a fs.promise.writeFile that uses promises instead.

Here's a scrape function that should work better:

async function scrape(results) {
  for (const result of results) {
    if(result.verdict === 'OK') {
      const solutionPage = await axios.get("https://codeforces.com/contest/" + result.contestId + "/submission/" + result.id);
      const $ = cheerio.load(solutionPage.data);
      const path = "/home/srujan/Desktop/crawlerapp/Data/Problems/" + result.problem.name + ".cpp";
      try {
        await fs.promises.writeFile(path, $('#program-source-text').text());
      } catch(err) { console.log(err) }
    }
  }
}