简体   繁体   中英

How do I wait for multiple async functions in NodeJS before returning them all to update a firestore document?

I'm going to simplify the code below and will add comments throughout.

The generally idea is, that once a "sales lead is saved" (customer call) within my android app (for my pops' construction gig), I am scraping a website with pupeeteer to gather some data about the address. I already had this all working but (so far - successfully) I have been saving the "external URL" from the scraped website, but now I'd like to just take the external URL, get the Image, upload it to my Firestore Storage, get the Signed URL and then update the firestore document with this "signed URL" (instead of the external URL) Everything seems to be working OKay-ish, just not in the right order. I am getting image Url and I am successfully saving it to firestore storage but my signed URL comes back way too late. I'm just stumped with the handling of callbacks (or async functions) here. I don't necessarily need code (but won't turn it down, Haha). maybe just a general guide how I need to organize my code, I've read many information about callbacks, promises, etc. but just can't seem to wrap my head around and/or apply it to this particular example.

Thanks for any help! Much appreciated!

exports.fetchData = functions.https.onCall(async (data, context) => {
    const urlLib = require("url");
    const pathLib = require("path"); 

    //Check for authentication
    if (!context.auth) { //not authed -> error
        throw new functions.https.HttpsError('failed-precondition', 'The function must be called ' +
            'while authenticated.');
    } else { //authed -> good to go

        var salesLeadId = data.salesLeadId; //grab leadID which is passed from my android app into this function (unique ID to identify the record I need to update)       

        try {    
            //Example URL
            let url = 'http://www.example.com'; //I left out code here where I build a custom URL etc.

            const browser = await puppeteer.launch(); 
            const [page] = await browser.pages();
            //Emulate Headless Chrome instance        
            await page.emulate({
                name: 'Desktop 1920x1080',
                userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
                viewport: {
                width: 1920,
                height: 1080
                }
            });
            //Opens Page, and waitws until loading is done
            await page.goto(url, { waitUntil: 'networkidle0' });

            //Read Data from web page and return as data object
            var data = await page.evaluate(() => {

                var scrapedPhoto = document.querySelector("#imgExamplePhoto").getAttribute('src'); //This is the external URL to the image
                // [...] grabbing other info here and adding it to "data"
                var accNo = "123456789"; //Example Data but in reality also fetched via document.queryselector
                var pId "12-44-66-88-88"; //Example Data but in reality also fetched via document.queryselector
                // etc...

                return { scrapedPhoto, accNo, pId }; //return all scraped data in var data

            });

            await browser.close();

            //below function is to grab the external URL, save it to Firestore Storage and return a Signed URL which I would then save with my lead document
            var uploadedPhotoUrl = await saveToStorage(data.scrapedPhoto); // THIS IS WHERE MY PROBLEM IS....the code keeps going. The function eventually returns the correct URL but way too late.

            //Firestore Document Reference for record to be updated
            let leadRef = admin.firestore().collection("sales_leads").doc(salesLeadId);

            //I've changed this in every way that I can think of by trial and error (i.e. tried chaining "thens", but unsuccessfully). I assume first I need to resolve the saveStorage function before proceeding to update my firestore document
            return leadRef.update({ //Here is where I'm going back to my Firestore Document and update it with this "new data"
                photo: uploadedPhotoUrl, //Problem: this is undefined because my code does NOT wait on "saveToStorage" above (Previously this was just the external "scrapedPhotoUrl" and thus worked fine. But instead of saving the external URL, I'm trying to save this image to firestore to avoid constant calls to their web page)
                pId: data.pID,
                accNo: data.accNo
            }).then(writeResult => {
                console.log(`Sales Lead updated with Data on: ${writeResult.writeTime.toDate()}`);
                return `Sales Lead updated with Data on: ${writeResult.writeTime.toDate()}`
            });        

        } catch (err) {
            console.error(err);
            return 'Error';
        }


    }
});

async function saveToStorage(fileUrl) {
    var storage = admin.storage();
    var fetch = require("node-fetch");
    var urlLib = require("url");
    var pathLib = require("path");
    //Get File Name from provided URL
    var parsed = urlLib.parse(fileUrl);
    var fileName = pathLib.basename(parsed.pathname);

    //Create Storage Reference with new File Name
    var bucket = storage.bucket('gs://myprojetname.appspot.com');
    var folderPath = 'sales/photos/';
    var internalFilePath = folderPath + fileName;
    var file = bucket.file(internalFilePath);

    //Fetch file from url
    return await fetch(fileUrl).then(res => {
      const contentType = res.headers.get('content-type');
      const writeStream = file.createWriteStream({
        metadata: {
          contentType
        }
      });
      return res.body.pipe(writeStream)
        .on('finish', () => {
          console.log('Image saved')
          return getDownloadURL(file); //so this eventually returns the corret URL, but by the time this "comes back" my main function has finished execution
        })
        .on('error', err => {
          writeStream.end();
          return console.error(err);
        });

    });


  }

//Get Signed Download URL from Firestore Storage. This works fine for me.
  function getDownloadURL(file) {     
    const config = {
        action: 'read',
        expires: '10-30-2050'
      };

    return file.getSignedUrl(config).then((downloadUrl) => {
      let result = downloadUrl[0];
      return result;
    }).catch(error => {
      console.error(error);
      return 'Error';
    });

  }




So I think 'I am getting there'. I realized the issue seems to be within my "saveToStorage" function as my logging output indicated. (console.log posted way before the result was returned there).

Now I fundamentally (I think) have it in the right order.

saveToStorage function =>

[...]
  return await fetch(fileUrl).then(res => {
    const contentType = res.headers.get('content-type');
    const writeStream = file.createWriteStream({
      metadata: {
        contentType
      }
    });
    let p = new Promise((resolve, reject) => {
      res.body.pipe(writeStream).on('finish', function() {
        resolve(file);
      });
      res.body.pipe(writeStream).on('error', function () {
        reject();
      });
    });

    return p.then(function(file) {
      console.log('Image saved')
      return getDownloadURL(file);
    });

  });
[...]

I added a Promise within this function to be resolved before returning the data. I am getting the proper URL and the correct storage (signed) url is written/updated to my firestore document.... BUT...

My Images are only written about half way. Bottom half is grey pixels. :head scratching: (Previously the Image was fine, but url was returned too late. Now the url is returned "in time" but the image doesn't seem to be complete.)

Edit: I'm going to mark this an answer and open a new question with my new problem. I believe my original problem is resolved here. (Edit: Tomorrow)

The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.

 
粤ICP备18138465号  © 2020-2024 STACKOOM.COM