Object 在承诺有时间解决之前返回

Question

I am trying to build a web scraper to get informations about some products and store them inside a database. I'm getting the HTML source code with Nightmare (because javascript code has to run on the server before the page content is created) then I'm parsing that source with Cheerio. Once I do the parsing there are some images I have to download for the products. I have a simple download function and, based on if the image which I'm trying to download is available or not on the server, I'd like to return a string (or an array of strings) containing either the image name (which I downloaded) or a default image name from my computer. I tried calling the download function as a promise, I tried passing Promise.all() when I know there are multiple images to download, but to no avail. While I'm positive my code is working (the images are downloaded as should, the final object looks great at almost every property and value), it is the images properties fields which, when I'm printing the object to the console, still holds [Promise] / [ Promise { } ] and I'm not quite sure how to solve this matter. I'm positive those promises resolve, but they're not resolved when I'm outputting the resulting object to the console. And that's a problem, 'cause I have to pass that object to be stored in the database and I don't think they'll be resolved then.

The code (minus the exact links) is down below:

const cheerio = require('cheerio')
const nightmare = require('nightmare')()
const download = require('image-downloader')

const settings = new function() {
    this.baseURL = 'https://baseurl.whatever'
    this.urlSearch = `${this.baseURL}/Product/Search?keyword=`
    this.urlVariant = 'https://cdn.baseurl.whatever/Variant/'
    this.urlProduct = 'https://cdn.baseurl.whatever/Product/'
    this.imgPath = './img/'
}

var review_id = 0

function downloadImage(url, filepath, success, error) {
    return download.image({ url, dest: filepath }).then(success, error)
}

const url = 'https://someurl.nevermind.meh/product?pid=50M3NUMB3R',
      code = '50M3C0D3'

async function scrapeProduct(code) {
    const product = await nightmare.goto(url)
        .wait()
        .evaluate(() => document.body.innerHTML)
        .end()
        .then(body => console.log(loadProduct(body, code)))
        .catch(err => console.log(`There was an error: [${err}]`))
}

function loadProduct(body, code) {
    $ = cheerio.load(body)

    return {
        title: $('li.LongName').text().trim(),
        category: $('a#categoryTitleLink').text().trim(),
        min_price: parseFloat($('span.MinPrice').text()),
        max_price: parseFloat($('span.MaxPrice')?.text()) || parseFloat($('span.MinPrice').text()),
        points: parseFloat($('div.AddtoCartUnderText span').text()),
        variants: [...$('div.productDetailClassicRnd')].map(variant => {
            const $field = $(variant).find('input'),
                  item_code = $field.attr('item_code')

            if (item_code.split('-')[0] == code) return null

            return {
                code: item_code.split('-')[0],
                title: $field.attr('item_name'),
                image: downloadImage(
                    `${settings.urlVariant}${item_code.replace(' ', '%20')}`,
                    `${settings.imgPath}${item_code}`,
                    result => result.filename.split('/').reverse()[0],
                    _ => 'variant_default-VC.jpg'
                )
            }
        }).filter(variant => variant !== null),
        images: [...$('img#imgProduct')].map(image => {
            const $image = $(image),
                  source = $image.attr('src')

            return downloadImage(
                source, 
                `${settings.imgPath}${source.split('/').reverse()[0]}`,
                result => result.filename.split('/').reverse()[0],
                _ => 'product_default.jpg'
            )
        }),
        other_images: [...$('img.productDetailOtherIMG')].map(image => {
            const $image = $(image),
                  source = $image.attr('src')

            // Check if the other image is not a default one
            if (/default_\d{1,2}/.test(source)) return null

            return downloadImage(
                source, 
                `${settings.imgPath}${source.split('/').reverse()[0]}`,
                result => result.filename.split('/').reverse()[0],
                _ => null
            )
        }).filter(other_image => other_image !== null),
        how_to_use: $('span#HowToUse p')?.text().trim() || "",
        technical_description: $('span#TechnicalDescription p')?.text().trim() || "",
        product_description: $('span#ProductDescription p')?.text().trim() || "",
        bought_with: [...$('a.redirectProductId')].map(item => $(item).attr('href').match(/=(\d+)$/)[1]),
        rank: $('div.productAverageMainDiv').find('i.activeStar').length,
        reviews_count: parseInt($('span#spnReviewCount').text()),
        reviews: [...$('div.customerReviewsMainDiv')].map(review => {
            const $review = $(review)

            return {
                id: ++review_id,
                author: $review.find('div.customerName').text().trim(),
                posted_at: $review.find('div.starIconsForReviews span').text().trim(),
                rank: $review.find('span.productAverageMainDiv').find('i.activeStar').length,
                message: $review.find('div.customerReviewDetail span').text().trim()
            }
        })
    }
}

scrapeProduct(code)

I can't even filter the null values from my resulting array of image names because those promises don't resolve once I reach the filter function. I somehow was under the impression that

images: downloadImage(
    URL,
    filepath,
    resolve() {},
    reject() {}
)

will wait until the downloadImage function returns a value to the image property and then the filter function will be executed. On the other hand, given that I guess the execution flows to filter function long before my downloadImage function has any chance of resolving the promise, I'd chain a.then() to the downloadImage, but I can't, because the downloadImage is inside the return of the map() function - which is the one followed by the.filter() function in the code.

Any help would be much appreciated! Thank you!

PS: I'm pretty sure there's something elementary (logical) which I'm overseeing or I didn't understand properly and I apologize for wasting your time, but I'm struggling with this thing for two days now and I don't seem to have any more ideas ^_^

Answer 1

You are getting an array of Promises returned by the map() method, so you will need to use Promise.all() or one of its variants.

For example, here you get the array of promises of "images", then you use Promise.all() to wait for all promises to be resolved, and finally you chain a then() to use the values.

    const imagesPromises = [...$('img#imgProduct')].map(image => {
        const $image = $(image),
                source = $image.attr('src')

        return downloadImage(
            source, 
            `${settings.imgPath}${source.split('/').reverse()[0]}`,
            result => result.filename.split('/').reverse()[0],
            _ => 'product_default.jpg'
        )
    })

    return Promise.all(imagesPromises)
        .then(images => {
            return {
                images,
                ...
            }
        })

And here you have a possible implementation:

function loadProduct(body, code) {
    $ = cheerio.load(body)

    const result = {
        title: $('li.LongName').text().trim(),
        category: $('a#categoryTitleLink').text().trim(),
        min_price: parseFloat($('span.MinPrice').text()),
        max_price: parseFloat($('span.MaxPrice')?.text()) || parseFloat($('span.MinPrice').text()),
        points: parseFloat($('div.AddtoCartUnderText span').text()),
        variants: [...$('div.productDetailClassicRnd')].map(variant => {
            const $field = $(variant).find('input'),
                  item_code = $field.attr('item_code')

            if (item_code.split('-')[0] == code) return null

            return {
                code: item_code.split('-')[0],
                title: $field.attr('item_name'),
                image: downloadImage(
                    `${settings.urlVariant}${item_code.replace(' ', '%20')}`,
                    `${settings.imgPath}${item_code}`,
                    result => result.filename.split('/').reverse()[0],
                    _ => 'variant_default-VC.jpg'
                )
            }
        }).filter(variant => variant !== null),
        how_to_use: $('span#HowToUse p')?.text().trim() || "",
        technical_description: $('span#TechnicalDescription p')?.text().trim() || "",
        product_description: $('span#ProductDescription p')?.text().trim() || "",
        bought_with: [...$('a.redirectProductId')].map(item => $(item).attr('href').match(/=(\d+)$/)[1]),
        rank: $('div.productAverageMainDiv').find('i.activeStar').length,
        reviews_count: parseInt($('span#spnReviewCount').text()),
        reviews: [...$('div.customerReviewsMainDiv')].map(review => {
            const $review = $(review)

            return {
                id: ++review_id,
                author: $review.find('div.customerName').text().trim(),
                posted_at: $review.find('div.starIconsForReviews span').text().trim(),
                rank: $review.find('span.productAverageMainDiv').find('i.activeStar').length,
                message: $review.find('div.customerReviewDetail span').text().trim()
            }
        })
    }

    const imagesPromises = [...$('img#imgProduct')].map(image => {
        const $image = $(image),
                source = $image.attr('src')

        return downloadImage(
            source, 
            `${settings.imgPath}${source.split('/').reverse()[0]}`,
            result => result.filename.split('/').reverse()[0],
            _ => 'product_default.jpg'
        )
    })
    const otherImagesPromises = [...$('img.productDetailOtherIMG')].map(image => {
        const $image = $(image),
              source = $image.attr('src')

        // Check if the other image is not a default one
        if (/default_\d{1,2}/.test(source)) return null

        return downloadImage(
            source, 
            `${settings.imgPath}${source.split('/').reverse()[0]}`,
            result => result.filename.split('/').reverse()[0],
            _ => null
        )
    })

    return Promise.all(imagesPromises)
        .then(images => {
            result.images = images
            return Promise.all(otherImagesPromises)
        })
        .then(otherImages => {
            result.other_images = otherImages.filter(other_image => other_image !== null)
            return result
        }) 
}

scrapeProduct(code).then(product => console.log(product))