如何使用 Puppeteer 返回不在页面源中的元素

Question

I'm trying to return some information from a page using the following code to select a page element and return some values within it:

const puppeteer = require('puppeteer');

function run (numberOfPages) {
    return new Promise(async (resolve, reject) => {
        try {
            if (!numberOfPages) {
                numberOfPages = 1;
            }
            const browser = await puppeteer.launch();
            const page = await browser.newPage();
            await page.setRequestInterception(true);
            page.on('request', (request) => {
                if (request.resourceType() === 'document') {
                    request.continue();
                } else {
                    request.abort();
                }
            });
            await page.goto('https://careers.google.com/jobs/results/');
            let currentPage = 1;
            let urls=[];
            while (currentPage <= numberOfPages) {
                await page.waitForSelector('a.gc-card');
                let newUrls = await page.evaluate(() => {
                    let results = [];
                    let items = document.querySelectorAll('a.gc-card');
                    items.forEach((item) => {
                        results.push({
                            jobTitle: item.innerText,
                            url: item.getAttribute('href')
                        });
                    });
                    return results;
                });
                urls = urls.concat(newUrls);
                if (currentPage < pagesToScrape) {
                    await Promise.all([
                        await page.waitForSelector('a.gc-link gc-link--on-grey gc-action-group__item gc-h-larger-tap-target'),
                        await page.click('a.gc-link gc-link--on-grey gc-action-group__item gc-h-larger-tap-target'),
                        await page.waitForSelector('a.gc-link gc-link--on-grey gc-action-group__item gc-h-larger-tap-target')
                    ])
                }
                currentPage++;
                await page.waitFor(500);
            }
            browser.close();
            return resolve(urls);
        } catch (e) {
            return reject(e);
        }
    })
}
run(1).then(console.log).catch(console.error);

I can see using inspect in dev tools that the class gc-card is present in the dom when the page is loaded but for some reason await page.waitForSelector('a.gc-card'); times out every time I run the code. Not totally sure the reason for this, but think it could be something to do with the majority of the page body of the page being loaded through a script.

The desired outcome is to return an array with all the job titles and URLs on the page.

Answer 1

You request even is aborting all the javascript files the site needs to run.

page.on('request', (request) => {
   if (request.resourceType() === 'document') {
      request.continue();
   } else {
      request.abort();
   }
});

Instead of allowing only a document think in a negative way and stop the requests you are sure you won't need.