如何使用 Puppeteer Select RSS Feed 项目元素？

Question

I am trying to consume an RSS feed using Puppeteer and output a new feed, but so far every the value of every element in the new feed says "undefined." At first I thought it was due to the elements not having any attributes, but it seems that using querySelectorAll is also undefined when it should be grabbing the content of every item element in the feed. This is how I am querying the original feed

await page.goto(url, {waitUntil: 'networkidle2'});
let rssitems = await page.evaluate(() => {
    let results;
    let items = document.querySelectorAll('item');
    items.forEach((item) => {

        results += '<title>' + item.querySelector('title').innerText + '</title>';
        results += '<description>' + item.querySelector('description').innerText + '</description>';
        results += '<link>' + item.querySelector('link').innerText + '</link>';
        results += '<guid>' + item.querySelector('guid').innerText + '</guid>';
        results += '<pubDate>' + item.querySelector('pubDate').innerText + '</pubDate>';
    });
    return results;
});

Answer 1

If I understand correctly, you are trying to interact with RSS document. RSS is XML, not HTML, so you need API for Node classes, not HTMLElement classes. So instead of HTMLElement.innerText , you can try Node.textContent .

Answer 2

I ended up going with an RSS parser. This code uses Puppeteer to take a screenshot of images specified by the url attribute of an enclosure element. This is good for scraping sites that do not allow images to be downloaded using curl requests.

let Parser = require('rss-parser');
let parser = new Parser();
let rssitems = await (async () => {
        let results;
        let feed = await parser.parseURL(url);
        feed.items.forEach(item => {
            let enclurl = item.enclosure.url;
            const filepath = './images/' + unique() + '.jpg';
            takeScreenshot(enclurl, filepath)
                .then(() => {
                    console.log("Screenshot taken");
                })
                .catch((err) => {
                    console.log("Error occured!");
                    console.dir(err);
                });
            results += '<title><![CDATA[' + item.title.trim()  + ']]></title>';
            results += '<description><![CDATA[<img src="' + host + filepath.slice(1) + '">' + item.content.trim() + ']]></description>';
            results += '<link>' + item.link.trim() + '</link>';
            results += '<guid>' + item.guid.trim() + '</guid>';
            results += '<pubDate>' + item.pubDate.trim() + '</pubDate>';
        });
        return results;
    })();
async function takeScreenshot(enclurl, filepath) {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    await page.goto( enclurl,{waitUntil: 'networkidle2'});
    const buffer = await page.screenshot({
        path: filepath
    });

    await page.close();
    await browser.close();
    }function unique() {
    return 'xxxxxxxx-xxxx-xxxx-xxxxxxxxxxxx'.replace(/[x]/g, function(c) {
        var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8);
        return v.toString(16);
    });
}```