I am new to puppeteer and learning to scrape a web page. The web page is structured in this way:
What I'm trying to do is to scrape all <p>
tags between the <h2> Status </h2>
and the <h2>Naam</h2>
. With my current code, I can scrape all <p>
tags on this page. Only now I try to scrape all <p>
tags after the <h2> Status </h2>
up to the <h2>Naam</h2>
.
My current code:
const puppeteer = require('puppeteer');
const plaatsengids = async (place) => {
//Creates a Headless Browser Instance in the Background
const browser = await puppeteer.launch();
//Creates a Page Instance, similar to creating a new Tab
const page = await browser.newPage();
//Navigate the page to url
await page.goto('https://plaatsengids.nl/'+place);
/* page.waitForSelector('.title').then(async function(){
const title = await page.$eval('.title', element => element.innerHTML);
})*/
//Finds the first element with the id 'hplogo' and returns the source attribute of that element
const Title = await page.$eval('.title', element => element.innerHTML);
const description = await page.$eval('.body p', element => element.innerHTML);
let content = await page.evaluate(() => {
let divs = [...document.querySelectorAll('.body p')];
return divs.map((div) => div.textContent.replace("- ",""));
});
//Closes the Browser Instance
await browser.close();
return content;
};
module.exports = plaatsengids;
The relevant web page is: https://www.plaatsengids.nl/Stein
You can use Node.compareDocumentPosition()
:
const puppeteer = require('puppeteer');
(async function main() {
try {
const browser = await puppeteer.launch();
const [page] = await browser.pages();
await page.goto('https://www.plaatsengids.nl/Stein');
const paragraphs = await page.evaluate(() => {
const status = document.querySelector('h2[name="status"]');
const naam = document.querySelector('h2[name="naam"]');
return [...document.querySelectorAll('p')]
.filter(p => p.compareDocumentPosition(status) & Node.DOCUMENT_POSITION_PRECEDING &&
p.compareDocumentPosition(naam) & Node.DOCUMENT_POSITION_FOLLOWING)
.map(p => p.innerText);
});
console.log(paragraphs);
await browser.close();
} catch (err) {
console.error(err);
}
})();
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.