网页抓取 - 如何在有 Puppeteer JS 的可用链接时进行导航

Question

I want to perform a web scraping for all the data within the main table body in the url https://data.anbima.com.br/debentures/AGRU12/agenda ... However as it implements pagination, I am unable to get that done easily... I came up with the following code which is not workng... I am getting the error ReferenceError: list is not defined , though I have defned it right before the while loop...

const puppeteer = require('puppeteer');
const fs = require('fs');
(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto(`https://data.anbima.com.br/debentures/AGRU12/agenda`);
  await page.waitForSelector('.normal-text');
  var list = [];
  while (true) {
    let nextButton;
    await page.evaluate(async () => {
      const nodeList = document.querySelectorAll(
        '.anbima-ui-table > tbody > tr'
      );
      let nodeArray = [...nodeList];
      nextButton = document.querySelector('.anbima-ui-pagination__next-button');

      let listA = nodeArray
        .map((tbody) => [...tbody.children].map((td) => [...td.children]))
        .map((tr) =>
          tr.map((span) =>
            span[0].innerHTML
              .replace('<label class="flag__children">', '')
              .replace('</label>', '')
          )
        );
      list.push(listA);
    });

    if (!nextButton) {
      break;
    } else {
      await page.goto(nextButton.href);
    }
  }

  fs.writeFile('eventDates.json', JSON.stringify(list[0], null, 2), (err) => {
    if (err) throw new Error('Something went wrong');

    console.log('well done you got the dates');
  });
  await browser.close();
})();

Answer 1

List is undefined inside the callback function. You would need to return the array in page.evaluate and then use that returned array to push it to list.

const list = [];
while (true) {
    let nextButton;
    const listA = await page.evaluate(async () => {
        const nodeList = document.querySelectorAll(
            '.anbima-ui-table > tbody > tr'
        );
        let nodeArray = [...nodeList];
        nextButton = document.querySelector('.anbima-ui-pagination__next-button');

        return nodeArray
            .map((tbody) => [...tbody.children].map((td) => [...td.children]))
            .map((tr) =>
                tr.map((span) =>
                    span[0].innerHTML
                        .replace('<label class="flag__children">', '')
                        .replace('</label>', '')
                )
            );
    });
    list.push(...listA);

Edit: Corrected the last line in my example.