在 node.js 中使用 wappalyzer 和 puppeteer

Question

I am trying to build a scraper to monitor web projects automatically.

So far so good, the script is running, but now I want to add a feature that automatically analyses what libraries I used in the projects. The most powerful script for this job is wappalyser. They have a node package ( https://www.npmjs.com/package/wappalyzer ) and it's written that you can use it combined with pupperteer.

I managed to run pupperteer and to log the source code of the sites in the console, but I don't get the right way to pass the source code to the wappalyzer analyse function.

Do you guys have a hint for me?

I tryed this code but a am getting a TypeError: url.split is not a function

function getLibarys(url) {

  (async () => {
    const browser = await puppeteer.launch({ headless: true });
    const page = await browser.newPage();
    await page.goto(url);

// get source code with puppeteer
const html = await page.content();

const wappalyzer = new Wappalyzer();

(async function () {
  try {
    await wappalyzer.init()

    // Optionally set additional request headers
    const headers = {}

    const site = await wappalyzer.open(page, headers)

    // Optionally capture and output errors
    site.on('error', console.error)

    const results = await site.analyze()

    console.log(JSON.stringify(results, null, 2))
  } catch (error) {
    console.error(error)
  }

  await wappalyzer.destroy()
})()
await browser.close()
  })()
} 

Answer 1

Fixed it by using the sample code from wappalyzer.

function getLibarys(url) {


const Wappalyzer = require('wappalyzer');

 
  const options = {
    debug: false,
    delay: 500,
    headers: {},
    maxDepth: 3,
    maxUrls: 10,
    maxWait: 5000,
    recursive: true,
    probe: true,
    proxy: false,
    userAgent: 'Wappalyzer',
    htmlMaxCols: 2000,
    htmlMaxRows: 2000,
    noScripts: false,
    noRedirect: false,
  };
  
  const wappalyzer = new Wappalyzer(options)
  
  ;(async function() {
    try {
      await wappalyzer.init()


  // Optionally set additional request headers
  const headers = {}

  const site = await wappalyzer.open(url, headers)

  // Optionally capture and output errors
  site.on('error', console.error)

  const results = await site.analyze()

  console.log(JSON.stringify(results, null, 2))
} catch (error) {
  console.error(error)
}

await wappalyzer.destroy()
  })()
}