如何使用PhantomJS和node.js进行抓取？

Question

I have installed node-phantom by npm install node-phantom but when I am running this code, it is giving Cannot find module 'webpage' this error

var webpage = require('webpage').create(),
    url = "https://www.example.com/cba/abc",
    hrefs = new Array();
webpage.open(url,function(status){
    if(status=="success"){
        var results = page.evaluate(function(){
            $("#endpoints").each(function() {
                  hrefs.push($(this).attr("href"));
            });
            return hrefs;
        });
        console.log(JSON.stringify(results));
        phantom.exit();
    }
});

Answer 1

You don't require the webpage module in node-phantom. You would use its API to get a representation of the webpage module. It has to be done this way, because PhantomJS has a different execution runtime from node.js. They generally can't use the same modules. That is why there are bridges between those two execution environments like node-phantom and phantom . They essentially replicate the API of PhantomJS to be used in node.js.

As per documentation, you don't require the webpage, you get a page instead:

var phantom = require('node-phantom');
phantom.create(function(err,ph) {
  return ph.createPage(function(err,page) {
    // do something with page: basically your script
  });
});

You won't be able to just copy and paste existing PhantomJS code. There are differences, so you will have to study the API (basically the README on github).

Complete translation of your code:

var phantom = require('node-phantom');
phantom.create(function(err,ph) {
  return ph.createPage(function(err,page) {
    page.open(url,function(status){
      if(status=="success"){
        page.evaluate(function(){
          hrefs = [];
          $("#endpoints").each(function() {
            hrefs.push($(this).attr("href"));
          });
          return hrefs;
        }, function(err, results){
          console.log(JSON.stringify(results));
          ph.exit();
        });
      }
    });
  });
});

page.evaluate is still sandboxed, so you can't use variables from the outside like hrefs .