[英]Scraping javascript webpage with phantomJS
I am attempting to scrape a dynamic webpage with phantomJS. 我正在尝试使用phantomJS抓取动态网页。 Below is the code and url I'm trying to scrape. 以下是我要抓取的代码和网址。 The code works for other urls, but this one always comes back as a blank html document. 该代码可用于其他网址,但该网址始终以空白html文档的形式返回。 Does anyone know how to fix this? 有谁知道如何解决这一问题?
I am not super familiar with javascript, so this code was copied from somewhere else. 我对javascript不太熟悉,因此这段代码是从其他地方复制的。 I have increased the timeout time from 2.5s to 30s and it did not make a difference. 我已将超时时间从2.5s增加到30s,但没有改变。
var url ='https://www.amazon.com/gp/profile/amzn1.account.AFJ6MBZ5CSY4R6K4USNMQ7JWEQCA/';
var page = new WebPage()
var fs = require('fs');
page.open(url, function (status) {
just_wait();
});
function just_wait() {
setTimeout(function() {
fs.write('page.html', page.content, 'w');
phantom.exit();
}, 30000);
}
This is how I solved such issues. 这就是我解决此类问题的方法。
app.js app.js
var url ='https://www.amazon.com/gp/profile/amzn1.account.AFJ6MBZ5CSY4R6K4USNMQ7JWEQCA/';
var steps=[];
var testindex = 0;
var loadInProgress = false;
//This is set to true when a page is still loading
/*********SETTINGS*********************/
var settings = require('./settings');
var webPage = require('webpage');
var page = webPage.create();
var fs = require('fs');
page.settings.userAgent = settings.userAgents.desktop;
page.settings.javascriptEnabled = true;
page.settings.loadImages = false;
//Script is much faster with this field set to false
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
page.viewportSize = {
width: settings.viewport.desktop.width,
height: settings.viewport.desktop.height
};
/*********SETTINGS END*****************/
console.log('All settings Loaded, Start With Execution');
/**********DEFINE STEPS THAT PHANTOM SHOULD DO***********************/
steps = [
function(){
console.log("Step 1 - Load Page => "+url);
page.open(url, function(status){
if(status === 'success'){
console.log('Loaded');
}else{
console.log('Error Loading Page. Try Logging In Again');
phantom.exit(0);
}
});
},
function(){
page.render('./test.png');
},
];
/**********END STEPS THAT PHANTOM SHOULD DO***********************/
interval = setInterval(executeRequestsStepByStep, 3000);
function executeRequestsStepByStep(){
if(loadInProgress == false && typeof steps[testindex] == "function") {
steps[testindex]();
testindex++;
return;
}
if(typeof steps[testindex] != "function") {
console.log("Quiting");
fs.write('page.html', page.content, 'w');
phantom.exit(0);
}
}
/*
* These listeners are very important in order to phantom work properly.
* Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
* Without this, we will get content of the page, even a page is not fully loaded.
*/
page.onLoadStarted = function() {
loadInProgress = true;
};
page.onLoadFinished = function() {
loadInProgress = false;
};
page.onConsoleMessage = function(msg) {
// console.log(msg);
};
phantom.onError = function(msg, trace) {
var msgStack = ['PHANTOM ERROR: ' + msg];
if (trace && trace.length) {
msgStack.push('TRACE:');
trace.forEach(function(t) {
msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line + (t.function ? ' (in function ' + t.function +')' : ''));
});
}
console.error(msgStack.join('\n'));
phantom.exit(1);
};
settings.js settings.js
module.exports = {
viewport: {
desktop: {
height: 663,
width: 1200
}
},
userAgents: {
desktop: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36'
}
};
I have tested with this and this works fine. 我已经对此进行了测试,并且效果很好。
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.