[英]PhantomJS crawl multiple pages
我正在嘗試讓我的PhantomJS使用while循環來爬網多個頁面,但是我意識到它是異步的,並且只返回最后一頁的結果。 有什么想法可以讓我返回每頁的結果嗎?
這是我的代碼如下:
var _output = {'cookies':[],'resources':{'js':[]}};
var fs = require('fs');
var file_h = fs.open('file.csv', 'r');
var line = file_h.readLine();
while(line)
{
var page = require('webpage').create(),
system = require('system'),
address;
console.log(line);
// open web page
phantom.cookiesEnabled = true;
address = line;
page.open(address, function (status) {
console.log("status " + status);
if(status=='success'){
_output.cookies = phantom.cookies; // record cookies
_output.cookies.forEach(function(cookie){
console.log("cookie " + cookie);
});
}else{
console.log('Unable to open provided URL: '+address);
phantom.exit(-2); // -2: unable to open provided URL
}
});
// to avoid errors detected while parsing the page (eg. Syntax Error, Type Error, etc.)
// getting into stdout, so breaking the JSON decoding of returned output.
page.onError = function (msg, trace) {
}
line = file_h.readLine();
}
file_h.close();
將Promise用於異步任務。
var page = require('webpage').create(),
system = require('system');
new Promise(function(resolve, reject) {
var _output = {'cookies':[],'resources':{'js':[]}};
var fs = require('fs');
var file_h = fs.open('file.csv', 'r');
var line = file_h.readLine();
var promises = [];
while(line)
{
promises.push(process_line(line));
}
file_h.close();
Promise.all(promises).then(resolve, reject);
function process_line(line) {
return Promise((resolve, reject) => {
var address;
console.log(line);
// open web page
phantom.cookiesEnabled = true;
address = line;
page.open(address, function (status) {
console.log("status " + status);
if(status=='success'){
_output.cookies = phantom.cookies; // record cookies
_output.cookies.forEach(function(cookie){
console.log("cookie " + cookie);
});
resolve(status)
}else{
reject('Unable to open provided URL: '+address);
}
});
// to avoid errors detected while parsing the page (eg. Syntax Error, Type Error, etc.)
// getting into stdout, so breaking the JSON decoding of returned output.
page.onError = function (msg, trace) {
reject(msg);
}
line = file_h.readLine();
});
}
})
.then((statuses) => {
// use the results
})
.catch((err) => {
// report an error
phantom.exit(-2); // -2: unable to open provided URL
});
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.