[英]Node.js + request + for loop : Runs twice
I created a simple scraper using cheerio and request client but it doesn't work the way I want.我使用cheerio和请求客户端创建了一个简单的刮板,但它不能按我想要的方式工作。
First I see all the "null returned, do nothing" messages on the terminal and then see the names, so I think it first checks all the urls that returns a null, then non-nulls.首先我在终端上看到所有“返回空值,什么都不做”的消息,然后看到名称,所以我认为它首先检查所有返回空值的 url,然后是非空值。
I want it to run in the right order, from 1 to 100.我希望它以正确的顺序运行,从 1 到 100。
app.get('/back', function (req, res) {
for (var y = 1; y < 100; y++) {
(function () {
var url = "example.com/person/" + y +;
var options2 = {
url: url,
headers: {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
}
};
request(options2, function (err, resp, body) {
if (err) {
console.log(err);
} else {
if ($ = cheerio.load(body)) {
var links = $('#container');
var name = links.find('span[itemprop="name"]').html(); // name
if (name == null) {
console.log("null returned, do nothing");
} else {
name = entities.decodeHTML(name);
console.log(name);
}
}
else {
console.log("can't open");
}
}
});
}());
}
});
If you are not using promises and you want to run the requests sequentially, then this is a common design pattern for running a sequential async loop:如果您不使用 promise 并且希望按顺序运行请求,那么这是运行顺序异步循环的常见设计模式:
app.get('/back', function (req, res) {
var cntr = 1;
function next() {
if (cntr < 100) {
var url = "example.com/person/" + cntr++;
var options2 = {
url: url,
headers: {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
}
};
request(options2, function (err, resp, body) {
if (err) {
console.log(err);
} else {
if ($ = cheerio.load(body)) {
var links = $('#container');
var name = links.find('span[itemprop="name"]').html(); // name
if (name == null) {
console.log("null returned, do nothing");
} else {
name = entities.decodeHTML(name);
console.log(name);
}
} else {
console.log("can't open");
}
// do the next iteration
next();
}
});
}
}
// start the first iteration
next();
});
If you want to make all the requests in parallel (multiple requests in flight at the same time) which will be a faster end result and then accumulate all the results in order at the end, you can do this:如果您想并行发出所有请求(同时进行多个请求),这将是更快的最终结果,然后在最后按顺序累积所有结果,您可以这样做:
// create promisified version of request()
function requestPromise(options) {
return new Promise(function(resolve, reject) {
request(options2, function (err, resp, body) {
if (err) return reject(err);
resolve(body);
});
});
}
app.get('/back', function (req, res) {
var promises = [];
var headers = {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
};
for (var i = 1; i < 100; i++) {
promises.push(requestPromise({url: "example.com/person/" + i, headers: headers}));
}
Promise.all(promises).then(function(data) {
// iterate through all the data here
for (var i = 0; i < data.length; i++) {
if ($ = cheerio.load(data[i])) {
var links = $('#container');
var name = links.find('span[itemprop="name"]').html(); // name
if (name == null) {
console.log("null returned, do nothing");
} else {
name = entities.decodeHTML(name);
console.log(name);
}
} else {
console.log("can't open");
}
}
}, function(err) {
// error occurred here
});
});
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.