[英]Implement high-perf proxy on NodeJS
我正在使用帶有Luminati代理的NodeJS ,但是當發生502錯誤時,我需要請求另一個ip 。 他們的文檔本身有一個非常完整的示例,但是它是如此完整,以至於使我感到困惑。
我無法使該示例適應我的情況。
這是他們的示例也處理502錯誤:
var request = require('request-promise');
var promise = require('bluebird'); // promises lib used by request-promise
var lookup = promise.promisify(require('dns').lookup);
var http = require('http');
var username = 'myusername';
var password = 'mypassword';
var port = 22225;
var at_req = 0;
var n_total_req = 1000;
var n_parallel_exit_nodes = 100;
var switch_ip_every_n_req = 50;
var max_failures = 3;
var req_timeout = 60*1000;
function main(){
http.Agent.defaultMaxSockets = Infinity;
for (var i=0; i < n_parallel_exit_nodes; i++)
new Session(i).start();
}
function Session(id){
this.id = id;
this.n_req_for_exit_node = 0;
this.fail_count = 0;
this.switch_session_id();
}
var proto = Session.prototype;
proto.start = proto.next = function(){
if (at_req >= n_total_req)
return this.cleanup(); // all done
at_req++;
var _this = this;
promise.try(function(){
if (!_this.have_good_super_proxy())
return _this.switch_super_proxy();
}).then(function(){
if (_this.n_req_for_exit_node==switch_ip_every_n_req)
_this.switch_session_id();
var options = {
url: 'http://lumtest.com/myip.json',
timeout: req_timeout,
pool: _this.pool,
forever: true,
proxy: _this.super_proxy_url,
};
return request(options);
}).then(function success(res){
console.log(res);
_this.fail_count = 0;
_this.n_req_for_exit_node++;
}, function error(err){
if (err.statusCode
&& !status_code_requires_exit_node_switch(err.statusCode))
{
// this could be 404 or other website error
_this.n_req_for_exit_node++;
return;
}
_this.switch_session_id();
_this.fail_count++;
}).finally(function(){
_this.next();
});
};
proto.have_good_super_proxy = function(){
return this.super_proxy_host && this.fail_count < max_failures;
};
proto.update_super_proxy_url = function(){
this.super_proxy_url = 'http://'+username+
'-session-'+
this.session_id+':'+password+'@'+this.super_proxy_host+':'+port;
};
proto.switch_session_id = function(){
connection_pool_cleanup(this.pool);
this.pool = {};
this.session_id = (1000000 * Math.random())|0;
this.n_req_for_exit_node = 0;
this.update_super_proxy_url();
};
proto.switch_super_proxy = function(){
var _this = this;
this.switch_session_id();
return promise.try(function(){
return lookup('session-'+_this.session_id+
'.'+
'zproxy.luminati.io');
}).then(function success(res){
_this.super_proxy_host = res;
_this.update_super_proxy_url();
});
};
proto.cleanup = function(){
connection_pool_cleanup(this.pool);
};
function connection_pool_cleanup(pool){
if (!pool)
return;
for (let key in pool)
{
var sockets = pool[key].sockets;
for (let name in sockets)
sockets[name].forEach(s=>s.destroy());
}
}
function status_code_requires_exit_node_switch(status_code){
return [403, 429, 502, 503].indexOf(status_code)>=0;
}
main();
我的代碼:
function add(req, res, next) {
request({ url: 'myurl', proxy: 'myproxy' }, (err, response, body) => {
if (err) {
return next(err);
}
res.json(body);
});
};
function list(req, res, next) {
request({ url: 'myurl', proxy: 'myproxy' }, (err, response, body) => {
if (err) {
return next(err);
}
res.json(body);
});
};
如何以與示例相同的方式調整代碼以請求另一個IP?
可以這樣編寫代碼以重試失敗:
const request = require('request-promise');
const co = require('co');
function add(req, res, next) {
co(function*(){
let last_err;
const max_try = 10, max_fail = 3;
for (let count=0, fail = 0; count<max_try; count++)
{
try {
let body = yield request({url: 'myurl', proxy: 'myproxy'});
res.json(body);
return;
} catch(err){
last_err = err;
fail++;
if (fail==max_fail)
{
fail = 0;
proxy = 'myproxy2'; // switch proxy
}
}
}
next(last_err);
});
};
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.