很多HTTP请求并另存为json：连接ETIMEDOUT

Question

I have a problem with my code. I need to capture a lot of data by API. First, i had a problem with memory and now I see: connect ETIMEDOUT error

My code:

 var fetch = require('node-fetch'); const async = require('async'); const request = require('request'); var jsonfile = require('jsonfile'); var file = '/temp/data.json' let urls = []; for (var i = 1; i <= 608469; i++) { let url = "https://api.demo.com/v2.1/people?user_key=auth_key&page="+i+"&sort_order=created_at%20DESC"; urls.push(url); } function httpGet(url, callback) { const options = { url : url, json : true }; request(options, function(err, res, body) { callback(err, body); } ); } async.map(urls, httpGet, function (err, res){ if (err) return console.log(err); console.log(typeof(res)); jsonfile.writeFileSync(file, res); }); 

Answer 1

Just replace async's map method with eachLimit . Key here is limit . It still will run http request for each link, but in no more than limit paraller threads. Here is a doc: https://caolan.github.io/async/docs.html#eachLimit