[英]Limit concurrency of promise being run
我正在尋找一個 Promise 函數包裝器,它可以在給定的 Promise 運行時限制/限制,以便在給定的時間只有一組該 Promise 正在運行。
在下面的例子中, delayPromise
不應該同時運行,它們都應該按照先到先得的順序一次運行一個。
import Promise from 'bluebird'
function _delayPromise (seconds, str) {
console.log(str)
return Promise.delay(seconds)
}
let delayPromise = limitConcurrency(_delayPromise, 1)
async function a() {
await delayPromise(100, "a:a")
await delayPromise(100, "a:b")
await delayPromise(100, "a:c")
}
async function b() {
await delayPromise(100, "b:a")
await delayPromise(100, "b:b")
await delayPromise(100, "b:c")
}
a().then(() => console.log('done'))
b().then(() => console.log('done'))
關於如何設置這樣的隊列的任何想法?
我有一個很棒的Benjamin Gruenbaum
的“去抖動”功能。 我需要修改它以根據它自己的執行而不是延遲來限制承諾。
export function promiseDebounce (fn, delay, count) {
let working = 0
let queue = []
function work () {
if ((queue.length === 0) || (working === count)) return
working++
Promise.delay(delay).tap(function () { working-- }).then(work)
var next = queue.shift()
next[2](fn.apply(next[0], next[1]))
}
return function debounced () {
var args = arguments
return new Promise(function (resolve) {
queue.push([this, args, resolve])
if (working < count) work()
}.bind(this))
}
}
我不認為有任何庫可以做到這一點,但實現自己實際上很簡單:
function queue(fn) { // limitConcurrency(fn, 1)
var q = Promise.resolve();
return function(x) {
var p = q.then(function() {
return fn(x);
});
q = p.reflect();
return p;
};
}
對於多個並發請求,它變得有點棘手,但也可以完成。
function limitConcurrency(fn, n) {
if (n == 1) return queue(fn); // optimisation
var q = null;
var active = [];
function next(x) {
return function() {
var p = fn(x)
active.push(p.reflect().then(function() {
active.splice(active.indexOf(p), 1);
})
return [Promise.race(active), p];
}
}
function fst(t) {
return t[0];
}
function snd(t) {
return t[1];
}
return function(x) {
var put = next(x)
if (active.length < n) {
var r = put()
q = fst(t);
return snd(t);
} else {
var r = q.then(put);
q = r.then(fst);
return r.then(snd)
}
};
}
順便說一句,您可能想看看actor 模型和CSP 。 他們可以簡化處理這些事情,也有一些 JS 庫可供他們使用。
示例
import Promise from 'bluebird'
function sequential(fn) {
var q = Promise.resolve();
return (...args) => {
const p = q.then(() => fn(...args))
q = p.reflect()
return p
}
}
async function _delayPromise (seconds, str) {
console.log(`${str} started`)
await Promise.delay(seconds)
console.log(`${str} ended`)
return str
}
let delayPromise = sequential(_delayPromise)
async function a() {
await delayPromise(100, "a:a")
await delayPromise(200, "a:b")
await delayPromise(300, "a:c")
}
async function b() {
await delayPromise(400, "b:a")
await delayPromise(500, "b:b")
await delayPromise(600, "b:c")
}
a().then(() => console.log('done'))
b().then(() => console.log('done'))
// --> with sequential()
// $ babel-node test/t.js
// a:a started
// a:a ended
// b:a started
// b:a ended
// a:b started
// a:b ended
// b:b started
// b:b ended
// a:c started
// a:c ended
// b:c started
// done
// b:c ended
// done
// --> without calling sequential()
// $ babel-node test/t.js
// a:a started
// b:a started
// a:a ended
// a:b started
// a:b ended
// a:c started
// b:a ended
// b:b started
// a:c ended
// done
// b:b ended
// b:c started
// b:c ended
// done
使用 throttled-promise 模塊:
https://www.npmjs.com/package/throttled-promise
var ThrottledPromise = require('throttled-promise'),
promises = [
new ThrottledPromise(function(resolve, reject) { ... }),
new ThrottledPromise(function(resolve, reject) { ... }),
new ThrottledPromise(function(resolve, reject) { ... })
];
// Run promises, but only 2 parallel
ThrottledPromise.all(promises, 2)
.then( ... )
.catch( ... );
我有同樣的問題。 我寫了一個庫來實現它。 代碼在這里。 我創建了一個隊列來保存所有的承諾。 當您將一些承諾推送到隊列時,隊列頭部的前幾個承諾將被彈出並運行。 一旦一個承諾完成,隊列中的下一個承諾也將被彈出並運行。 一次又一次,直到隊列中沒有Task
。 您可以查看代碼以了解詳細信息。 希望這個圖書館能幫到你。
- 您可以定義並發承諾的數量(幾乎同時請求)
- 一致的流程:一旦一個承諾解決,另一個請求開始,無需猜測服務器能力
- 對數據阻塞的魯棒性,如果服務器停止片刻,它只會等待,並且不會因為時鍾允許而啟動下一個任務
- 不要依賴第三方模塊,它是 Vanila node.js
第一件事是讓 https 成為一個承諾,所以我們可以使用等待來檢索數據(從示例中刪除)第二創建一個承諾調度程序,在任何承諾得到解決時提交另一個請求。 3 撥打電話
const https = require('https')
function httpRequest(method, path, body = null) {
const reqOpt = {
method: method,
path: path,
hostname: 'dbase.ez-mn.net',
headers: {
"Content-Type": "application/json",
"Cache-Control": "no-cache"
}
}
if (method == 'GET') reqOpt.path = path + '&max=20000'
if (body) reqOpt.headers['Content-Length'] = Buffer.byteLength(body);
return new Promise((resolve, reject) => {
const clientRequest = https.request(reqOpt, incomingMessage => {
let response = {
statusCode: incomingMessage.statusCode,
headers: incomingMessage.headers,
body: []
};
let chunks = ""
incomingMessage.on('data', chunk => { chunks += chunk; });
incomingMessage.on('end', () => {
if (chunks) {
try {
response.body = JSON.parse(chunks);
} catch (error) {
reject(error)
}
}
console.log(response)
resolve(response);
});
});
clientRequest.on('error', error => { reject(error); });
if (body) { clientRequest.write(body) }
clientRequest.end();
});
}
const asyncLimit = (fn, n) => {
const pendingPromises = new Set();
return async function(...args) {
while (pendingPromises.size >= n) {
await Promise.race(pendingPromises);
}
const p = fn.apply(this, args);
const r = p.catch(() => {});
pendingPromises.add(r);
await r;
pendingPromises.delete(r);
return p;
};
};
// httpRequest is the function that we want to rate the amount of requests
// in this case, we set 8 requests running while not blocking other tasks (concurrency)
let ratedhttpRequest = asyncLimit(httpRequest, 8);
// this is our datase and caller
let process = async () => {
patchData=[
{path: '/rest/slots/80973975078587', body:{score:3}},
{path: '/rest/slots/809739750DFA95', body:{score:5}},
{path: '/rest/slots/AE0973750DFA96', body:{score:5}}]
for (let i = 0; i < patchData.length; i++) {
ratedhttpRequest('PATCH', patchData[i].path, patchData[i].body)
}
console.log('completed')
}
process()
如果您不想使用任何插件/依賴項,則可以使用此解決方案。
假設您的數據在一個名為datas
的數組中
datas
數組中的數據,我們稱之為processData()
processData()
,直到datas
數組上沒有數據為止,讓我們調用該函數bufferedExecution()
。buffer_size
的數組bufferedExecution()
填充數組Promise.all()
或Promise.allSettled()
中解決這是一個工作示例,其中數據是數字,操作等待一段時間並返回數字的平方,它也隨機拒絕。
const datas = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]; // this datas array should not contain undefined values for this code to work const buffer_size = 3; const finishedPromises = []; // change this function to your actual function that processes data async function processData(item) { return new Promise((resolve, reject) => { // wait for some time setTimeout(() => { // randomly resolve or reject if (Math.random() > 0.5) { resolve(item ** 2); } else { reject("error message"); } }, 1500); }); } // this function executes one function per loop, but magic happens when you // execute the function below, multiple times async function bufferedExecution(callback, i) { return new Promise(async (resolve, reject) => { // take first vale to process let next = datas.shift(); // check if there is a value, (undefined means you have reached the end of datas array) while (next != undefined) { // just to show which function is running (index of function in array) console.log(`running function id: ${i}`); let result; try { // process data with your function's callback result = await callback(next); // result finishes without error finishedPromises.push({ input: next, result: result, }); } catch (error) { // rejected, so adds error instead of result finishedPromises.push({ input: next, error: error, }); } // get next data from array and goes to next iteration next = datas.shift(); } // once all that is done finish it resolve(); }); } // here is where the magic happens // we run the bufferedExecution function n times where n is buffer size // bufferedExecution runs concurrently because of Promise.all()/Promise.allsettled() const buffer = new Array(buffer_size) .fill(null) .map((_, i) => bufferedExecution(processData, i + 1)); Promise.allSettled(buffer) .then(() => { console.log("all done"); console.log(finishedPromises); // you will have your results in finishedPromises array at this point // you can use input KEY to get the actual processed value // first check for error, if not get the results }) .catch((err) => { console.log(err); });
輸出
// waits a while
running function id: 1
running function id: 2
running function id: 3
// waits a while
running function id: 1
running function id: 2
running function id: 3
// waits a while
running function id: 1
running function id: 2
running function id: 3
// waits a while
running function id: 1
running function id: 2
running function id: 3
// waits a while
running function id: 1
all done
[
{ input: 1, error: 'error message' },
{ input: 2, result: 4 },
{ input: 3, result: 9 },
{ input: 4, result: 16 },
{ input: 5, error: 'error message' },
{ input: 6, result: 36 },
{ input: 7, result: 49 },
{ input: 8, error: 'error message' },
{ input: 9, result: 81 },
{ input: 10, result: 100 },
{ input: 11, result: 121 },
{ input: 12, error: 'error message' },
{ input: 13, result: 169 }
]
串行運行異步進程的經典方法是使用async.js
和使用async.series()
。 如果你更喜歡基於 promise 的代碼,那么有一個async.js
的 promise 版本: async-q
使用async-q
您可以再次使用series
:
async.series([
function(){return delayPromise(100, "a:a")},
function(){return delayPromise(100, "a:b")},
function(){return delayPromise(100, "a:c")}
])
.then(function(){
console.log(done);
});
同時運行其中兩個將同時運行a
和b
但在每個內部它們將是順序的:
// these two will run concurrently but each will run
// their array of functions sequentially:
async.series(a_array).then(()=>console.log('a done'));
async.series(b_array).then(()=>console.log('b done'));
如果你想運行b
后a
接着把它的.then()
async.series(a_array)
.then(()=>{
console.log('a done');
return async.series(b_array);
})
.then(()=>{
console.log('b done');
});
如果您不想按順序運行每個,而是希望限制每個同時運行一定數量的進程,那么您可以使用parallelLimit()
:
// Run two promises at a time:
async.parallelLimit(a_array,2)
.then(()=>console.log('done'));
閱讀 async-q 文檔: https : //github.com/dbushong/async-q/blob/master/READJSME.md
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.