Node.js: work with slower third party

Question

I'm working with Node.js since 5 years and from 2 years on big projects with this framework. For two years, I'm confronted to a problem: how to work asynchronously and faster with non-async third party applications who's stacks requests, like MySQL, MongoDB or Apache SolR ?

I'm used to work with promises and to prepared several promises requests, like this:

const promises = []

for (let i = 0; i < 1000; i += 1) {
  const promise = mySqlRequest()
  promises.push(promise)
}

Promise.all(promises)
.then()
.catch()

This example will work but will send 1000 requests at the same time to MySQL server, who's will stacks these requests and become very slow, will consume very large quantity of RAM.

The best solution is to do only one big request, but in some case it's impossible and I'm forced to make recursive function, which comes down to be synchronous and slower.

So, what the best way to work fast and asynchronous with Node.js and a stacking third party ?

Answer 1

If sending all requests at once doesn't work and sending them one by one doesn't work either, you'd need something similar to a thread-pool where some arbitrary number of tasks execute simultaneously. This is easily implementable using promises, for example like this:

Promise.pooled = function(arr, num = 5) {
    return new Promise(function(resolve, reject) {
        var i = -1;
        var error = false;

        var end = function() {
            num--;
            if(num === 0) resolve();
        }

        var next = function() {
            if(error) return;
            i++;
            if(i >= arr.length)
                end();
            else
                arr[i]().then(next).catch(onerr);
        }

        var onerr = function() {
            if(error) return
            error = true
            reject.call(arguments)
        }

        for(var j = 0; j < num; j++)
            next()
    });
}

What this allows you is pass an array of functions as first argument, those functions should take no parameter and return a promise. It will then execute exactly num simultaneously. If one of the promises fail, it will fail its own promise aswell and stop executing (this is changeable easily).

Example:

 Promise.after = function(ms) { return new Promise(function(resolve) { setTimeout(resolve, ms) }); } Promise.pooled = function(arr, num = 5) { return new Promise(function(resolve, reject) { var i = -1; var error = false; var end = function() { num--; if(num === 0) resolve(); } var next = function() { if(error) return; i++; if(i >= arr.length) end(); else arr[i]().then(next).catch(onerr); } var onerr = function() { if(error) return error = true reject.call(arguments) } for(var j = 0; j < num; j++) next() }); } var test = [ afterH(1000), afterH(500), afterH(800), afterH(600), afterH(3000), afterH(300), afterH(900), afterH(2000), afterH(1500), afterH(900), afterH(700), afterH(600), afterH(700) ]; // helper function, returns a function which when invoked returns a promise function afterH(ms) { return function() { console.log("Starting one job") return Promise.after(ms); } } Promise.pooled(test, 3).then(function() {console.log("All jobs finished") }).catch(function() {console.log("Job failed")})