NodeJS: how to run spawn in parallel properly?

Question

I am running a for loop in which the following is executed:

let perform_vrp = function() {
    //..
    perform_tsp();
}

let perform_tsp = function() {
  //..

  const pyProg = spawn('python3', [process.env.PWD + '/server/vrp_solver/tsp_solver.py', '/../../route_data/' + depot.city + '/' + moment(route.date_driven).format('Y-MM-DD'), 'morning',route.name]);

  winston.info('Solving the TSP for %s...', route.name);

  pyProg.stdout.on('data', function (data) {

    let result_string = data.toString();
    winston.info('Route result for %s is: %s', route.name, result_string);

    let result_array = eval(result_string); 
    //...
  });
}

It basically calls a python script for each item in the for loop.

However, when one script has been completed it finalises all the others as well and continues with stdout with the same 'data' for all items in the for loop.

How can I prevent this from happening and let stdout wait for the proper child process to complete?

UPDATE:

the above is triggered from the following command:

const winston = require('../../server/winston');
const perform_vrp = require('../../server/modules/vrp');
const moment = require('moment');
const Utils = require('../modules/utils');
let utils = new Utils();

let do_vrp = async function(next_delivery_date, cities) {
  winston.info('Generating routes for %s', next_delivery_date);
  for (let index in cities) {
    if (cities.hasOwnProperty(index)) {
      let city = cities[index];
      winston.info('Generating route for %s on %s', city, next_delivery_date);
      await perform_vrp(next_delivery_date,[city]);
      await utils.sleep(60000);
    }
  }
};


let process_routes = async function() {

  //...

  let morning_cities = ['Boston','Chicago'];
  await do_vrp(next_delivery_date.format('YYYY-MM-DD'), morning_cities);
};

process_routes();

Answer 1

The problem is you are using await inside the for loop to wait for the script to finish but you do not wrap the execution of the script in a Promise .

In perform_tsp

warp the spawn and the event in a Promise that resolve when the data event fire and resolve with the data received like this:

  let perform_tsp = function () {
  //..
  return new Promise((resolve, reject) => {
    const pyProg = spawn('python3', [process.env.PWD + '/server/vrp_solver/tsp_solver.py', '/../../route_data/' + depot.city + '/' + moment(route.date_driven).format('Y-MM-DD'), 'morning', route.name]);
    winston.info('Solving the TSP for %s...', route.name);
    pyProg.stdout.on('data', function (data) {
      resolve(data);
    })
  });
}

In perform_vrp

Just return the returned promise from perform_tsp so you can await on it in the for loop:

let perform_vrp = function () {
  //..
  return perform_tsp();
}

In do_vrp

Receive the data of the resolved promise which is the data of the spawn and after that you can perform the logic that was in the data event callback:

 let do_vrp = async function (next_delivery_date, cities) {
  winston.info('Generating routes for %s', next_delivery_date);
  for (let index in cities) {
    if (cities.hasOwnProperty(index)) {
      let city = cities[index];
      winston.info('Generating route for %s on %s', city, next_delivery_date);
      let data = await perform_vrp(next_delivery_date, [city]);
      let result_string = data.toString();
      winston.info('Route result for %s is: %s', route.name, result_string);
      let result_array = eval(result_string);
      await utils.sleep(60000);
    }
  }
};

The previous structure will run the spawns in series if you want to reun spawn in parallel use Promise.all api.

  let do_vrp = async function (next_delivery_date, cities) {
  winston.info('Generating routes for %s', next_delivery_date);

  const promises = cities.map((city) => {
    return perform_vrp(next_delivery_date, [city]);
  });

  const results = await Promise.all(promises);
  results.forEach((data) => {
    winston.info('Generating route for %s on %s', city, next_delivery_date);
    let result_string = data.toString();
    winston.info('Route result for %s is: %s', route.name, result_string);
    let result_array = eval(result_string);
  });
};