How to download 128 files asynchronously using wget

Question

I have the following Node script:

var fs = require('fs');
var url = require('url');
var http = require('http');
var exec = require('child_process').exec;


var DOWNLOAD_DIR = './downloads/';

var generate_width_and_height = function() {
  const random = Math.floor((Math.random() * 100) + 200);
  console.log(random);
  return random
}


var create_file_url = function() {
  return "http://placekitten.com/" + generate_width_and_height() + "/" + generate_width_and_height()
}


var download_file_wget = function(file_url, file_number) {
  // extract the file name
  var file_name = "file_" + file_number
  // compose the wget command
  var wget = 'wget -P ' + DOWNLOAD_DIR + ' ' + file_url;
  // excute wget using child_process' exec function

  var child = exec(wget, function(err, stdout, stderr) {
    if (err) throw err;
    else console.log(file_name + ' downloaded to ' + DOWNLOAD_DIR);
  });
};

It does successfully download 1 file. I need to asynchronously download 128 files at once. How can I do this with this code?

Answer 1

If the files requested are big consider use spawn instead of exec.

const http = require('http');
const exec = require('child_process').exec;

const DOWNLOAD_DIR = './downloads/';

const generate_width_and_height = function() {
  const random = Math.floor((Math.random() * 100) + 200);
  console.log(random);
  return random
}

const create_file_url = function() {
  return "http://placekitten.com/" + generate_width_and_height() + "/" + generate_width_and_height() 
}

const oneHundredTwentyEightElementsArray = Array.from(Array(127), (_,x) => x);
const oneHundredTwentyEightUrlsArray = oneHundredTwentyEightElementsArray.map( _ => create_file_url())

const download_file_wget = function(file_url, file_number) {
  // extract the file name
  const file_name = "file_" + file_number
  // compose the wget command
  const wget = 'wget -P ' + DOWNLOAD_DIR + ' ' + file_url;

  // excute wget using child_process' exec function
  const child = exec(wget, function(err, stdout, stderr) {
    if (err) throw err;
    else console.log(file_name + ' downloaded to ' + DOWNLOAD_DIR);
  });
};

for (let index = 0; index < oneHundredTwentyEightElementsArray.length; index++) {
    const url = oneHundredTwentyEightUrlsArray[index];
    download_file_wget(url, index)
}