Download videos in series one after the other in Node JS using async?

Question

I want to download videos one after the other in a series.

That is the first one should be completely downloaded before the second one starts & the second one should be completely downloaded before the third one starts & so on.

I have the following directory structure -

video-downloader
├── index.js
├── videos.js
├── package.json

package.json

{
  "name": "video-downloader",
  "version": "1.0.0",
  "main": "index.js",
  "license": "MIT",
  "dependencies": {
    "download": "^7.1.0"
  },
  "scripts": {
    "start": "node index"
  }
}

video.js

const videos = [
  {
    url: 'https://video.com/lesson1.mp4',
    name: 'Lesson 1',
  },
  {
    url: 'https://video.com/lesson2.mp4',
    name: 'Lesson 2',
  },
  .
  .
  .
  {
    url: 'https://video.com/lesson2.mp4',
    name: 'Lesson 100',
  }
]

index.js

const fs = require('fs')
const download = require('download')

const videos = require('./videos')

const OUTPUT_DIR = 'Downloads'

fs.mkdir(OUTPUT_DIR, () => {
   main()
})

const main = () => {
    videos.map((video, i) => {
        console.log(`Downloaded file ${i + 1} of ${videos.length} (${video.name})`)
        download(video.url).pipe(
            fs.createWriteStream(`${OUTPUT_DIR}/${video.name}.mp4`),
        )
    })
}

This downloads videos chunk by chunk parallelly. All the videos are downloaded at once but none of them gets completed before the other one starts.

How do I download it serially?

I know I should use something like http://caolan.github.io/async/ but it needs a function signature & I have videos as an array so I'm not sure how to go about it.

Answer 1

You can use the await keyword on standard for loops, and things will process in order, and wait on each download before proceeding.

const fs = require('fs')
const download = require('download')
const videos = require('./videos')
const util = require('util')

const mkdirAsync = util.promisify(fs.mkdir)

const OUTPUT_DIR = 'Downloads'

const main = async () => {
  await mkdirAsync(OUTPUT_DIR)

  for (let i = 0; i < videos.length; i++) {
    const video = videos[i]
    const data = await download(video.url)
    fs.writeFileSync(`${OUTPUT_DIR}/${video.name}.mp4`, data)
    console.log(`Downloaded file ${i + 1} of ${videos.length} (${video.name})`)
  }
}

main()

Answer 2

You can use .reduce with promises to resolve sequentially, as follows:

 const fs = require('fs') const sh = require('shelljs') const download = require('download') const videos = require('./videos') const OUTPUT_DIR = 'Downloads' sh.mkdir('-p', OUTPUT_DIR) videos = videos.reduce((acc, item) => { return acc.then(() => { return new Promise((resolve) => { // Here you are using it as a Duplex Stream, not a promise, // therefore, you must check when the stream emits the 'end' event // so you can proceed further let stream = download(video.url) .pipe(fs.createWriteStream(`${OUTPUT_DIR}/${video.name}.mp4`)); stream.on('end', () => { console.log(`stream done ${item}`); resolve(item); }) }) }); }, Promise.resolve()); // 'videos' is now a promise videos.then((lastPromise) => { // using reduce will return the last evaluated item(promise) // but reaching the last one means the promises before that have been resolved console.log('all files were downloaded'); }) 

Answer 3

Try async await for this. First download and then write in Sync.

const fs = require('fs');
const sh = require('shelljs');
const download = require('download');

const videos = require('./videos');

const OUTPUT_DIR = 'Downloads';

sh.mkdir('-p', OUTPUT_DIR);

videos.forEach(async (video, i) => {
  console.log(`Downloading ${video.name}. Fil${i + 1}/${videos.length} - `);
  const data = await download(video.url);
  fs.writeFileSync(`${OUTPUT_DIR}/${video.name}.mp4`, data);
});