Node.js return a promise in recursive loop

Question

I am having trouble getting a loop to return nested promises in the correct order. I am trying to recursively loop through a directory with subdirectories to get files and copy them to a new directory. This functionality is happening inside a Promise chain since other processes need to complete in order after the files are copied.

The following code works but when I console.log out a string after the .then() it is logging before the promise has resolved.

The code is:

    let newDirectory = /// helper function that returns promise of mkdirSync(exampleDir)
    newDirectory.then(function(result){
        getAllFiles('/TestDir').then((result) => {
            console.log(result);
        });
        console.log('this should fire after the "result" from above');
        //// rest of promises in promise chain
    });
        

The recursive function I'm calling "getAllFiles" to go through "TestDir" and its sub folders and copy files to "ExampleDir"

const getAllFiles = function(dirPath, arrayOfFiles) {
   return new Promise((resolve, reject) => {
   var promises = [];
   files = fs.readdirSync(dirPath)
   arrayOfFiles = arrayOfFiles || []
   files.forEach(function(file) {
     if (fs.statSync(dirPath + "/" + file).isDirectory()) {
        arrayOfFiles =  resolve(getAllFiles(dirPath + "/" + file, arrayOfFiles));
      } else {
      promises.push(helper.copyFile(path.join(dirPath, "/", file),`/TestDir/${file}`))
     }
   }
 })
  Promise.all(promises).then(function(result){
    resolve(result);
    console.log('done with Promises');
  }).catch((error) => {
    reject(error);
  });
}

The helper that copies the files returns a promise after the file has been copied

exports.copyFile = function(file, destDir){
  return new Promise(function ( resolve, reject ){
        fs.copyFile(file, destDir, (err) => {
          if(err) reject( err );
          else resolve('Successfully copied');
        });
   });
}

This actually seems to work but I am concerned it wont work with a large set of data since the

console.log('this should fire after the "result" from above');

fires before the other logs. The console looks like:

this should fire after the "result" from above
done with Promises        /// how ever many files are copied
[ 'Successfully copied']   //// with a length of however many files are copied)
done with Promises        //// this is fired once

Is this a log to be expected, or should all of the promises resolve and be logged before the "'this should fire after the "result" from above'" line is logged?

Answer 1

Is this a log to be expected, or should all of the promises resolve and be logged before the "'this should fire after the "result" from above'" line is logged?

Yes, it is to be expected. You are writing your code as if it is synchronous when Promises are not meant for that. What is actually happening in the code snippet below is after the newDirectory Promise resolves, both the getAllFiles and console.log('this should fire after the "result" from above'); function will be executed immediately. That is, the console.log will not wait on the getAllFiles to resolve before executing.

let newDirectory = /// helper function that returns promise of mkdirSync(exampleDir)
newDirectory.then(function(result){
    getAllFiles('/TestDir').then((result) => {
        console.log(result);
    });
    console.log('this should fire after the "result" from above');
    //// rest of promises in promise chain
});

So if you wanted to change the order of the console.log to ensure that it is executed after the getAllFiles Promise resolves, you could rewrite as follows.

newDirectory.then(function(result){
    getAllFiles('/TestDir').then((result) => {
        console.log(result);
        // or you could add it here
    }).then(() => { 
      console.log('this should fire after the "result" from above');
    });
});

You should also notice that I am saying when the Promise resolves, not when the function has finished executing. There is a very important distinction. If we take your example above yet again and and say we wanted to perform some other action after all the files had been copied.

newDirectory.then(function(result){
    getAllFiles('/TestDir').then((result) => {
       ...
    });
}).then(() => {
    console.log('doing some other task');
});

In the above example, what would happen is the newDirectory Promise would resolve, then the getAllFiles would be invoked, and before the getAllFiles has finished executing, the final console.log would be logged. This is an important principal of Promises, if you wish them to behave synchronously, you need to chain the Promises, ie you need to return the promise through all the chained then functions. So to fix the above problem we would need to return the promise that is resolved from the getAllFiles function as follows

newDirectory.then(function(result){
    return getAllFiles('/TestDir').then((result) => {
       ...
    });
}).then(() => {
    console.log('doing some other task');
});

Answer 2

fs/promises and fs.Dirent

Here's an efficient, non-blocking ls program using Node's fast fs.Dirent objects and fs/promises module. This approach allows you to skip wasteful fs.exist or fs.stat calls on every path.

Using async and await , we can avoid having to think much about how to specifically wire up the Promises -

// main.js
import { readdir } from "fs/promises"
import { join } from "path"

async function* ls (path = ".")
{ yield path
  for (const dirent of await readdir(path, { withFileTypes: true }))
    if (dirent.isDirectory())
      yield* ls(join(path, dirent.name))
    else
      yield join(path, dirent.name)
}

async function* empty () {}

async function toArray (iter = empty())
{ let r = []
  for await (const x of iter)
    r.push(x)
  return r
}

toArray(ls(".")).then(console.log, console.error)

Let's get some sample files so we can see ls working -

$ yarn add immutable     # (just some example package)
$ node main.js

[
  '.',
  'main.js',
  'node_modules',
  'node_modules/.yarn-integrity',
  'node_modules/immutable',
  'node_modules/immutable/LICENSE',
  'node_modules/immutable/README.md',
  'node_modules/immutable/contrib',
  'node_modules/immutable/contrib/cursor',
  'node_modules/immutable/contrib/cursor/README.md',
  'node_modules/immutable/contrib/cursor/__tests__',
  'node_modules/immutable/contrib/cursor/__tests__/Cursor.ts.skip',
  'node_modules/immutable/contrib/cursor/index.d.ts',
  'node_modules/immutable/contrib/cursor/index.js',
  'node_modules/immutable/dist',
  'node_modules/immutable/dist/immutable-nonambient.d.ts',
  'node_modules/immutable/dist/immutable.d.ts',
  'node_modules/immutable/dist/immutable.es.js',
  'node_modules/immutable/dist/immutable.js',
  'node_modules/immutable/dist/immutable.js.flow',
  'node_modules/immutable/dist/immutable.min.js',
  'node_modules/immutable/package.json',
  'package.json',
  'yarn.lock'
]

See this related Q&A for a dir program that recursively lists directories, a search program for finding files, and more.

Answer 3

If you want to console log after a Promise , you'll have to do it in a .then() like

Promise.all(promises).then(function(result) {
  resolve(result);
  console.log('done with Promises');
})
  .then(() => console.log("this should fire after the "result" from above"))
  .catch((error) => reject(error));

This is because Promise s are non-blocking and anything after it won't wait for it to finish before executing.