Node.js with Express: Push to an empty Array returns an empty Array

Question

I am listing all files from all directories in /home/myComputer/Desktop/Research , and then filtering them with an if statement to only get the .txt files that I would like to read and store into arrays. All works fine, but pushing the data into the arrays is not functioning. When I console log them, they return no value [] .

I tried promise as well as call back function, but they didn't work for me because I didn't know how to implement them properly.

app.get('/jsonData', function(req, res) {

    /* Define Arrays */
    var theFile = [];
    var theCategory = [];
    var theContent = [];

    var walk = function(dir, done) {
      var results = [];
      fs.readdir(dir, function(err, list) {
        if (err) return done(err);
        var i = 0;
        (function next() {
          var file = list[i++];
          if (!file) return done(null, results);
          file = dir + '/' + file;
          fs.stat(file, function(err, stat) {
            if (stat && stat.isDirectory()) {
              walk(file, function(err, res) {
                results = results.concat(res);
                next();
              });
            } else {
              results.push(file);
              next();
            }
          });
        })();
      });
    };


    //walk(process.env.HOME, function(err, results) {
    walk("/home/myComputer/Desktop/Research", function(err, results) {
      if (err) throw err;
      //console.log(results);

        results.map(function(val) {
            //Get the filename
            var fileName = val.match(/[^\/]+$/).join();
            //Get the category
            var category = val.substr(48).match(/[^\/]+/);

            if (fileName == 'written-speech.txt') {
                console.log('FOUND!: ' + fileName + ' Category: ' + category) //this works

                fs.readFile(val, 'utf8', function(err, contents) {
                    console.log(contents); // this works

                    theFile.push(fileName);
                    theCategory.push(category);
                    theContent.push(contents);
                });
            }
        })

    });

        console.log(theFile); // The problem: This returns an empty Array []
        console.log(theCategory); // The problem: This returns an empty Array []
        console.log(theContent); // The problem: This returns an empty Array []

});

I expect console.log(theFile); console.log(theCategory); and console.log(theContent); to return the data pushed in them.

Answer 1

The reason for this is that many callbacks in Javascript are asynchronous, which means both fs.readdir and fs.readFile are asynchronous and their callbacks are not called immediately but slightly later (please read about Event Loop in javascript). So at the moment, when you log your arrays they are empty and data to them will be pushed later, eg in future. To avoid this you can either use synchronous methods (fs.readdirSync and fs.readFileSync) which is ugly and can cause performance issues if the app has a lot of other asynchronous operations. If in your case it is just a simple script to read some data, it might be fine. And the other, preferred way is to use promises or some library for managing callbacks, eg async . Please read some articles regarding managing async code if these concepts are fully unfamiliar for you, eg https://dev.to/mrm8488/from-callbacks-to-fspromises-to-handle-the-file-system-in-nodejs-56p2 to get a basic understanding and see some use case examples.

Regarding your current version, there is no easy way to make it work without a lot of changes. It is better to rewrite it to use the concepts I described earlier.

Answer 2

walk is an asynchronous function because fs.readdir is an asynchronous method and the console.log statements are running (in a synchronous manner) before the callback of fs.readdir getting invoked.

You can console the values of these variables at the end inside the callback of walk.