How to handle asynchronous behaviour of Node.js - Puppeteer?

Question

So basically I am using puppeteer to download a CSV file and then read the CSV file and then create a new file with my naming structure and sore that data in the dropbox folder. This process is repeated again and again for different accounts. The problem is that due to the asynchronous behavior of the node.js, it starts downloading the other file and doesn't wait for the file writing process to get completed. Due to this problem, the file gets stored in the wrong folder since the folder name is also passed in the loop. How can I solve this issue?

     const puppeteer = require('puppeteer');
     var datetime = require('node-datetime');
     var dt = datetime.create();
     dt.offsetInDays(-2);
     var start_date = dt.format('Y-m-d');
     dt.offsetInDays(1);
     var end_date = dt.format('Y-m-d');
     dt.offsetInDays(0);
     var date =  dt.format('Y-m-d');
     var fs = require('fs'),
     path = require('path'),
     _ = require('underscore');
     var user_details = [
          {
               username : 'user1', 
               password : 'password1',
               folder_name : 'folder1'
          },
          {
               username : 'user2',
               password : 'password2',
               folder_name : 'folder2'
          },
          {
               username : 'user3',
               password : 'password3',
               folder_name : 'folder3'
          },
          {
               username : 'user4',
               password : 'password4',
               folder_name : 'folder4'

          }];


     (async () => {

          console.log(user_details.length); 
          for( var i=0; i<user_details.length; i++){               
          var username = user_details[i]['username'];
          var password = user_details[i]['password'];
          var folder_name = user_details[i]['folder_name'];
          console.log(username);
          console.log(folder_name);
          const browser = await puppeteer.launch({ headless: false});
          const page = await browser.newPage();
          await page.goto('url', {waitUntil: 'networkidle2'});
          await page.waitFor('input[name=name]');
          await page.$eval('input[name=name]', (el, _username) => el.value = _username, username);
          await page.$eval('input[name=password]', (el, _password) => el.value = _password, password);
          await page.click('input[type=submit]');
          await page.waitFor('.breadcrumb')
          await page.goto('url', {waitUntil: 'networkidle2'});
          await page.waitForSelector('.csv_toggle');
          await page.click('.action');
          await page.click('.fa-files-o');  
          await page.waitFor(10000);
          var dir = '/users/user/Downloads/';
          var files = fs.readdirSync(dir);

          // use underscore for max()
          var file_name =  _.max(files, function (f) {
               var fullpath = path.join(dir, f);

               // ctime = creation time is used
               // replace with mtime for modification time
               return fs.statSync(fullpath).ctime;
          });
          var filePath = path.join('/users/user/Downloads/',file_name);
          fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
               if (!err) {
                    lines = data;

                    fs.writeFile("/users/user/Dropbox/Puppeteer/"+folder_name+"/data_"+folder_name+"_"+date+"_.csv", data, function(err) {
                         if(err) {

                                   console.log(err);

                         }
                              console.log("The file was saved!");
                              console.log('Closing Browser');
                              browser.close();     
                         }); 
               } else {
                    console.log('-----SOMETHING IS WRONG-----');
               }
               });
              await page.waitFor(10000); 
     }
})();

The above code works perfectly fine since I have used delays of 10 seconds so that the writing process gets completed, but this is a kind of hack and definitely will not work when the internet connection is slow or the machine is slow. What I want is that it should first wait for the file to get completely downloaded and then wait for the file writing process to get completed then continue the loop for the next user.

Answer 1

Add await in fs.readFile & fs.writeFile methods too.

Use

let data = await fs.readFile(filePath, {encoding: 'utf-8'});
await fs.writeFile("/users/user/Dropbox/Puppeteer/"+folder_name+"/data_"+folder_name+"_"+date+"_.csv", data);
browser.close();

instead of

fs.readFile(filePath, { encoding: 'utf-8' }, function (err, data) {
        if (!err) {
            lines = data;

            fs.writeFile("/users/user/Dropbox/Puppeteer/" + folder_name + "/data_" + folder_name + "_" + date + "_.csv", data, function (err) {
                if (err) {

                    console.log(err);

                }
                console.log("The file was saved!");
                console.log('Closing Browser');
                browser.close();
            });
        } else {
            console.log('-----SOMETHING IS WRONG-----');
        }
    });