how to combine sync and async function in nodejs

Question

I have problems with the synchronous and asynchronous functions on NodeJS, here is my problem,

I have 4 functions, a global variable (users), and a rendering function to have the html page.

the scarper1 and scarper2 function can be performed asynchronously, they scarpe a website and fill in the global user variable, and the data_selector1 cannot be done unless scarper1 has done the same for data_selector2 with the scarper2 functions.

1-I need that scarper1 and scarper2 work asynchronously and fill the available users, and only render the HTML page if the 2 scarpers have finished their work.

2-I need an animation to be displayed in the browser during the drilling process, how can I do this

Here is what I tried ..

var express = require('express');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var router = express.Router();

 /*  globale variable to populate */

    var users = { 
                'name':null,
                'age':null,


            };

//function of scarping link1

function scarper1(callback){
    console.log("-------------scarper---------");
    var url = 'link1';

request(
  {
    method: 'GET',
    url: 'http://api.myscarperwebservice.com/?url=' + url,
    headers: {
      Accept: 'application/json',
    },
  },
  function(error, response, body) {
    if (error) throw error;
    // call the data-selector1 after scarper are finish rendering
    data_selector1(body);




  }
);


}

//function of scarping link2
function scarper2(callback){
  console.log("-------------scarper2---------");
  var url = 'link2';

request(
  {
    method: 'GET',
    url: 'http://api.myscarperwebservice.com/?url=' + url,
    headers: {
      Accept: 'application/json',
    },
  },
   function(error, response, body) {
    if (error) throw error;
    // call the data-selector2 after scarper are finish rendering
    data_selector2(body);




  }
);


}


function data_selector1(body)
{
console.log("-------------data-selector---------");
  const $ = cheerio.load(body);
  $("div[class='.user']").each(function(i,elem){


            users['name'] =$(elem).find('span[class=".name]').text();

            users['age'] =$(elem).find('span[class=".age]').text();


  });

}

function data_selector2(body)
{
console.log("-------------data-selector2---------");
  const $ = cheerio.load(body);
  $("ul[class='.user']").each(function(i,elem){


        users['name'] =$(elem).find('li[class=".name]').text();

        users['age'] =$(elem).find('li[class=".age]').text();


  });

}




/* GET home page. */

router.post('/recherche', function(req, res, next) {
   // i dont know how to make it here to say that scarper1 and scarper2 can be executed async and to render page after that the two scarper are finished 
   // and while scarper are working to display animation in the client
    scarper1(function(results){
        console.log(results);res.render('rechercher', { title: 'Express' });

    });



});

Answer 1

You can use promise.all() to do that but before you can use it, you need to promisify your functions:

function scarper1(callback) {
    return new Promise((resolve, reject) => {
        console.log("-------------scarper---------");
        var url = 'link1';

        request(
            {
                method: 'GET',
                url: 'http://api.myscarperwebservice.com/?url=' + url,
                headers: {
                    Accept: 'application/json',
                },
            },
            function (error, response, body) {
                if (error) reject(error);
                // call the data-selector1 after scarper are finish rendering
                data_selector1(body);
                resolve('Done successfully');
            }
        );
    });
}

function scarper2(callback) {
    return new Promise((resolve, reject) => {
        console.log("-------------scarper2---------");
        var url = 'link2';
        request(
            {
                method: 'GET',
                url: 'http://api.myscarperwebservice.com/?url=' + url,
                headers: {
                    Accept: 'application/json',
                },
            },
            function (error, response, body) {
                if (error) reject(error);
                // call the data-selector2 after scarper are finish rendering
                data_selector2(body);
                resolve('Done successfully');
            }
        );
    });
}

let scarper1 = scarper1(function(results){
    console.log(results);res.render('rechercher', { title: 'Express' });

});
let scarper2 = scarper2(function(results){
    console.log(results);res.render('rechercher', { title: 'Express' });

});

Promise.all([scarper1, scarper2]).then(function(values) {
    console.log(values);
});

For more about promise.all check this docs.

An even better approach is using async.eachLimit() to loop through requests (asynchronously) but first you need to install async package then merge both scraper functions:

const async = require("async");
let urls = [
    'link1',
    'link2'
]
async.eachLimit(urls, 2, (url) => {
    console.log("-------------scarper---------");
    request(
        {
            method: 'GET',
            url: 'http://api.myscarperwebservice.com/?url=' + url,
            headers: {
                Accept: 'application/json',
            },
        },
        function (error, response, body) {
            if (error) reject(error);
            // call the data-selector1 after scarper are finish rendering
            if(url == 'link1')
                data_selector1(body);
            else
                data_selector2(body);

            resolve('Done successfully');
        }
    );
}, (err) => {
    console.log("Finished all urls")
});