使用 Cheerio 的 Nodejs Webscraping 功能在完成前返回

Question

I'm currently working on a simple web scraping nodejs program. It is based on cheerio and I get items from a website and extract some information from there.

As far as I understand it all functions I call inside the foreach loop are sync so they should execute from top to bottom. And because the foreach loop is also only a normal loop, which executes sync in js, the function should return my finished array. But instead it is getting undefined and when I log it inside directly to console it works(?).

function getIntensiv(){
    var intensivregister = [];
    request.post({url: 'SOMEURL', form: {SOMEFORM}}, function(err,res,body){
        var $ = cheerio.load(body);
        $('#dataList').children('tbody').children('tr').each(function(i, elem){
            var name = $(elem).children('td').first().text().trim().split("\n")[0].trim();                        
            var zipcity = $(elem).children('td').first().children('small').last().text();                         
            var streetnr = $(elem).children('td').first().children('br').last().prev().text();                    
            intensivregister.push({'name': name, 'zipcity': zipcity, 'streetnr': streetnr});
        });
        console.log(intensivregister);  //works and prints the finished array
        return intensivregister;        //returns undefined before function finished
    });
}

I would appreciate it if you could explain me where my mistake is and help me fix it.

Answer 1

function getIntensiv(){
const cheerio = require('cheerio')
const request = require('request')

var intensivregister = [];
request.get({url: 'https://www.w3schools.com/html/html_tables.asp'}, function(err,res,body){


var $ = cheerio.load(body);

    $('#customers').children('tbody').children('tr').each(function(i, elem){
        var name = $(elem).children('td').first().text().trim().split("\n")[0].trim();                        
        var zipcity = $(elem).children('td').first().children('small').last().text();                         
        var streetnr = $(elem).children('td').first().children('br').last().prev().text();                    
        intensivregister.push({'name': name, 'zipcity': zipcity, 'streetnr': streetnr});
    });
    console.log(intensivregister);  //works and prints the finished array
    return null;        //returns undefined before function finished
});
return null;   //***<---This is returning and not the above return. If no return statement is written then undefined is passed.***
};

var retrunVal = getIntensiv()
console.log(retrunVal);

Please find the highlighted comment

Answer 2

Ok I figured out that my idea of javascript was not how you should use it. I worked around my problem with getting rid of the idea of returning values from functions (which comes mainly from my experiences from async programming) and instead using callback parameters which I give to my function and call at the end of my request.

function getIntensiv(callback){
    var intensivregister = [];
    request.post(...);
    **callback(intensivregister);**
}

What also is working (and I think a better solution) is working with promises eg with request-promise and calling the callback in the finally call.