如何在解决之前等待承诺中的所有代码完成？ （但稍微复杂一点）

Question

Sorry for the very confusing question, I have this code that gets information from a website without any node modules or libraries. It is a list of users separated into different pages use ?page= at the end of the URL. I have managed to iterate through the pages and split up the raw HTML just right. However, my promise resolves before all the data is collected. How can I wait for everything to finish before I resolve the promise? I have tried countless solutions, but none seem to work. Please don't ask to use a node package, as my goal is to not use one :) A friend helped with the regex and splitting it up. Here is the code I am using:

function getData() {
    return new Promise((resolve, reject) => {
        let final = [] //the array of users returned in the end
        const https = require("https"), url = "https://buildtheearth.net/buildteams/121/members";
        https.get(url + "?page=1", request => { //initial request, gets the number of user pages.
            let rawList = '';

            request.setEncoding("utf8"), 
            request.on("data", data => {rawList += data}), 
            request.on("end", () => {
                if(request = (request = (request = rawList.substring(rawList.indexOf('<div class="pagination">'))).substring(0, request.indexOf("</div>"))).match(/<a(.+)>(.+)<\/a>/g)) {
                    for(let t = parseInt(request[request.length - 1].match(/(\d+)(?!.*\d)/g)), a = 1; a < t + 1; a++) { //iterates through member pages
                        https.get(url + "?page=" + a, request2 => { //https request for each page of members
                            let rawList2 = '';
                            
                            request2.setEncoding('utf8'), 
                            request2.on("data", data => {rawList2 += data}), 
                            request2.on("end", () => {
                                let i = rawList2.match(/<td>(.+)<\/td>/g); //finds table in HTML
                                if (i) 
                                    for (var t = 1; t < i.length; t += 3) //iterates through rows in table
                                        console.log(i[t].replace(/<td>/g, "").replace(/<\/td>/g, "")), /* logs element to the console (for testing) */
                                        final.push(i[t].replace(/<td>/g, "").replace(/<\/td>/g, "")); //pushes element to the array that is resolved in the end
                            })
                        })
                        
                    }
                }
                resolve(final) //resolves promise returning final array, but resolves before elements are added with code above
            })
        })
    })
}

If this helps, here is the website I am trying to get info from. I am still a little new to JS so if you could help, I would really appreciate it :)

Answer 1

I ended up turning each action into an async function with a try and catch block and then chained the functions together with .then() For the base (getting data from a website) I took inspiration from an article on Medium . Here is the site I am pulling data from, and here is the function to get data from a website:

const getData = async (url) => {
    const lib = url.startsWith('https://') ? https : http;
  
    return new Promise((resolve, reject) => {
        const req = lib.get(url, res => {
            if (res.statusCode < 200 || res.statusCode >= 300) {
                return reject(new Error(`Status Code: ${res.statusCode}`));
            }

            const data = [];

            res.on('data', chunk => data.push(chunk));
            res.on('end', () => resolve(Buffer.concat(data).toString()));
        });
  
        req.on('error', reject);
        req.end();
    });
};

and then I got the number of pages (which can be accessed by appending ?page=<page number> to the end of the url) with this this function:

const pages = async () => {
    try {
        let html = await getData('https://buildtheearth.net/buildteams/121/members',);
        let pages = await (html = (html = html.substring(html.indexOf('<div class="pagination">'))).substring(0, html.indexOf("</div>"))).match(/<a(.+)>(.+)<\/a>/g)
        let pageCount = await parseInt(pages[pages.length - 1].match(/(\d+)(?!.*\d)/g))
        return pageCount
    } catch (error) {
        console.error(error);
    }
}

and then I used the page count to iterate through the pages and add the HTML of each to an array with this function:

const getPages = async pageCount => {
    let returns = []
    try {
        for (page = 1; page <= pageCount; page++) {
            try {
                let pageData = await getData('https://buildtheearth.net/buildteams/121/members?page=' + page)
                returns.push(pageData)
            } catch (error) {
                return error
            }
        }
    } catch (error) {
        return error
    } finally {return returns}
}

and then I iterated through the array of strings of HTML of each page, and extracted the data I needed out of each with this function which would return the list of members I need:

const iteratePages = async pages => {
    if (!Array.isArray(pages)) return
    try {
        let returns = []
        await pages.forEach(page => {
            let list = page.match(/<td>(.+)<\/td>/g);
            if (list)
                for (var element = 1; element < list.length; element += 3)
                    returns.push(list[element].replace(/<td>/g, "").replace(/<\/td>/g, ""));
        })
        return returns
    } catch (error) {
        return error
    }   
}

And then it was a matter of chaining each together to get the array I needed:

pages().then(pageCount => getPages(pageCount)).then(pages => iteratePages(pages)).then(finalList => {console.log(finalList); console.log(finalList.length)})