How to push the data returned by a promise into an array?

Question

I am making a call to Udemy API. To make simultaneous calls, I am using a loop. By doing so, I am automatically incrementing the page numbers and trying to fetch the data from each page and store it into an array so that I can write all the data into a single file in json format. But all I am getting is an empty array. How do I access the value returned by the promise and store into the doc.table array?

My code:

const fetch=require("node-fetch");
const fs=require("fs");
let doc={};
doc.table=[];

for(let i=1;i<=10;i++){

fetch('https://www.udemy.com/api-2.0/courses/ page='+i+'&page_size=10&client_id=${client_id}&client_secret=${client_secret},{
      method:'GET',
      body:null,
      headers:{authorization: ${auth_code}}
      })
      .then(res=>res.json())
      .then(json=>doc.table.push(json))
};


fs.writeFile("UDEMY.json",JSON.stringify(doc),function(err){
    if(err) throw err;
    console.log("Complete");
});

Answer 1

I'd suggest using await so that your for loop will be paused for each iteration:

const fetch = require("node-fetch");
const fsp = require("fs").promises;

let doc = { table: []};

async function run() {
    for (let i = 1; i <= 10; i++) {

        let data = await fetch('https://www.udemy.com/api-2.0/courses/ page='+i+'&page_size=10&client_id=${client_id}&client_secret=${client_secret},{
              method:'GET',
              body:null,
              headers:{authorization: ${auth_code}}
        }).then(res=>res.json());

        doc.table.push(data);
    }

    await fsp.writeFile("UDEMY.json",JSON.stringify(doc));
    console.log("done");
}

run().catch(err => {
    console.log(err);
});

The other possibility is to run all the requests in parallel and use Promise.all() to know when they are all done. They key to both solutions is to use the promise that fetch() returns to control knowing when things are done.

If you really want to run them in parallel and you're sure that your target host will allow it, you can do this:

const fetch = require("node-fetch");
const fsp = require("fs").promises;

let doc = { table: []};

function run() {
    let promises = [];
    for (let i = 1; i <= 10; i++) {

        promises.push(fetch('https://www.udemy.com/api-2.0/courses/ page='+i+'&page_size=10&client_id=${client_id}&client_secret=${client_secret},{
              method:'GET',
              body:null,
              headers:{authorization: ${auth_code}}
        }).then(res=>res.json()));

    }
    return Promise.all(promises).then(data => {
        doc.table = data;
        return fsp.writeFile("UDEMY.json",JSON.stringify(doc));
    });

}

run().then(() => {
    console.log('done');
}).catch(err => {
    console.log(err);
});

And, if you want some level of parallel requests, but want to limit how many are in parallel, you can use mapConcurrent() described here .

Answer 2

You can try to check for current loop index, and write your file within last Promise fullfillment:

const fetch = require('node-fetch');
const fs = require('fs');

let url;
let counter = 10;
const doc = {
  table: []
};

for (let i = 1; i <= 10; i++) {
  url = `https://www.udemy.com/api-2.0/courses/page=${i}&page_size=10&client_id=${client_id}&client_secret=${client_secret}`;
  fetch(url, {
    method: 'GET',
    body: null,
    headers: {
      authorization: auth_code
    }
  })
  .then(res => res.json())
  .then(json => {
    // next line will not guarantee the order of pages
    // doc.table.push(json);
    // so we can use current loop index and counter
    doc.table[i] = json;
    // when counter is 0 we can write a file 
    if (!--counter) {
      fs.writeFile('UDEMY.json', JSON.stringify(doc), function(err) {
        if (err) {
          throw err;
        }
        console.log("Complete");
      });
    }
  })
};

I also fixed minor errors in your URL's template string...

Answer 3

You can also save the promises in an array and access them once each of them are fulfilled if the order is important

const promises = []
promises.push(...)
Promise.all(promises).then(data -> ....)

data will now be an array containing the results of the individual promises. You can merge or process them as pleased. Be aware that the above function only resolves once all prior promises are resolved.