How can I use loops/recursion with promise-chains?

Question

Imagine for example that you want to store paginated data from an API to a database.

let db;
let pageitems = 35
var offset = 0;

dbConnect //establish connection to database 
  .then( fetch(apiLink+?offset=2) 
  .then( res => res.json())
  .then( res => {
     var total = res.count
     return collection.insertMany(res.data, {ordered: false})
     // If offset is less than total, I want to increase offset and go back to the fetch-event.
  .catch( err => {
    if(err.code !== 11000){log(err)}
    else{log({completed: err.result.nInserted, duplicates: 
    err.result.result.writeErrors.length});}
  })
  .then(() => {
    connection.close();
  })

Answer 1

Basically, you will want to wrap your fetch and insert into a function that you will call many times. See the below as an example to illustrate my point...

let db;
let pageitems = 35
var offset = 0;


var db = dbConnect() //establish connection to database 

function fetch_and_insert(offset) {
    db
    .then(fetch(apiLink + "?" + offset))
    .then(res => res.json())
    .then(res => {
        var total = res.count
        collection.insertMany(res.data, { ordered: false })
        .catch(err => {
            if (err.code !== 11000) { log(err) }
            else {
                log({
                    completed: err.result.nInserted, duplicates: err.result.result.writeErrors.length
                });
            }
        })
        if (offset < total) return fetch_and_insert(offset + pageitems)
        return null;
    })
}

fetch_and_insert(offset)
.then(() => {
    connection.close();
})

Answer 2

You could just use a regular loop:

 (async function() {
    const conn = await dbConnect;
    for(let offset = 0; true; offset++) { 
      const { data, count } = await (await fetch(`api?page=${offset}`)).json();
      // Exit if the page is empty
      if(count === 0) break;
      await collection.insertMany(data, { ordered: false });
    }
 })();

To speed that up you could execute multiple requests in parallel:

 const chunkSize = 10; // 10 in parallel
 for(let offset = 0; offset < chunkSize; offset++) {
   (async function() {
      const conn = await dbConnect;
      for(let offset2 = 0; true; offset2 += chunkSize) { 
        const { data, count } = await (await fetch(`api?page=${offset + offset2}`)).json();
        // Exit if the page is empty
        if(count === 0) break;
        await collection.insertMany(data, { ordered: false });
      }
   })();
 }