How to achieve recursive Promise calls in Node.js

Question

I am calling an API where I can only fetch 1000 records per request, I was able to achieve this using recursion.

I am now trying to achieve the same using promises, I am fairly new to Node.js and JavaScript too.

I tried adding the recursion code in an if else block but failed

var requestP = require('request-promise');

const option = {
    url: 'rest/api/2/search',
    json: true,
    qs: {
        //jql: "project in (FLAGPS)",
    }
}

const callback = (body) => {

    // some code
    .
    .
    .//saving records to file
    .
    //some code
    if (totlExtractedRecords < total) {  

        requestP(option, callback).auth('api-reader', token, true)
     .then(callback)
    .catch((err) => {
        console.log('Error Observed ' + err)
    })
    }
}

requestP(option).auth('api-reader', token, true)
    .then(callback)
    .catch((err) => {
        console.log('Error Observed ' + err)
    })

I want to execute the method using promise and in a synchronous way, ie I want to wait until the records are all exported to a file and continue with my code

Answer 1

I think its better to create your own promise and simply resolve it when your done with your recursion. Here's a simply example just for you to understand the approach

 async function myRecursiveLogic(resolveMethod, ctr = 0) { // This is where you do the logic await new Promise((res) => setTimeout(res, 1000)); // wait - just for example ctr++; console.log('counter:', ctr); if (ctr === 5) { resolveMethod(); // Work done, resolve the promise } else { await myRecursiveLogic(resolveMethod, ctr); // recursion - continue work } } // Run the method with a single promise new Promise((res) => myRecursiveLogic(res)).then(r => console.log('done')); 

Answer 2

Here's a clean and nice solution using the latest NodeJS features. The recursive function will continue executing until a specific condition is met (in this example asynchronously getting some data).

const sleep = require('util').promisify(setTimeout)

const recursive = async () => {
  await sleep(1000)
  const data = await getDataViaPromise() // you can replace this with request-promise

  if (!data) {
    return recursive() // call the function again
  }

  return data // job done, return the data
}

The recursive function can be used as follows:

const main = async () => {
  const data = await recursive()
  // do something here with the data
}

Answer 3

Using your code, I'd refactored it as shown below. I hope it helps.

const requestP = require('request-promise');
const option = {
    url: 'rest/api/2/search',
    json: true,
    qs: {
        //jql: "project in (FLAGPS)",
    }
};

/* 
    NOTE: Add async to the function so you can udse await inside the function 
*/

const callback = async (body) => {

    // some code

    //saving records to file

    //some code

    try {
        const result = await requestP(option, callback).auth('api-reader', token, true);
        if (totlExtractedRecords < total) {
            return callback(result);
        }
        return result;
    } catch (error) {
        console.log('Error Observed ' + err);
        return error;
    }
}

Answer 4

Created this code using feed back from Amir Popovich

const rp = require('Request-Promise')
const fs = require('fs')

const pageSize = 200

const options = {
    url: 'https://jira.xyz.com/rest/api/2/search',
    json: true,
    qs: {
        jql: "project in (PROJECT_XYZ)",            
        maxResults: pageSize,
        startAt: 0,
        fields: '*all'
    },
    auth: {
        user: 'api-reader',
        pass: '<token>',
        sendImmediately: true
    }
}



const updateCSV = (elment) => {
    //fs.writeFileSync('issuedata.json', JSON.stringify(elment.body, undefined, 4))
}


async function getPageinatedData(resolve, reject, ctr = 0) {
    var total = 0

    await rp(options).then((body) => {
        let a = body.issues
        console.log(a)
        a.forEach(element => {
            console.log(element)
            //updateCSV(element)
        });

        total = body.total

    }).catch((error) => {
        reject(error)
        return
    })

    ctr = ctr + pageSize
    options.qs.startAt = ctr

    if (ctr >= total) {
        resolve();
    } else {
        await getPageinatedData(resolve, reject, ctr);
    }
}

new Promise((resolve, reject) => getPageinatedData(resolve, reject))
    .then(() => console.log('DONE'))
    .catch((error) => console.log('Error observed - ' + error.name + '\n' + 'Error Code - ' + error.statusCode));