How to handle Google Apps script 6 minute execution time limit while fetching data from an API endpoint to google sheets?

Question

I have to fetch data from an API to google sheets which is supposed to plot nearly 1600 entries. However, the execution stops at 6 mins plotting only about 1000 entries. My initial code in the Apps Script was:

function myFunction() {
  var spreadsheet=SpreadsheetApp.getActive();
  var sheet=spreadsheet.getActiveSheet();
  var nextPage=1;
  sheet.clear();


  var headerRow=["ID","NAME","SOURCE","STATUS","PRICING_MIN_PRICE","PRICING_MAX_PRICE","LOACTION_COUNTRY","LOACTION_LOCALITY","IMAGES_COUNT","VIDEOS_COUNT","FEATURES_COUNT"];
  sheet.appendRow(headerRow);
  sheet.getRange(
    spreadsheet.getCurrentCell().getRow(),
    1, 1, sheet.getMaxColumns()).activate();
  spreadsheet.getActiveRangeList().setFontWeight('bold');
  

  
  var curr;
  while(nextPage){
    curr=nextPage;
    var apiURL=`https://base.amberstudent.com/api/v0/inventories?p=${curr}&limit=10&sort_key=relevance&sort_order=desc&statuses=active`;
    var response=UrlFetchApp.fetch(apiURL);
    var json=response.getContentText();
    var dataPoints=JSON.parse(json);
    var resArray=dataPoints.data.result;
    for(var i=0;i<resArray.length;i++){
      var id=resArray[i].id!=null?resArray[i].id:"";
      var name=resArray[i].name!=null?resArray[i].name:"";
      var source=resArray[i].source!=null?resArray[i].source:"";
      var status=resArray[i].status!=null?resArray[i].status:"";
      var pricing_min_price=resArray[i].pricing?resArray[i].pricing.min_price:"";
      var pricing_max_price=resArray[i].pricing?resArray[i].pricing.max_price:"";
      var location_country=(resArray[i].location&&resArray[i].location.country)?resArray[i].location.country.long_name:"";
      var location_locality=(resArray[i].location&&resArray[i].location.locality)?resArray[i].location.locality.long_name:"";
      var images_count=resArray[i].images.length;
      var vid_count=resArray[i].videos.length;
      var feature_count=resArray[i].features.length;
      var row=[id,name,source,status,pricing_min_price,pricing_max_price,location_country,location_locality,images_count,vid_count,feature_count];
      sheet.appendRow(row);

    }
    nextPage=dataPoints.data.meta.next;  //for every page the nextPage stores the value of the next page, and for the last page (159 approx),     nextPage=null
  }
  
}

This did not work as I already mentioned. After some searching in the internet, I found some ways to bypass the execution time and I modified my code as shown below:

var spreadsheet=SpreadsheetApp.getActive();
var sheet=spreadsheet.getActiveSheet();
var nextPage=1;   //set nextPage as a global variable so that it can be accessed by all functions


function isTimeUp(today) {
  var now = new Date();
  return now.getTime() - today.getTime() > 300000;  //setting up a limit of 5 minutes
}


function myFunction() {

  sheet.clear();

  var today=new Date();

  var headerRow=["ID","NAME","SOURCE","STATUS","PRICING_MIN_PRICE","PRICING_MAX_PRICE","LOACTION_COUNTRY","LOACTION_LOCALITY","IMAGES_COUNT","VIDEOS_COUNT","FEATURES_COUNT"];
  sheet.appendRow(headerRow);
  sheet.getRange(
    spreadsheet.getCurrentCell().getRow(),
    1, 1, sheet.getMaxColumns()).activate();
  spreadsheet.getActiveRangeList().setFontWeight('bold');
  

  
  var curr;
  while(nextPage){

    if (isTimeUp(today)) {
        // schedule a trigger for a different function
        ScriptApp.newTrigger("repeatFunction")
            .timeBased()
            .everyMinutes(5)
            .create();
        break;
    }
    curr=nextPage;
    var apiURL=`https://base.amberstudent.com/api/v0/inventories?p=${curr}&limit=10&sort_key=relevance&sort_order=desc&statuses=active`;
    var response=UrlFetchApp.fetch(apiURL);
    var json=response.getContentText();
    var dataPoints=JSON.parse(json);
    var resArray=dataPoints.data.result;
    for(var i=0;i<resArray.length;i++){
      var id=resArray[i].id!=null?resArray[i].id:"";
      var name=resArray[i].name!=null?resArray[i].name:"";
      var source=resArray[i].source!=null?resArray[i].source:"";
      var status=resArray[i].status!=null?resArray[i].status:"";
      var pricing_min_price=resArray[i].pricing?resArray[i].pricing.min_price:"";
      var pricing_max_price=resArray[i].pricing?resArray[i].pricing.max_price:"";
      var location_country=(resArray[i].location&&resArray[i].location.country)?resArray[i].location.country.long_name:"";
      var location_locality=(resArray[i].location&&resArray[i].location.locality)?resArray[i].location.locality.long_name:"";
      var images_count=resArray[i].images.length;
      var vid_count=resArray[i].videos.length;
      var feature_count=resArray[i].features.length;
      var row=[id,name,source,status,pricing_min_price,pricing_max_price,location_country,location_locality,images_count,vid_count,feature_count];
      sheet.appendRow(row);

    }
    nextPage=dataPoints.data.meta.next;  //for every page the nextPage stores the value of the next page, and for the last page (159 approx),     nextPage=null
  }
  
}


function repeatFunction(){
  while(nextPage){
    var curr=nextPage;
    var apiURL=`https://base.amberstudent.com/api/v0/inventories?p=${curr}&limit=10&sort_key=relevance&sort_order=desc&statuses=active`;
    var response=UrlFetchApp.fetch(apiURL);
    var json=response.getContentText();
    var dataPoints=JSON.parse(json);
    var resArray=dataPoints.data.result;
    for(var i=0;i<resArray.length;i++){
      var id=resArray[i].id!=null?resArray[i].id:"";
      var name=resArray[i].name!=null?resArray[i].name:"";
      var source=resArray[i].source!=null?resArray[i].source:"";
      var status=resArray[i].status!=null?resArray[i].status:"";
      var pricing_min_price=resArray[i].pricing?resArray[i].pricing.min_price:"";
      var pricing_max_price=resArray[i].pricing?resArray[i].pricing.max_price:"";
      var location_country=(resArray[i].location&&resArray[i].location.country)?resArray[i].location.country.long_name:"";
      var location_locality=(resArray[i].location&&resArray[i].location.locality)?resArray[i].location.locality.long_name:"";
      var images_count=resArray[i].images.length;
      var vid_count=resArray[i].videos.length;
      var feature_count=resArray[i].features.length;
      var row=[id,name,source,status,pricing_min_price,pricing_max_price,location_country,location_locality,images_count,vid_count,feature_count];
      sheet.appendRow(row);

    }
    nextPage=dataPoints.data.meta.next;  //for every page the nextPage stores the value of the next page, and for the last page (159 approx),     nextPage=null
    if (nextPage==null) {
      var triggers = ScriptApp.getProjectTriggers();
      for (var i = 0; i < triggers.length; i++) {
          // delete all triggers
          ScriptApp.deleteTrigger(triggers[i]);
      }
      break;
    }
  }

} 

I tried to set nextPage as a global variable and set up a trigger which called the repeatFunction method every 5 minutes. This however, creates something like an infinite loop. Data keeps gettind added into the spreadsheet. I cannot figure out how to overcome this problem as I am new to the concept of Google App Scripts and its usage. Please help me out with a solution for this. Please ask for more details if necessary. Thank you!

Answer 1

I believe your goal as follows.

• Your myFunction() works. But, you want to reduce the process cost of your script.

Modification points:

• In your script, appendRow is used in the loop. I thought that this might be one of the reason of your issue.
  • This can be also seen at the benchmark .
• Although I'm not sure about the detail of the API you want to use, in your query parameter for the endpoint, limit=10 is used. And, from I have to fetch data from an API to google sheets which is supposed to plot nearly 1600 entries. , if limit=10 is the number of values for one API call, 160 requests are required to be done. I thought that this will be another reason of your issue. In this case, I would like to propose to modify the value of limit .
  • In this modification, limit=2000 is used. In my test, no error occurs. But if an error occurs, please modify this value and test it again.

When above points are reflected to your script, it becomes as follows.

Modified script:

In this modification, please modify your myFunction() as follows.

function myFunction2() {
  var spreadsheet = SpreadsheetApp.getActive();
  var sheet = spreadsheet.getActiveSheet();
  var nextPage = 1;
  sheet.clear();
  var headerRow = ["ID", "NAME", "SOURCE", "STATUS", "PRICING_MIN_PRICE", "PRICING_MAX_PRICE", "LOACTION_COUNTRY", "LOACTION_LOCALITY", "IMAGES_COUNT", "VIDEOS_COUNT", "FEATURES_COUNT"];
  sheet.appendRow(headerRow);
  sheet.getRange(spreadsheet.getCurrentCell().getRow(),1, 1, sheet.getMaxColumns()).activate();
  spreadsheet.getActiveRangeList().setFontWeight('bold');
  var ar = [];
  var curr;
  while (nextPage) {
    curr = nextPage;
    var apiURL = `https://base.amberstudent.com/api/v0/inventories?p=${curr}&limit=2000&sort_key=relevance&sort_order=desc&statuses=active`;
    var response = UrlFetchApp.fetch(apiURL);
    var json = response.getContentText();
    var dataPoints = JSON.parse(json);
    var resArray = dataPoints.data.result;
    for (var i = 0; i < resArray.length; i++) {
      var id = resArray[i].id != null ? resArray[i].id : "";
      var name = resArray[i].name != null ? resArray[i].name : "";
      var source = resArray[i].source != null ? resArray[i].source : "";
      var status = resArray[i].status != null ? resArray[i].status : "";
      var pricing_min_price = resArray[i].pricing ? resArray[i].pricing.min_price : "";
      var pricing_max_price = resArray[i].pricing ? resArray[i].pricing.max_price : "";
      var location_country = (resArray[i].location && resArray[i].location.country) ? resArray[i].location.country.long_name : "";
      var location_locality = (resArray[i].location && resArray[i].location.locality) ? resArray[i].location.locality.long_name : "";
      var images_count = resArray[i].images.length;
      var vid_count = resArray[i].videos.length;
      var feature_count = resArray[i].features.length;
      var row = [id, name, source, status, pricing_min_price, pricing_max_price, location_country, location_locality, images_count, vid_count, feature_count];
      ar.push(row);
    }
    nextPage = dataPoints.data.meta.next;  //for every page the nextPage stores the value of the next page, and for the last page (159 approx),     nextPage=null
  }
  sheet.getRange(sheet.getLastRow() + 1, 1, ar.length, ar[0].length).setValues(ar);
}

Note:

• In this modification, setValues is used in stead of appendRow , and limit=2000 is used. In this case, the 1,585 values are retrieved by one API call. And in my environment, the process time of above modified script was about 20 seconds.

References:

• Benchmark: Reading and Writing Spreadsheet using Google Apps Script
• setValues(values)