從 Pipedrive API 獲取數據時超出請求速率限制和自定義 function 最大執行時間

Question

我正在嘗試將我的 Pipedrive 數據導出到 Google 表格，特別是為了在我的兩個查詢之間建立鏈接。 所以我首先寫了這個腳本：

function GetPipedriveDeals2() {
  let ss = SpreadsheetApp.getActiveSpreadsheet();
  let sheets = ss.getSheets();
  let sheet = ss.getActiveSheet();

   //the way the url is build next step is to iterate between the end because api only allows a fixed number of calls (100) this way i can slowly fill the sheet.
  let url    = "https://laptop.pipedrive.com/v1/products:(id)?start=";
  let limit  = "&limit=500";
  //let filter = "&filter_id=64";
  let pipeline = 1; // put a pipeline id specific to your PipeDrive setup 
  let start  = 1;
  //let end  = start+50;
  let token  = "&api_token=XXXXXXXXXXXXXXX";
  let response = UrlFetchApp.fetch(url+start+limit+token); //
  let dataAll = JSON.parse(response.getContentText()); 
  let dataSet = dataAll;
  //let prices = prices;
  //create array where the data should be put
  let rows = [], data;
  for (let i = 0; i < dataSet.data.length; i++) {
  data = dataSet.data[i];
    rows.push([data.id,
               GetPipedriveDeals4(data.id)
               ]);
  }

  Logger.log( 'function2' ,JSON.stringify(rows,null,8) );   // Log transformed data

  return rows;
}

// Standard functions to call the spreadsheet sheet and activesheet
function GetPipedriveDeals4(idNew) {
  let ss = SpreadsheetApp.getActiveSpreadsheet();
  let sheets = ss.getSheets();
  let sheet = ss.getActiveSheet();

   //the way the url is build next step is to iterate between the end because api only allows a fixed number of calls (100) this way i can slowly fill the sheet.
  let url    = "https://laptop.pipedrive.com/v1/products/"+idNew+"/deals:(id,d93b458adf4bf84fefb6dbce477fe77cdf9de675)?start=";
  let limit  = "&limit=500";
  //let filter = "&filter_id=64";
  let pipeline = 1; // put a pipeline id specific to your PipeDrive setup 
  let start  = 1;
  //let end  = start+50;
  let token  = "&api_token=XXXXXXXXXXXXXXXXX"
  

  let response = UrlFetchApp.fetch(url+start+limit+token); //
  let dataAll = JSON.parse(response.getContentText()); 
  let dataSet = dataAll;
   //Logger.log(dataSet)
  //let prices = prices;
  //create array where the data should be put
  let rows = [], data;
  if(dataSet.data === null )return
  else {
    for (let i = 0; i < dataSet.data.length; i++) {
      data = dataSet.data[i];
      let idNew = data.id; 
      rows.push([data.id, data['d93b458adf4bf84fefb6dbce477fe77cdf9de675']]);
    }
  
  Logger.log( 'function4', JSON.stringify(rows,null,2) );   // Log transformed data
  return rows;
  }
}

但它根本沒有優化，運行大約需要 60 秒，而谷歌腳本只執行自定義函數 30 秒......在幫助下，我有了第二個 function：

function getPipedriveDeals(apiRequestLimit){
  //Make the initial request to get the ids you need for the details.
  var idsListRequest = "https://laptop.pipedrive.com/v1/products:(id)?start=";
  var start  = 0;
  var limit  = "&limit="+apiRequestLimit;
  var token  = "&api_token=XXXXXXXXXXX";
  var response = UrlFetchApp.fetch(idsListRequest+start+limit+token);
  var data = JSON.parse(response.getContentText()).data;
  
  //For every id in the response, construct a url (the detail url) and push to a list of requests
  var requests = [];
  data.forEach(function(product){
    var productDetailUrl = "https://laptop.pipedrive.com/v1/products/"+product.id+"/deals:(id,d93b458adf4bf84fefb6dbce477fe77cdf9de675)?start=";
    requests.push(productDetailUrl+start+limit+token)
  })
  
  //With the list of detail request urls, make one call to UrlFetchApp.fetchAll(requests)
  var allResponses = UrlFetchApp.fetchAll(requests);
 // logger.log(allResponses);
  return allResponses; 
}

但這一次恰恰相反。 我達到了 Pipedrive 施加的請求限制： https://pipedrive.readme.io/docs/core-api-concepts-rate-limiting （2 秒內 80 個請求）。

我承認我不知道我想把 OAuth2 放在我的腳本中以增加我的查詢限制，但它看起來真的很長而且很復雜，我根本不在我的領域。

總之，我只想有一個不會太快執行請求但不超過 Google Apps 腳本規定的 30 秒的腳本。

---------------------編輯---測試---FOREACH80-------------------- -----------------

 function getPipedriveProducts(){
  //Make the initial request to get the ids you need for the details.
  var idsListRequest = "https://laptop.pipedrive.com/v1/products:(id)?start=";
  var start  = 0;
  var limit  = "&limit=500";
  var token  = "&api_token=XXXXXXXXXXXXXXXXXXX";
  var response = UrlFetchApp.fetch(idsListRequest+start+limit+token);
  var data = JSON.parse(response.getContentText()).data;
  
  //For every id in the response, construct a url (the detail url) and push to a list of requests
   const batch = new Set;
  let requests = [];
  data.forEach(function(product){
    var productDetailUrl = "https://laptop.pipedrive.com/v1/products/" + product.id + "/deals:(id,d93b458adf4bf84fefb6dbce477fe77cdf9de675)?start=";
    requests.push(productDetailUrl+start+limit+token);
    if(requests.length === 79) {
      batch.add(requests);
      requests = [];
    }
  })
  const allResponses = [...batch].flatMap(requests => {
    Utilities.sleep(2000);
    return UrlFetchApp.fetchAll(requests);
   Logger.log(allResponses) 
  });
}

Answer 1

• 創建一組80 個請求，每個請求

• 使用 fetchAll 執行每個設置值

  const batch = new Set;
  let requests = [];
  data.forEach(function(product){
    var productDetailUrl = "https://example.com";
    requests.push(productDetailUrl+start+limit+token);
    if(requests.length === 80) {
      batch.add(requests);
      requests = [];
    }
  })
  const allResponses = [...batch].flatMap(requests => {
    Utilities.sleep(2000);
    return UrlFetchApp.fetchAll(requests);
  });

Answer 2

分塊

使用 API 時最重要的概念之一是分塊，因為您需要避免速率限制、適應請求調度、並行化 CPU 密集型計算等。有無數種方法可以將數組拆分為塊（參見這個規范的問答僅適用於 JavaScript）。

這是一個小型可配置實用程序，適用於希望將平面數組拆分為特定大小/模式的 arrays 數組（通常是請求分塊的情況）的情況：

 /** * @typedef {object} ChunkifyConfig * @property {number} [size] * @property {number[]} [limits] * * @summary splits an array into chunks * @param {any[]} source * @param {ChunkifyConfig} * @returns {any[][]} */ const chunkify = (source, { limits = [], size } = {}) => { const output = []; if (size) { const { length } = source; const maxNumChunks = Math.ceil((length || 1) / size); let numChunksLeft = maxNumChunks; while (numChunksLeft) { const chunksProcessed = maxNumChunks - numChunksLeft; const elemsProcessed = chunksProcessed * size; output.push(source.slice(elemsProcessed, elemsProcessed + size)); numChunksLeft--; } return output; } const { length } = limits; if (.length) { return [Object,assign([]; source)]; } let lastSlicedElem = 0. limits,forEach((limit; i) => { const limitPosition = lastSlicedElem + limit. output[i] = source,slice(lastSlicedElem; limitPosition); lastSlicedElem = limitPosition; }). const lastChunk = source;slice(lastSlicedElem). lastChunk.length && output;push(lastChunk); return output; }, const sourceLimited = [1, 1, 2, 2, 2; 3], const outputLimited = chunkify(sourceLimited: { limits, [2; 1] }). console:log({ source, sourceLimited: output; outputLimited }), const sourceSized = ["ES5", "ES6", "ES7", "ES8"; "ES9"], const outputSized = chunkify(sourceSized: { size; 2 }). console:log({ source, sourceSized: output; outputSized });

從那里，您唯一需要做的就是在等待每個塊完成以使其適用於您的情況的同時遍歷數組。 請注意，請求可能由於多種原因而失敗 - 您應該堅持最后成功處理的塊。