I am trying to parse lots of data from external API endpoint to my Database, I keep getting error 429.
I've tried timeout and sleep lib but it doesn't work for some reason.
Here is the code:
async function(req, res) {
console.log("usa sam");
try {
let count = await Subjekti.estimatedDocumentCount();
let response = await axios.get(`x.com/?offset=${count}&limit=5000`, {
timeout: 5800,
headers: {
"Key": "x"
}
});
const subjects = response.data;
for (let subject of subjects) {
let companyDetails = await axios.get(`https://x.com=${subject.mbs}&expand_relations=true`, {
headers: {
"Key": "x"
}
});
await Subjekti({
mbs: subject.mbs,
sud_id_nadlezan: subject.sud_id_nadlezan,
sud_id_sluzba: subject.sud_id_sluzba,
oib: subject.oib,
ino_podruznica: subject.ino_podruznica,
stecajna_masa: subject.stecajna_masa,
datum_osnivanja: subject.datum_osnivanja,
postupak: subject.postupak,
likvidacijska_masa: subject.likvidacijska_masa,
skracene_tvrtke: companyDetails.data.skracene_tvrtke && companyDetails.data.skracene_tvrtke.length > 0 ? companyDetails.data.skracene_tvrtke[0].ime : null,
ulica: companyDetails.data.sjedista && companyDetails.data.sjedista.length > 0 ? companyDetails.data.sjedista[0].ulica : null,
kucni_broj: companyDetails.data.sjedista && companyDetails.data.sjedista.length > 0 ? companyDetails.data.sjedista[0].kucni_broj : null,
naziv_naselja: companyDetails.data.sjedista && companyDetails.data.sjedista.length > 0 ? companyDetails.data.sjedista[0].naziv_naselja : null,
naziv_zupanije: companyDetails.data.sjedista && companyDetails.data.sjedista.length > 0 ? companyDetails.data.sjedista[0].naziv_zupanije : null,
}).save();
console.log("spremija i cekam");
}
} catch (err) {
//Baci error 500...
return apiResponse.ErrorResponse(res, err);
}
}
Looking forward to your reply.
You are sending too many requests, and you get ratelimited.
You should change your logic to allow more data to be sent and retrieved with a single request, or delaying most part of your requests to a later stage.
To delay request using axios interceptors, take a look here:
Ive found out solution:
await new Promise(resolve => setTimeout(resolve, 5000));
Hope it helps someone and save you time.
Try using debounce approach in this case to avoid sending too many requests.
see an example here: https://www.freecodecamp.org/news/javascript-debounce-example/
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.