Save responses to multiple GET requests in a single local json file - node.js

Question

The problem:

I have a function that maps over countries and regions and creates an array of urls, then makes a GET request to each one. I want to save the responses in a single json file, and I want this function to handle that as well.

Expected results:

I expected to be able to run the function as needed (like when source data is updated), and get a new or updated local json file with all the data objects in one array.

Actual results:

A file with only one record, an array with the last response object.

What I've tried:

I tried using fs.writeFile and fs.readFile . I did not get any errors, but the resulting file had only one record, even though console showed all the requests being made. It seemed that each response was being written over the previous.

Minimum reproducable (node.js) example:

const fs = require('fs')

// subset of countries and codes for demo purposes
const countryDirList = [
  'antarctica',
 'central-asia',
 ]

const fbCountryCodes = [
 { "region": "antarctica", "codes": ["ay", "bv"] },
 { "region": "central-asia", "codes": ["kg", "kz"] },
]

const callingUrlsInSequence = async () => {
  fs.writeFile('./test.json', '[]', function (err) { 
    if (err) throw err
    console.log('File - test.json - was created successfully.')
  })
  try {
    const urlConstructor = countryDirList.map(async (country) => {
      console.log('countries mapped', country)
      return fbCountryCodes.filter(async (f) => {
        if (country === f.region) {
          const urls = f.codes.map(async (c) => {
            const response = await axios({
              method: 'get',
              url: `https://raw.githubusercontent.com/factbook/factbook.json/master/${country}/${c}.json`,
              responseType: 'json',
              headers: {
                'Content-Type': 'application/json',
              },
            })
           
            fs.readFile('./test.json', function (err, data) {
              let json = JSON.parse(data)
              json.push(response.data)
              setTimeout(() => {
                fs.writeFile('./test.json', JSON.stringify(json), function (err) { 
                  if (err) throw err
                  console.log('The "data to append" was appended to file!')
                })
              }, 1000)
            })
            return response.data
          })
          const dataArr = await Promise.all(urls)
          dataArr.map((item) =>
            console.log(
              'dataArr',
              item.Government['Country name']['conventional short form']
            )
          )
        }
      })
    })
  } catch (err) {
    console.log('axios error: ', err)
  }
}
callingUrlsInSequence()

I'm re-writing this question now because it kept getting downvoted, and I could see that it was not very concise.

I can also see now, that obviously, the fs.readFile inside the fs.writeFile is not going to work in the code I provided, but I'm leaving it there in case it might help someone else, combined with the solution I provided in response to my own question.

Answer 1

I ended up learning how to solve this problem with both node-fetch and axios. They are not exactly the same.

For both: First, check for existence of destination file, and create one if it's not already there.

    const createNew = () => {
      try {
        if (existsSync('./data.json')) {
          console.log('file exists')
          return
        } else {
          writeFile('./data.json', '[]', (error, data) => {
            if (error) {
              console.log('fs.writeFile - create new - error: ', error)
              return
            }
          })
        }
      } catch (err) {
        console.log('fs.existsSync error: ', err)
      }
    }
    
    createNew()

Then make the array of urls:

    const countryDirList = [...countries]
    
    const fbCountryCodes = [...codes]
    
    const urls = []
    
    // maybe a reducer function would be better, but my map + filter game is much stronger X-D
    const makeUrls = (countriesArr, codesArr) =>
      countriesArr.map((country) => {
        return codesArr.filter((f) => {
          if (country === f.region) {
            return f.codes.map((code) => {
              return urls.push(
                `https://raw.githubusercontent.com/factbook/factbook.json/master/${country}/${code}.json`
              )
            })
          }
        })
      })
    
    makeUrls(countryDirList, fbCountryCodes)

Next, make the requests.

Axios:

  fs.readFile('./data.json', (error, data) => {
  if (error) {
    console.log(error)
    return
  }

  Promise.all(
    urls.map(async (url) => {
      let response
      try {
        response = await axios.get(url)
      } catch (err) {
        console.log('axios error: ', err)
        return err
      }
      return response
    })
  )
    .then((res) => {
      const responses = res.map((r) => r.data)

      fs.writeFile('./data.json', JSON.stringify(responses, null, 2), (err) => {
        if (err) {
          console.log('Failed to write data')
          return
        }
        console.log('Updated data file successfully')
      })
    })
    .catch((err) => {
      console.log('axios error: ', err)
    })
})

Node-fetch:

  //same basic structure, readFile with fetch and write file inside
    fs.readFile('./data2.json', (error, data) => {
      if (error) {
        console.log(error)
        return
      }
    
      async function fetchAll() {
        const results = await Promise.all(
          urls.map((url) => fetch(url).then((r) => r.json()))
        )
    
        fs.writeFile('./data2.json', JSON.stringify(results, null, 2), (err) => {
          if (err) {
            console.log('Failed to write data')
            return
          }
          console.log('Updated data file successfully')
        })
      }
    
      fetchAll()
    })

Both methods produce exactly the same output: a json file containing a single array with however many response objects in it.