Save JSON data from Node.js Get request to global variable or file

Question

I need to save data from GET requests to a variable and then save it in a file. However, in some cases GET request does not save data to global variables.

var fs = require("fs");
var http = require("http");
var request = require('request');

var tmp_json = {};
var g_last = 0;
var data = {};

 //request 1
http.get('server:api', (resp) => {
  let data = '';

  resp.on('data', (chunk) => {
    data += chunk;
  });

  resp.on('end', () => {
    tmp_json.server1 = {};
    tmp_json.server1 = JSON.parse(data);
    g_last = tmp_json.height; // 100500
    console.log(g_last); // 100500
  });

}).on("error", (err) => {
  console.log("Error: " + err.message);
});

 //request 2
http.get('server2:api', (resp) => {
  let data = '';

  resp.on('data', (chunk) => {
    data += chunk;
  });

  resp.on('end', () => {
    tmp_json.server2 = {};    
    tmp_json.server2 = JSON.parse(data);
    g_last = tmp_json.height; // 256
    console.log(g_last); // 256
  });

}).on("error", (err) => {
  console.log("Error: " + err.message);
});

console.log(g_last); // 0

data = JSON.stringify(tmp_json);
fs.writeFile('data.json', data, 'utf8'); // empty file

Also I was trying to do it with fs.createWriteStream , but again I can save one request to file, but if there more then one I catch only buffer data.

Answer 1

Your problem is that request1 and request2 are happening while you are writing the file. This is because of the async nature of node. The execution order looks something like this:

1. Declare empty variables
2. Request1 goes out
3. Request2 goes out
4. Write empty variables to file
5. Request1 comes back and writes to variables
6. Request2 comes back and writes to variables

One way to fix this would be with Promises . The following allows for the function in then to be executed after the promises in Promise.all([ ... ]) have resolved:

var fs = require("fs");
var http = require("http");

var tmp_json = {};
var g_last = 0;
var data = {};

 //request 1
var req1 = new Promise((resolve, reject) => {
  http.get('server:api', (resp) => {
    let data = '';

    resp.on('data', (chunk) => {
      data += chunk;
    });

    resp.on('end', () => {
      tmp_json.server1 = {};
      tmp_json.server1 = JSON.parse(data);
      g_last = tmp_json.height; // 100500
      console.log(g_last); // 100500
      resolve()
    });

  }).on("error", (err) => {
    console.log("Error: " + err.message);
    reject(error)
  });
});

 //request 2
var req2 = new Promise((resolve, reject) => {
  http.get('server2:api', (resp) => {
    let data = '';

    resp.on('data', (chunk) => {
      data += chunk;
    });

    resp.on('end', () => {
      tmp_json.server2 = {};
      tmp_json.server2 = JSON.parse(data);
      g_last = tmp_json.height; // 256
      console.log(g_last); // 256
      resolve()
    });

  }).on("error", (err) => {
    console.log("Error: " + err.message);
    reject(error)
  });
});

Promise.all([ req1, req2 ]).then(() => {
  console.log(g_last);
  data = JSON.stringify(tmp_json);
  fs.writeFile('data.json', data, 'utf8');
})

Edit:

function handleGet (url) {
  return new Promise((resolve, reject) => {
    http.get(url, (resp) => {
      let data = '';

      resp.on('data', (chunk) => {
        data += chunk;
      });

      resp.on('end', () => {
        tmp_json.server1 = {};
        tmp_json.server1 = JSON.parse(data);
        g_last = tmp_json.height; // 100500
        console.log(g_last); // 100500
        resolve()
      });

    }).on("error", (err) => {
      console.log("Error: " + err.message);
      reject(error)
    });
  })
}

// Then use it
Promise.all([
  handleGet('http://google.ca'),
  handleGet('http://somesite.com')
])