Node.js respond with asynchronous data

Question

Recently I started learning a little bit about Node.js and it's capabilities and tried to use it for some web services. I wanted to create a web service which will serve as a proxy for web requests. I wanted my service to work that way:

1. User will access my service -> http://myproxyservice.com/api/getuserinfo/tom
2. My service will perform request to -> http://targetsite.com/user?name=tom
3. Responded data would get reflected to the user.

To implement it I used the following code:

app.js:

var express = require('express');
var bodyParser = require('body-parser');
var app = express();

app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());

var proxy = require('./proxy_query.js')

function makeProxyApiRequest(name) {
  return proxy.getUserData(name, parseProxyApiRequest);
}

function parseProxyApiRequest(data) {
  returned_data = JSON.parse(data);
  if (returned_data.error) {
    console.log('An eror has occoured. details: ' + JSON.stringify(returned_data));
    returned_data = '';
  }
  return JSON.stringify(returned_data);
}

app.post('/api/getuserinfo/tom', function(request, response) {
  makeProxyApiRequest('tom', response);
  //response.end(result);
});

var port = 7331;

proxy_query.js:

var https = require('https');

var callback = undefined;

var options = {
    host: 'targetsite.com',
    port: 443,
    method: 'GET',
};

function resultHandlerCallback(result) {
    var buffer = '';

    result.setEncoding('utf8');
    result.on('data', function(chunk){
        buffer += chunk;
    });

    result.on('end', function(){
        if (callback) {
            callback(buffer);
        }
    });
}

exports.getUserData = function(name, user_callback) {
    callback = user_callback
    options['path'] = user + '?name=' + name;

    var request = https.get(options, resultHandlerCallback);

    request.on('error', function(e){
        console.log('error from proxy_query:getUserData: ' + e.message)
    });

    request.end();
}
app.listen(port);

I wish I didn't screwed this code because I replaced some stuff to fit my example.

Anyway, the problem is that I want to post the response to the user when the HTTP request is done and I cant find how to do so because I use express and express uses asynchronous calls and so do the http request. I know that if I want to do so, I should pass the makeProxyApiRequest the response object so he would be able to pass it to the callback but it is not possible because of asyn problems.

any suggestions? help will be appreciated.

Answer 1

As you're using your functions to process requests inside your route handling, it's better to write them as express middleware functions, taking the specific request/response pair, and making use of express's next cascade model:

function makeProxyApiRequest(req, res, next) {
  var name = parseProxyApiRequest(req.name);
  res.locals.userdata = proxy.getUserData(name);
  next();
}

function parseProxyApiRequest(req, res, next) {
  try {
    // remember that JSON.parse will throw if it fails!
    data = JSON.parse(res.locals.userdata);
    if (data .error) {
      next('An eror has occoured. details: ' + JSON.stringify(data));
    }
    res.locals.proxyData = data;
    next();
  }
  catch (e) { next("could not parse user data JSON.");  }
}

app.post('/api/getuserinfo/tom',
  makeProxyApiRequest,
  parseProxyApiRequest,
  function(req, res) {
    // res.write or res.json or res.render or
    // something, with this specific request's
    // data that we stored in res.locals.proxyData
  }
);

Even better would be to move those middleware functions into their own file now, so you can simply do:

var middleware = require("./lib/proxy_middleware");
app.post('/api/getuserinfo/tom',
  middleware.makeProxyApiRequest,
  middleware.parseProxyApiRequest,
  function(req, res) {
    // res.write or res.json or res.render or
    // something, with this specific request's
    // data that we stored in res.locals.proxyData
  }
);

And keep your app.js as small as possible. Note that the client's browser will simply wait for a response by express, which happens once res.write , res.json or res.render etc is used. Until then the connection is simply kept open between the browser and the server, so if your middleware calls take a long time, that's fine - the browser will happily wait a long time for a response to get sent back, and will be doing other things in the mean time.

Now, in order to get the name , we can use express's parameter construct:

app.param("name", function(req, res, next, value) {
  req.params.name = value;
  // do something if we need to here, like verify it's a legal name, etc.
  // for instance:
  var isvalidname = validator.checkValidName(name);
  if(!isvalidname) { return next("Username not valid"); }
  next(); 
});

...

app.post("/api/getuserinfo/:name", ..., ..., ...);

Using this system, the :name part of any route will be treated based on the name parameter we defined using app.param. Note that we don't need to define this more than once: we can do the following and it'll all just work:

app.post("/api/getuserinfo/:name", ..., ..., ...);
app.post("/register/:name", ..., ..., ... );
app.get("/api/account/:name", ..., ..., ... );

and for every route with :name, the code for the "name" parameter handler will kick in.

As for the proxy_query.js file, rewriting this to a proper module is probably safer than using individual exports:

// let's not do more work than we need: http://npmjs.org/package/request
// is way easier than rolling our own URL fetcher. In Node.js the idea is
// to write as little as possible, relying on npmjs.org to find you all
// the components that you need to glue together. If you're writing more
// than just the glue, you're *probably* doing more than you need to.
var request = require("request");
module.exports = {
  getURL: function(name, url, callback) {
   request.get(url, function(err, result) {
     if(err) return callback(err);
     // do whatever processing you need to do to result:
     var processedResult = ....
     callback(false, processedResult);
   });
  }
};

and then we can use that as proxy = require("./lib/proxy_query"); in the middleware we need to actually do the URL data fetching.