watching streaming HTTP response progress in NodeJS, express

Question

i want to stream sizeable files in NodeJS 0.10.x using express@4.8.5 and pipes. currently i'm doing it like this (in CoffeeScript):

app.get   '/', ( request, response ) ->
  input = P.create_readstream route
  input
    .pipe P.$split()
    .pipe P.$trim()
    .pipe P.$skip_empty()
    .pipe P.$skip_comments()
    .pipe P.$parse_csv headers: no, delimiter: '\t'
    .pipe response

( P is pipedreams .)

what i would like to have is something like

    .pipe count_bytes       # ???
    .pipe response
    .pipe report_progress response

so when i look at the server running in the terminal, i get some indication of how many bytes have been accepted by the client. right now, it is very annoying to see the client loading for ages without having any indication whether the transmision will be done in a minute or tomorrow.

is there any middleware to do that? i couldn't find any.

oh, and do i have to call anything on response completion? it does look like it's working automagically right now.

Answer 1

For your second question, you don't have to close anything. The pipe function handles everything for you, even throttling of the streams (if the source stream has more data than the client can handle due to poor download speed, it will pause the source stream until the client can consume again the source instead of using a bunch of memory server side by completely reading the source).

For your first question, to have some stats server side on your streams, what you could use is a Transform stream like:

var Transform = require('stream').Transform;
var util = require('util').inherits;

function StatsStream(ip, options) {
    Transform.call(this, options);
    this.ip = ip;
}

inherits(StatsStream, Transform);

StatsStream.prototype._transform = function(chunk, encoding, callback) {
    // here some bytes have been read from the source and are
    // ready to go to the destination, do your logging here
    console.log('flowing ', chunk.length, 'bytes to', this.ip);

    // then tell the tranform stream that the bytes it should
    // send to the destination is the same chunk you received...
    // (and that no error occured)
    callback(null, chunk);
};

Then in your requests handlers you can pipe like (sorry javascript):

input.pipe(new StatsStream(req.ip)).pipe(response)

I did this on top of my head so beware :)