Stream from a mongodb cursor to Express response in node.js

Question

I am toying around with all the fancy node.js/mongodb/express platforms, and stumbled across a problem:

app.get('/tag/:tag', function(req, res){
  var tag=req.params.tag;
  console.log('got tag ' + tag + '.');
  catalog.byTag(tag,function(err,cursor) {
     if(err) {
       console.dir(err);
       res.end(err);
     } else {
       res.writeHead(200, { 'Content-Type': 'application/json'});

       //this crashes
       cursor.stream().pipe(res);

     }
  });
});

As you probably guessed, catalog.byTag(tag, callback) does a find() query to Mongodb and returns the cursor

This leads to an error:

TypeError: first argument must be a string or Buffer

According to mongodb driver doc , I tried to pass this converter to stream() :

function(obj) {return JSON.stringify(obj);}

but that does not help.

Can anybody tell me how to correctly stream something to a response?

Or is the only solution a boilerplate to manually pump the data using the 'data' and 'end' events?

Answer 1

将游标流与JSONStream结合使用，将其传递给响应对象。

cursor.stream().pipe(JSONStream.stringify()).pipe(res);

Answer 2

Simple. .stream({transform: JSON.stringify});

Answer 3

A working combination of other answers here

app.get('/comments', (req, res) => {
  Comment.find()
    .cursor()
    .pipe(JSONStream.stringify())
    .pipe(res.type('json'))
})

http://mongoosejs.com/docs/api.html#query_Query-cursor

• cursor() returns a Node streams3 compatible stream and is preferred over the deprecated query.stream() interface.
• Piping to JSONStream.stringify() to combine documents into an array instead of single objects
• Piping to res.type('json') which sets the HTTP Content-Type header to application/json and returns itself (the response stream) again.

Answer 4

Your mongo stream is dumping objects into the res stream which can only handle strings or buffers (hence the error).

Luckily, streams are easy to pipe together so its not too hard to make a transform stream to stringify your data.

in node v0.10.21:

var util = require('util')
var stream = require('stream')
var Transform = stream.Transform

util.inherits(Stringer, Transform)

function Stringer() {
  Transform.call(this, { objectMode: true } )
  // 'object mode allows us to consume one object at a time

}

Stringer.prototype._transform = function(chunk, encoding, cb) {
  var pretty = JSON.stringify(chunk, null, 2) 
  this.push(pretty) // 'push' method sends data down the pike.
  cb() // callback tells the incoming stream we're done processing 
}

var ss = new Stringer()

db.createObjectStreamSomehow()
  .pipe(ss)
  .pipe(res)

hope that helps

Answer 5

Using mongoose and express:

function(req, res){
    var stream = database.tracks.find({}).stream();
    stream.on('data', function (doc) {
        res.write(JSON.stringify(doc));
    });
    stream.on('end', function() {
        res.end();
    });
}

Answer 6

A better and recent solution for this problem:

const { JsonStreamStringify } = require("json-stream-stringify");

app.get("/", (req, res) => {
  const cursor = db.collection("hello").find({}).limit(10).stream();
  x = new JsonStreamStringify(cursor).pipe(res);
  x.on("data", (doc) => {
    res.write(doc);
  });
});

The above solution is using https://www.npmjs.com/package/json-stream-stringify package.