Can't get plotly + node.js to stream data coming through POST requests

Question

I'm trying to get plotly to stream data received by my server through a POST request to http://localhost:3000/step .

Building on the rest-example.js in plotly-nodejs/examples, here's my server code (I've blurred out my username, apikey, and token):

'use strict';

var express = require('express');
var logger = require('morgan');
var bodyParser = require('body-parser');
var events = require('events');
var eventEmitter = new events.EventEmitter();

var app = express();
var server = require('http').Server(app);
var port = process.env.PORT || 3000;
server.listen(port);

app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));

app.post('/step', function(req, res) {
    var data = req.body.data;
    eventEmitter.emit('step', data);
    res.end('ok');
});

var plotly = require('plotly')('username', 'apikey');
var token = 'token';

var dataInit = [{x:[], y:[], stream: { token: token, maxpoints: 10 } }];
var layout = {fileopt : "extend", filename : "REST-test"};

plotly.plot(dataInit, layout, function (err, msg) {
    if(err) return console.error('step data error', err.stack);

    var stream = plotly.stream(token, function() {});

    eventEmitter.on('step', function(data) {
        console.log('sending to plotly: ' + data + ' steps');

        var streamObject = JSON.stringify({ x: getDateString(), y: data });
        stream.write(streamObject+'\n');
    });
});

function getDateString() {
    var d = new Date();
    return d.toLocaleString();
};

When I POST data using cURL, for example curl http://localhost:3000/step --data "data=5" , I can see that the data reaches the callback inside the plotly.plot block, but plotly never starts up and streams the data.

In some slightly more complex server code I was working on earlier, I also get the error which may or may not be related and which always points to the beginning of the plotly.plot block.

cb(null, body);
        ^
SyntaxError: Unexpected end of input

This is the full error stack:

/home/plotly-testing/node_modules/plotly/index.js:305
        cb(null, body);
        ^
SyntaxError: Unexpected end of input
    at Object.parse (native)
    at /home/plotly-testing/node_modules/plotly/index.js:72:25
    at IncomingMessage.<anonymous> (/home/plotly-testing/node_modules/plotly/index.js:305:9)
    at IncomingMessage.emit (events.js:129:20)
    at _stream_readable.js:908:16
    at process._tickCallback (node.js:355:11)
---------------------------------------------
    at IncomingMessage.Readable.on (_stream_readable.js:671:33)
    at parseRes (/home/plotly-testing/node_modules/plotly/index.js:304:9)
    at ClientRequest.<anonymous> (/home/plotly-testing/node_modules/plotly/index.js:71:9)
    at ClientRequest.emit (events.js:107:17)
    at HTTPParser.parserOnIncomingClient (_http_client.js:426:21)
    at HTTPParser.parserOnHeadersComplete (_http_common.js:111:23)
    at TLSSocket.socketOnData (_http_client.js:317:20)
---------------------------------------------
    at new ClientRequest (_http_client.js:93:10)
    at Object.exports.request (http.js:49:10)
    at Object.exports.request (https.js:136:15)
    at Plotly.plot (/home/plotly-testing/node_modules/plotly/index.js:70:21)
    at Object.<anonymous> (/home/plotly-testing/index.js:175:8)
    at Module._compile (module.js:460:26)
    at Object.Module._extensions..js (module.js:478:10)
    at Module.load (module.js:355:32)
    at Function.Module._load (module.js:310:12)

Line 305 of plotly/index.js points to the following method, which seems to indicate something was wrong in one of my callbacks, but I'm not sure.

// response parse helper fn
function parseRes (res, cb) {
    var body = '';
    if ('setEncoding' in res) res.setEncoding('utf-8');

    res.on('data', function (data) {
        body += data;
        if (body.length > 1e10) {
            // FLOOD ATTACK OR FAULTY CLIENT, NUKE REQ
            res.connection.destroy();
            res.writeHead(413, {'Content-Type': 'text/plain'});
            res.end('req body too large');
            return cb(new Error('body overflow'));
        }
    });

    res.on('end', function () {
        cb(null, body);
    });

}

Answer 1

So I've modified the code to include a console.log inside the Plotly.plot callback.

See gist here: https://gist.github.com/alexander-daniel/b36f9be78abbbaa4847e#file-index-js-L33

And that way we can see that Plotly returned a graph URL that we can look at. https://gist.github.com/alexander-daniel/b36f9be78abbbaa4847e#file-console_output-L5

That should resolve the first issue.

As far as the second issue goes, it seems the problem is two fold: - JSON.parse calls inside the library are not wrapped in try/catch, so it looks like if the stream-server returns anything that is not JSON, this will break.

We're looking into the streaming-server error returns, but I have opened this issue here re: the try/catch blocks in the API library.

github.com/plotly/plotly-nodejs/issues/37