How can I create a Twitter stream using Node.js and Websockets?

Question

A few months ago (August 2011) I successfully created a node.js websockets server which connected to Twitter's Streaming API using basic HTTP user/password authentication. To do this, I employed Andre Goncalves' twitter-nodejs-websocket library.

Since creating this working implementation, Twitter has eliminated access to the streaming API via basic HTTP auth, in favor of OAuth . After this shift, I utilized Ciaran Jessup's node-oauth library, which has successfully given me access to the Streaming API again (when I run the server I am successfully outputting the tweets via console.log(tweet) -- see below ).

The problem now is that my websockets server is no longer working. When I run my server from the command line and hit the client web page from the browser, the websocket "onclose" event is immediately fired.

I've tried everything I can think of to get this working. Any help would be very greatly appreciated!

server.js

var sys    = require('sys'),
    http   = require('http'),
    ws     = require("./vendor/ws"),
    base64 = require('./vendor/base64'),
    arrays = require('./vendor/arrays')

var OAuth = require('./oauth/oauth').OAuth;

var consumer_key        = '[...]'; //removed for obvious security reasons...
var consumer_secret     = '[...]';
var access_token        = '[...]';
var access_token_secret = '[...]';

oa = new OAuth("https://twitter.com/oauth/request_token",
                "https://twitter.com/oauth/access_token", 
                consumer_key,
                consumer_secret,
                "1.0A",
                null,
                "HMAC-SHA1");

var request = oa.get("https://stream.twitter.com/1/statuses/filter.json?track=google", access_token, access_token_secret );

// Response Parsing -------------------------------------------- //

var clients = [];
var message = "";

request.addListener('response', function (response) {

    response.setEncoding('utf8');

    response.addListener("data", function (chunk) {

        message += chunk;

        var newlineIndex = message.indexOf('\r');
        // response should not be sent until message includes '\r'.
        // Look at the section titled "Parsing Responses" in Twitter's documentation.
        if (newlineIndex !== -1) {
            var tweet = message.slice(0, newlineIndex);

            clients.forEach(function(client){
                // Send response to all connected clients
                client.write(tweet);
            });

            // this just tests if we are receiving tweets -- we are: terminal successfully outputs stream //
            var pt = JSON.parse(tweet);
            console.log('tweet: ' + pt.text);
        }
        message = message.slice(newlineIndex + 1);
    });

});
request.end();

// Websocket TCP server

ws.createServer(function(websocket){
  clients.push(websocket);
  websocket.addListener("connect", function(resource){
    // emitted after handshake
    sys.debug("connect: " + resource);
  }).addListener("close", function(){
    // emitted when server or client closes connection
    clients.remove(websocket);
    sys.debug("close");
  });
}).listen(8081);


// This basic http server works, so we know this port is open.
//
// var http = require('http');
// http.createServer(function (req, res) {
//   res.writeHead(200, {'Content-Type': 'text/plain'});
//   res.end('Hello World\n');
// }).listen(8081);

client code

<script type="text/javascript" charset="utf-8">
    ws = new WebSocket("ws://ec2-67-202-6-10.compute-1.amazonaws.com:8081");
    ws.onmessage = function(evt) {
        console.log('tweet')
    };
    ws.onclose = function() {
        console.log("socket closed");
    };
    ws.onopen = function() {
        console.log("connected...");
    };
</script>

Answer 1

Maybe you updated the browser? The websocket spec is chaning rapidly. Anyway, I'd propose using socket.io because it will even still work with fallbacks if the browser is outdated or websockets got incompatible again or a crappy proxy is preventing websockets from working.

Answer 2

Have a look at this sample event stream (it uses server sent events) from a twitter stream:

https://github.com/chovy/nodejs-stream