Windows Azure Node.js Internal Server Error

Question

I would get this error now and then when I hit this address:

http://apps.facebook.com/pazooza

iisnode encountered an error when processing the request.

HRESULT: 0x2
HTTP status: 500
HTTP reason: Internal Server Error
You are receiving this HTTP 200 response because system.webServer/iisnode/@devErrorsEnabled configuration setting is 'true'.

In addition to the log of stdout and stderr of the node.exe process, consider using debugging and ETW traces to further diagnose the problem.

You may get additional information about this error condition by logging stdout and stderr of the node.exe process.To enable logging, set the system.webServer/iisnode/@loggingEnabled configuration setting to 'true' (current value is 'false').

But when I hit refresh, the site loads correctly and everything is working fine. It is only when I leave the site for a couple of hours comeback and perhaps I would see this error again, or perhaps not. It's annoying because users who navigate to my site think it's broken and aren't inclined to hit refresh!

So my question is whether anybody else has encountered this type of problem before, where you've checked your code and everything seems correctly the site loads, but every now and then Node or IIS-Node would chuck this error.

My setup is as follows:

Server: Windows Azure/Websites (Share, 2 Instances) Language: Node.js Tools: Webmatrix (I used a template project Starter Site)

The most annoying thing is I've turned on logging of error messages etc, and the logging system isn't picking up this error, when I go to check my logs no error log gets created when this error occurs, so I haven't figured out a way to capture it. Anyone got any suggestions?

This is what my server.js file looks like:

var express = require('express')
  , connect = require('connect')
  , config = require('./utils/config')
  , observer = require('./utils/observer')
  , cronJob = require('cron').CronJob
  , azure = require('azure')
  , uuid = require('node-uuid')
  , db = require("./utils/database")
  , async = require('async')
  , serialNumbers = require("./utils/serialNumbers");

var app = module.exports = express.createServer();

// Create a cron job to listen for servicebus queue messages
var jobProcessServices = new cronJob({
    cronTime: '*/1 * * * * *',
    onTick: function () {
        observer.processQueue(function () { });
    },
    start: false
});

app.configure(function() { 
    app.set('views', __dirname + '/views');
    app.set('view engine', 'ejs');
    app.set('view options', {
      layout: false
    });
    app.use(express.favicon());
    app.use(express.bodyParser());
    app.use(express.methodOverride());
    app.use(connect.static(__dirname + '/public'));
    app.use(require('./middleware/siteTotals'));
    app.use(require('./middleware/authenticate'));
    app.use(express.cookieParser());
    app.use(express.session({ secret: 'ljklkdsflkfdj4848384' }));
    app.use(app.router);

    // start listening for queue messages
    jobProcessServices.start();
    console.log('starting job service\n\n');

    // create table stores
    var ts1 = azure.createTableService(config.storageAccount, config.storageAccessKey, config.tableHost);

    ts1.createTableIfNotExists('channels', function(error) {
        ts1.createTableIfNotExists('users', function(error) {
            ts1.createTableIfNotExists('snusers', function(error) {
                ts1.createTableIfNotExists('snchannels', function(error) {
                    var query = azure.TableQuery
                        .select()
                        .from('snchannels');

                    ts1.queryEntities(query, function(error, result) {
                        if(error === null && result.length == 0) {
                            // must be site initialization, generate serial numbers for channels and users
                            serialNumbers.generateNewNumbers('snchannels', config.maxNumber, config.usageRates[2], function() {
                                serialNumbers.generateNewNumbers('snusers', config.maxNumber, config.usageRates[2], function() {
                                    initializeDefaultQueues(function() {
                                        // Create default storage container
                                        var bc1 = azure.createBlobService(config.storageAccount, config.storageAccessKey, config.blobHost);
                                        bc1.createContainerIfNotExists(config.container, function () { });
                                    });
                                });
                            });
                        }
                        else initializeDefaultQueues(function() { });
                    }); 
                });
            });
        });
    });

    ts1.createTableIfNotExists('sitetotals', function(error) {
        if(error === null) {
            var query = azure.TableQuery
                .select()
                .from('sitetotals');

            ts1.queryEntities(query, function(error, siteTotals) {
                if(error === null && siteTotals.length == 0) {
                    // must be site initialization create defaults
                    ts1.insertEntity('sitetotals', { PartitionKey: 'users', RowKey: uuid(), Total: '0', Deleted: '0' }, function(error) {
                        ts1.insertEntity('sitetotals', { PartitionKey: 'channels', RowKey: uuid(), Total: '0', Deleted: '0' }, function(error){ });
                    });
                }
            });
        }
    });
});

/**
* ERROR MANAGEMENT
* -------------------------------------------------------------------------------------------------
* error management - instead of using standard express / connect error management, we are going
* to show a custom 404 / 500 error using jade and the middleware errorHandler (see ./middleware/errorHandler.js)
**/
var errorOptions = { dumpExceptions: true, showStack: true }
app.configure('development', function() { });
app.configure('production', function() {
    errorOptions = {};
});
app.use(require('./middleware/errorHandler')(errorOptions));

// static vars
app.helpers({ config: {
        fbAppNamespace: config.fbAppNamespace,
        siteUrl: config.siteUrl,
        canvasUrl: config.canvasUrl,
        appID: config.appID,
        commentPageSize: config.commentPageSize,
        videoPageSize: config.videoPageSize,
        channelPageSize: config.channelPageSize,
        userFollowPageSize: config.userFollowPageSize,
        followPageSize: config.followPageSize,
        friendPageSize: config.friendPageSize,
        pazoozaFbUrl: config.pazoozaFbUrl,
        pazoozaTwitterUrl: config.pazoozaTwitterUrl,
        anonymousUser: config.anonymousUser,
        usageRates: config.usageRates,
        categorys: config.categorys,
        channelTypes: config.channelTypes,
        ratings: config.ratings
    },
    serverYear: new Date().getFullYear()
});

// all views have access to these variables
// note: can't do http calls with these functions
app.dynamicHelpers({
    session: function (req, res) {
        return req.session;
    },
    signed_request: function (req, res) {
        return req.param('signed_request');
    }
});

/**
* ROUTING
* -------------------------------------------------------------------------------------------------
* include a route file for each major area of functionality in the site
**/

require('./routes/home')(app);
require('./routes/channel')(app);
require('./routes/user')(app);
require('./routes/search')(app);
require('./routes/utils')(app);
require('./routes/facebook')(app);
require('./routes/testing')(app);

// Global Routes - this should be last!
require('./routes/global')(app);

app.listen(process.env.PORT || 3000);
console.log("Express server in %s mode", app.settings.env);

// helper functions
function initializeDefaultQueues(callback){
    // create default service bus message queues
    var qs1 = azure.createQueueService(config.storageAccount, config.storageAccessKey, config.queueHost);
    qs1.createQueueIfNotExists('serialnumbers', function(error){
        callback();
    });
}

There are middleware modules such as siteTotals and authenticate that manage Facebook Users who install the app and ensure that their FB info is available to the application at all times.

I don't know maybe I need to install the latest version of IISNode? How do I determine the version of IISNode and whether it needs to be updated?

Also I should point out that this error occurs regardless of which Windows Azure Websites Mode your in either: FREE, SHARE or RESERVED.

And here is my web.config file:

<iisnode      
 node_env="%node_env%"
 nodeProcessCommandLine="&quot;%programfiles%\nodejs\node.exe&quot;"
 nodeProcessCountPerApplication="1"
 maxConcurrentRequestsPerProcess="1024"
 maxNamedPipeConnectionRetry="3"
 namedPipeConnectionRetryDelay="2000"      
 maxNamedPipeConnectionPoolSize="512"
 maxNamedPipePooledConnectionAge="30000"
 asyncCompletionThreadCount="0"
 initialRequestBufferSize="4096"
 maxRequestBufferSize="65536"
 watchedFiles="*.js;node_modules\*;routes\*.js;views\*.jade;middleware\*.js;iisnode.yml"
 uncFileChangesPollingInterval="5000"      
 gracefulShutdownTimeout="60000"
 loggingEnabled="true"
 debuggingEnabled="false"
 debuggerPortRange="5058-6058"
 debuggerPathSegment="debug"
 maxLogFileSizeInKB="128"
 devErrorsEnabled="true"
 flushResponse="false"      
 enableXFF="false"
 promoteServerVars=""
 />

it's the standard file that gets created when using Webmatrix templates which I used to start this whole project.

Answer 1

We had a similar issue with our Node site that used a mongolab backend as it's database. We ended up adding a keep alive flag to our connection string(and setting it to on) to resolve the issue. I think the issue was due to azure's load balancing but I never spent the time to figure it out exactly as we moved to Amazon. Hope that helps you out.

Answer 2

Can you provide some more details about your application? It's difiicult to pinpoint the exact problem without more info, especially about resources your application is connecting to.

The issue is likely caused by an application refresh after a period of inactivity. Because websites are shared, each site is periodically refreshed if not used to improve application density. There is likely some resource that you need to reestablish the connection to after an application restart.

You can get more details on the error (basically any data logged to the stdout or stderr streams) by editing your iisnode.yml file and setting loggingEnabled=true, then dowloading the log (azure site log download using the cross-platform tool).

-Mark

Answer 3

If you have iisnode.yml, use it to turn on logging and debugging. iisnode.yml overrides any settings in web.config, and the default one we drop would have deverrors and logging both disabled, as those are the normal settings for production apps. Editing iisnode.yml to change these values should enable logs for you.