Stream Rest API HTTP Response in Node.js & Express

Question

I am creating a Rest API in Node.js and Express. It connects with remote HANA database & execute one query. Now I want to stream HTTP response so that I can send it to browser into chunks, rather than sending it completely since it's a very large data.

I tried something which is giving me no output. I don't know the reason. If I send the complete response to browser using response.send (data), it's working fine. But streaming is now working.

I have added code snippet below.

const express = require("express");
const APP = express();
const HANA_DB = require('hdb');
const BODY_PARSER = require("body-parser");

start();

function start() {
    startServer();
    initializeExpress();
    APP.get("/data", function(request, response) {
        var connection = HANA_DB.createClient({
            host     : "hostname",
            port     : "port",
            user     : "username",
            password : "password"
        });
        connection.on('error', function (error) {
            console.log("Error in database connection...");
        });
        connection.connect(function (error) {
            if (error) {
                console.log("Error in database connection...");
                return;
            }
            var query = "SELECT * FROM TableName";
            connection.exec(query, function(error, result) {
                if(error) {
                    response.send("Getting error while fetching result...");
                    return;
                }
                //response.send(data);
                var datalength = 0;
                request.on('data', function(chunk) {
                    datalength += chunk.length;
                    console.log("DATA EVENT: " + datalength);
                    response.send(datalength);
                })
                .on('end', function() {
                    console.log("END EVENT: " + datalength);
                    response.send(datalength);
                });
            });
        });     
    });
};

function initializeExpress() {
    APP.all('/*', function(request, response, next) {
        response.header("Access-Control-Allow-Origin", "*");
        response.header("Access-Control-Allow-Headers", "X-Requested-With");
        response.header('Access-Control-Allow-Methods', 'PUT, GET, POST, DELETE, OPTIONS');
        response.header('Access-Control-Allow-Headers', 'Content-Type');
        next();
    });
    APP.use(BODY_PARSER.json());
    APP.use(BODY_PARSER.urlencoded({ extended: true }));
};

function startServer(config) {
    var server = APP.listen("8081", function(error) {
        if(error) {
            console.log("Unable to connect to 127.0.0.1:8081");
            return;
        }
        console.log("Server is listening at - 127.0.0.1:8081");
    });
};

Answer 1

The problem is here request.on('data', . request refers to the browser request.

You cannot use streaming with .exec() , because the callback function of exec is called with the rows as parameters.

To use streaming , use the .execute() method, which passes a resultset to the callback function.

I never used hdb , so I cannot give the code to use.

Answer 2

I think the logic you are using is wrong for the streaming the data.

Use res.write instead of res.send and you also have to read streaming data from your database instead of one time connection.exec

I am giving you an example code where you will get some idea about streaming data in Expressjs.

var http = require( 'http' );

http.createServer( function ( req, res ) {
res.writeHead( 200, {
    'Content-Type': 'text/plain; charset=utf-8',
    'Transfer-Encoding': 'chunked',
    'X-Content-Type-Options': 'nosniff'
} );
res.write( 'Beginning\n' );
var count = 10;
var io = setInterval( function () {
    res.write( 'Doing ' + count.toString() + '\n' );
    count--;
    if ( count === 0 ) {
        res.end( 'Finished\n' );
        clearInterval( io );
    }
}, 1000 );
} ).listen( 8000 );