HTTP request from within Express/Node.js

Question

I'm trying to set up an express service for a program that I'm writing that contacts an external API and then returns the results so it can be stored in a Mongo I have set up.

This seems like it should be fairly straightforward, but I'm new to Node.js/Express and I'm getting a "Can't set headers after they are sent" error.

I'm getting the data that I want from the external API, but how to send that data properly back to my Angular app.js so it can update in my table?

"addSelected()" is the function I'm calling in my app.js to kick off the process. The "data" prints part of the way through the full response but then cuts off and gives me the "Can't set Headers after they are sent" error. From what I understand this is from sending the response and then trying to modify the response header after the fact.. but I'm unsure of a workaround or if I'm just formatting everything wrong as this is my first swing at MEAN stack in general.

I know the problem is on the line "res.send(data)" in server.js but I don't know how to correctly format the response.

My code:

server.js

//server.js


//setup ==============================
var express = require ('express');
var request = require('request');
var app = express();
var mongoose = require('mongoose');
var https = require('https');

//config ============================
app.use(express.static(__dirname + '/public/'));

console.log("running PipeHelper");

mongoose.connect('mongoedit');
var Schema = mongoose.Schema;
var opSchema = new Schema({

        title: String,
        description: String,
        company: String,
        post_date: String,
        close_date: String,
        contact: String,
        location: String,
        url: String,
        notice_type: String
    });

var item = mongoose.model('item', opSchema);



//routes===========================

//returns full database
app.get('/api/db', function(req, res){

    item.find({},function(err, items){

        if (err) res.send(err);
        res.json(items);
    });

});


//searches FBO for opportunities to add to database
app.get('/api/search:FBO_key', function(req, res){


    var data;
    console.log("2");
    var baseURL = "api.data.gov"
    var params = "/gsa/fbopen/v0/opps?q=" + req.params.FBO_key;
    params += "&api_key="+"keyyyy";
    params += "&all";
    params += "&start=0";
    params += "&p=1";
    params += "&limit=10";
    url = baseURL+params;

    var options = {
        port: 443,
        host: 'api.data.gov',
        path: params,
        method: 'GET'


    };

    //get FBO data
    var request = https.request(options, function(response){
        console.log("4");
        response.on('data', function (chunk){

            //response data to send back to app.js
            data += chunk.toString();
            res.send(data);


        });


    });

    console.log("3");
    request.end();


    request.on('error', function(e){
        console.error(e);
    });


});

app.get('/', function(req,res){

    res.sendfile('./public/index.html');

});

app.listen(8000);

app.js

var app = angular.module("pipeHelper", ['smart-table']);

app.controller('mainCtrl', [
'$scope', '$http', function($scope, $http){

$scope.selected = [];
$scope.displayData= [];
$scope.items=[];
$scope.FBOsearch;



//populates table on startup with whole DB
$http.get('./api/db')
    .success(function(data){
        $scope.items=data;
        $scope.displayData = [].concat($scope.items);

    })
    .error(function(data){
        console.log('Error: '+data);
    });



$scope.addSelected = function(){

    //search FBO, add opportunities, update table
    console.log("1");


    $http.get('./api/search'+'NGA')
    .success(function(data){
        console.log("5");
        console.log(data);
        $scope.items=data;
        $scope.displayData= [].concat($scope.items);

    })
    .error(function(data){

        console.log('Error: ' +data);

    });

};




$scope.isSelected = function(item){
    //if its selected, remove it
    // if its unselected, add it
    if ($scope.selected.indexOf(item)==-1){
        $scope.selected.push(item);
    }
    else{

        $scope.selected.splice($scope.selected.indexOf(item), 1);
    }



    console.log($scope.selected);
    //temp placeholder function.  Eventually add to array of selected objects for placement in Pipeliner/deletion

};







}]);

Answer 1

solved the issue. I was unaware that response.on('data') gets called multiple times, thus calling my res.send(data) multiple times and incompletely causing it to crash with the error. I added the following to the request function:

response.on('end'function(){
    res.send(data);
};

basically when the external API data is finished coming in, send it with express. Learn by doing I suppose. Hope this helps someone eventually.

Answer 2

I can't leave a comment, so I will just make it an answer.

I would recommend installing node-inspector , npm install -g node-debug . Then run your app with node-debug server.js . This will spawn a new instance of Firefox or Chrome dev tools and allows you to debug your nodeJS code. Very useful.

The error you are seeing is most likely related to request.end() , if I were to guess. After .end() is called, you can no longer modify the header content. I doubt it would make a difference, but try putting the request.end() after you have the request.on('error') call.

EDIT: 10/15/15

I would highly recommend installing VS Code . It has a built-in debugger for node apps.