无法迭代数组并保存到mongoose。 回调问题？

Question

I'm learning node, express, mongo, and, in the process, javascript. I'm trying to get a feature where using rssparser, get a list of stories and save them to a mongo database with mongoose.

I've got the RSS pull working, and I'm iterating through the stories, it's the saving that I'm having problems with. I want to 1) check that the story doesn't already exist in the database, and 2) if not, save it. I think I'm getting lost in the way callbacks are handled. Here's my current code, with comments.

rssparser.parseURL(url, options, function(err,out){
    // out.items is an array of the items pulled
    var items = out.items;
    var story;
    for (var i=0; i<items.length; i++){

        //create a mongoose story
        story = new schemas.Stories({
            title: items[i].title,
            url: items[i].url,
            summary: items[i].summary,
            published: items[i].published_at
        });

        //TODO: for testing - these show up correctly.  
        //If I pull 10 stories, I get 10 entries from here that match
        //So "story" is holding the current story
        console.log("items[i] is :" + items[i].title);
        console.log("story title is : " + story.title);

        // setup query to see if it's already in db
        var query = schemas.Stories.findOne({
            "title" : story.title,
            "url" : story.url
        });


        //execute the query
        query.exec( function(err, row){
            if(err) console.log("error-query: " + err);
            console.log("row: "+ row);
            if(!row) {
                // not there, so save
                console.log('about to save story.title: ' + story.title);
                story.save(function (err){
                    console.log("error in save: " + err);
                });
            }
        });

    }
});

When this runs, what I see is alot of console output:

It's starts showing all the stories (many omitted):

items[i] is :TSA Drops Plan to Let Passengers Carry Small Knives on Planes               
story title is : TSA Drops Plan to Let Passengers Carry Small Knives on Planes           
items[i] is :BUILDING COLLAPSE:1 Reportedly Dead, 13 Pulled From Philly Rubble           
story title is : BUILDING COLLAPSE:1 Reportedly Dead, 13 Pulled From Philly Rubble       
items[i] is :CONTROVERSIAL PAST: Obama's UN Nominee Once Likened US 'Sins' to Nazis'     
story title is : CONTROVERSIAL PAST: Obama's UN Nominee Once Likened US 'Sins' to Nazis' 
items[i] is :WRITING OUT WRIGHTS: Bill Gives First Powered Flight Nod to Whitehead       
story title is : WRITING OUT WRIGHTS: Bill Gives First Powered Flight Nod to Whitehead   
items[i] is :BREAKING NEWS: Rice Named to Top Security Post Despite Libya Fallout        
story title is : BREAKING NEWS: Rice Named to Top Security Post Despite Libya Fallout   

Then continues like (many omitted):

row: null                                                                     
about to save story.title: Best Ribs in America                               
row: null                                                                     
about to save story.title: Best Ribs in America                               
row: null                                                                     
about to save story.title: Best Ribs in America                               
row: null                                                                     
about to save story.title: Best Ribs in America                               
row: null                                                                     
about to save story.title: Best Ribs in America                               
row: null                                                                     
about to save story.title: Best Ribs in America                               
row: { title: 'Best Ribs in America',                                         
  url: 'http://www.foxnews.com/leisure/2013/06/05/10-best-ribs-in-america/',  
  published: 1370463800000,                                                   
  _id: 51af9f881995d40425000023,                                              
  __v: 0 }                                                                    

It repeats the "about to save" title (which is the last story in the feed), and it saves the story once, like the last row shows.

The console.log output shows just I put it, all the story title output at the top, then all the stuff from inside the query.exec() call at the bottom.

Any help is appreciated...

Answer 1

The problem with this is that the story referenced in the exec callback will have been set to whatever is the last thing iterated on in the for loop, once the callback will get executed, since all of the executed functions are referencing the same instance of the variable.

The easiest way to fix this is to simply wrap each thing in the for loop in a function that you execute right away with parameters, as in:

rssparser.parseURL(url, options, function(err,out){
    // out.items is an array of the items pulled
    var items = out.items;
    for (var i=0; i<items.length; i++){
        (function(item) {

            //create a mongoose story
            var story = new schemas.Stories({
                title: item.title,
                url: item.url,
                summary: item.summary,
                published: item.published_at
            });

            // setup query to see if it's already in db
            var query = schemas.Stories.findOne({
                "title" : story.title,
                "url" : story.url
            });

            //execute the query
            query.exec( function(err, row){
                if(err) console.log("error-query: " + err);
                console.log("row: "+ row);
                if(!row) {
                    // not there, so save
                    console.log('about to save story.title: ' + story.title);
                    story.save(function (err){
                        console.log("error in save: " + err);
                    });
                }
            });

        })(items[i]);
    }
});

I haven't tested this, but I'm sure you'll find that it will fix your issue

Another even easier, cleaner, better way would be to iterate over the items in a forEach loop on the array, if your platform supports that (which node.js does) - this version is even more prettier:

rssparser.parseURL(url, options, function(err,out){
    // out.items is an array of the items pulled
    out.items.forEach(function(item) {

        //create a mongoose story
        var story = new schemas.Stories({
            title: item.title,
            url: item.url,
            summary: item.summary,
            published: item.published_at
        });

        // setup query to see if it's already in db
        var query = schemas.Stories.findOne({
            "title" : story.title,
            "url" : story.url
        });

        //execute the query
        query.exec( function(err, row){
            if(err) console.log("error-query: " + err);
            console.log("row: "+ row);
            if(!row) {
                // not there, so save
                console.log('about to save story.title: ' + story.title);
                story.save(function (err){
                    console.log("error in save: " + err);
                });
            }
        });

    });
});

Answer 2

well, node is event driven server and javascript is also event driven, so you can call stuff asynchronously.

you need to use some async patterns to do what you want.

first, if you are using mongoose you can utilize it's schema class to check for items that are already exists without querying the db again:

var mongoose = require('mongoose');

var schema = new mongoose.Schema({
    title: String,
    url: { type: String, unique: true },
    summary: String,
    published: Date

})

var model = mongoose.model('stories', schema)

the url is unique, so the save will cause a duplicate error and mongoose won't save the query.

now to iterate through the items and save each one we need some kind of a pattern for it, luckily we have async for it:

var async = require('async');

rssparser.parseURL(url, options, function(err, out){
    async.each(out.items, function(item, callback){

        var m = new model({
            title: item.title,
            url: item.url,
            summary: item.summary,
            published: item.published_at
        })

        m.save(function(err, result){
            callback(null)
        });

    }, function(err){
        //we complete the saving we can do stuff here       
    });
}

we used async in a parallel mode as we don't care if some are duplicate or not. you can also track it with an array that you can push to it the err || result so you can see how many items you saved.