如何使用Phantomjs设置页面抓取之间的时间间隔

Question

Currently I wrote a script with Phantomjs that scrapes through multiple pages. My script works but I can't figure out how to set a time interval in between scrapes. I tried using setInterval and passing the items from the arrayList about every 5 seconds but it doesn't seem to work. My script keeps breaking. Here's my example phantomjs script code:

Without setInterval

var arrayList = ['string1', 'string2', 'string3'....]

arrayList.forEach(function(eachItem) {
    var webAddress = "http://www.example.com/eachItem"    
    phantom.create(function(ph) {
    return ph.createPage(function(page) {

        return page.open(yelpAddress, function(status) {
            console.log("opened site? ", status);


            page.injectJs('http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js', function() {

                setTimeout(function() {
                    return page.evaluate(function() {

                        //code here for gathering data


                    }, function(result) {
                        return result
                        ph.exit();
                    });

                }, 5000);

            });
        });
    });
});

With setInterval :

var arrayList = ['string1', 'string2', 'string3'....]
var i = 0
var scrapeInterval = setInterval(function() {
    var webAddress = "http://www.example.com/arrayList[i]"    
    phantom.create(function(ph) {
    return ph.createPage(function(page) {

        return page.open(yelpAddress, function(status) {
            console.log("opened site? ", status);


              page.injectJs('http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js', function() {

                setTimeout(function() {
                    return page.evaluate(function() {

                           //code here for gathering data


                    }, function(result) {
                           return result
                           ph.exit();
                    });

                }, 5000);

            });
        });
    });
    i++
    if(i > arrayList.length) {
    clearInterval(scrapeInterval);        
}, 5000);

Basically, I would like to send in a chunk of itemss (10-20 of them) within the arrayList and wait for 1 - 2 mins and send in next chunk of items without overwhelming the website. OR if there a way to set a time interval to loop through each item within the array every 2-3 secs.

Answer 1

The problem is that PhantomJS is asynchronous, but loop iteration is not. All iterations (in the first snippet) are executed even before the first page is loaded. You're essentially generating multiple such processes which run at the same time.

You can use something like async to let it run sequentially:

phantom.create(function(ph) {
    ph.createPage(function(page) {
        var arrayList = ['string1', 'string2', 'string3'....];

        var tasks = arrayList.map(function(eachItem) {
            return function(callback){
                var webAddress = "http://www.example.com/" + eachItem;
                page.open(webAddress, function(status) {
                    console.log("opened site? ", status);

                    page.injectJs('http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js', function() {

                        setTimeout(function() {
                            return page.evaluate(function() {
                                //code here for gathering data
                            }, function(result) {
                                callback(null, result);
                            });
                        }, 5000);
                    });
                });
            };
        });

        async.series(tasks, function(err, results){
            console.log("Finished");
            ph.exit();
        });
    });
});

Of course you can also move phantom.create() inside of each task which will create a separate process for each request, but the code above will be faster.

Answer 2

You have some typos in the second snippet where you added the setInterval approach:

var arrayList = ['string1', 'string2', 'string3'];
var i = 0;
var scrapeInterval = setInterval(function () {
    var webAddress = "http://www.example.com/arrayList[i]"
    phantom.create(function (ph) {
        return ph.createPage(function (page) {

            return page.open(yelpAddress, function (status) {
                console.log("opened site? ", status);


                page.injectJs('http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js', function () {

                    setTimeout(function () {
                        return page.evaluate(function () {
                            //code here for gathering data
                        }, function (result) {
                            return result
                            ph.exit();
                        });

                    }, 5000);

                });
            });
        });

        i++;
        if (i > arrayList.length) {
            clearInterval(scrapeInterval);
        } //This was missing;
    }); //This was missing;
}, 5000);

And something i've noticed, is the return statement in the following timeout:

setTimeout(function () {
    return page.evaluate(function () {
        //code here for gathering data
    }, function (result) {
        return result
        ph.exit();
    });
}, 5000);

ph.exit(); will never be reached, i don't know if this will cause any issue for you but you might want to take a look at it.