NodeJS Crawler登录站点

Question

i want to crawl geocaching.com but some data like coords are only for loged in users. Im using "crawler" from npm and have now idea how to log in with crawler but i already got the names of the login form:

• ctl00$ContentBody$tbUsername: user
• ctl00$ContentBody$tbPassword: passwaord
• ctl00$ContentBody$btnSignIn: "Sign+In"

Here is My code so far:

var Crawler = require("crawler");
var url = require('url');
var mongoose = require("mongoose");
var Cache = require("./models/cache.js");

mongoose.connect("localhost:27017/Cache");

var removeTags = function(text){
    return String(text).replace(/(<([^>]+)>)/ig,'');
};
var c = new Crawler({
    maxConnections: 10,
    skipDuplicates: true,

    callback: function (error, result, $) {

        if (result.request.uri.href.startsWith("http://www.geocaching.com/geocache/")) {
            var cache = new Cache();
            var id = removeTags($(".CoordInfoCode"));
            Cache.count({
                "_id": id
            }, function (err, count) {
                if (err)
                    return;
                else if (count < 1) {
                    //Saving the data
                }

            });


        }
        if (result.headers['content-type'] == "text/html; charset=utf-8") {
            if ($('a').length != 0) {
                $('a').each(function (index, a) {
                    var toQueueUrl = $(a).attr('href');
                    process.nextTick(function () {
                        process.nextTick(function () {
                            c.queue(toQueueUrl);
                        })
                    });

                });
            }
        }

    }
});

c.queue('http://www.geocaching.com/seek/nearest.aspx?ul=Die_3sten_3');

Answer 1

I made an example javascript crawler on github.

It's event driven and use an in-memory queue to store all the resources(ie. urls).

How to use in your node environment

var Crawler = require('../lib/crawler')
var crawler = new Crawler('http://www.someUrl.com');

// crawler.maxDepth = 4;
// crawler.crawlInterval = 10;
// crawler.maxListenerCurrency = 10;
// crawler.redisQueue = true;
crawler.start();

Here I'm just showing you 2 core method of a javascript crawler.

Crawler.prototype.run = function() {
  var crawler = this;
  process.nextTick(() => {
    //the run loop
    crawler.crawlerIntervalId = setInterval(() => {

      crawler.crawl();

    }, crawler.crawlInterval);
    //kick off first one
    crawler.crawl();
  });

  crawler.running = true;
  crawler.emit('start');
}


Crawler.prototype.crawl = function() {
  var crawler = this;

  if (crawler._openRequests >= crawler.maxListenerCurrency) return;


  //go get the item
  crawler.queue.oldestUnfetchedItem((err, queueItem, index) => {
    if (queueItem) {
      //got the item start the fetch
      crawler.fetchQueueItem(queueItem, index);
    } else if (crawler._openRequests === 0) {
      crawler.queue.complete((err, completeCount) => {
        if (err)
          throw err;
        crawler.queue.getLength((err, length) => {
          if (err)
            throw err;
          if (length === completeCount) {
            //no open Request, no unfetcheditem stop the crawler
            crawler.emit("complete", completeCount);
            clearInterval(crawler.crawlerIntervalId);
            crawler.running = false;
          }
        });
      });
    }

  });
};

Here is the github link https://github.com/bfwg/node-tinycrawler . It is a javascript web crawler written under 1000 lines of code. This should put you on the right track.