Node.js 网页抓取问题 | 请求 | 啦啦队

Question

I am writing a fairly simple web scraper using Node.js and the request module and the cheerio module. My code doesn't work as I want it to for two reasons:

1. When trying to scrape the image url I am only being returned with a single url multiple times for each page.
2. The iteration of each 'href' and 'title' happens in a seemingly random order (it is the same order each time but is still not in order eg 1, 2, 3 etc.)

Here is my code:

var request = require('request'),
    cheerio = require('cheerio');

var sqlite3 = require('sqlite3').verbose();
var database = "storage.db"
console.log('[+] Creating database: ' + database);
var db = new sqlite3.Database(database);

var pw_url = "https://primewire.unblocked.ink"

console.log('[+] Creating table with rows...');
db.serialize(function() {
  db.run("CREATE TABLE IF NOT EXISTS main (title TEXT, film_page_links TEXT, img_url TEXT)");
});

var img_urls = {}

function iter_pages(page_number) {
  request(pw_url + '/index.php?sort=featured&page=' + page_number, function(err, resp, body) {
    if(!err && resp.statusCode == 200) {
      console.log('[+] The request response status code is: ' + resp.statusCode);
      var $ = cheerio.load(body);
      console.log('[+] Inserting values into database.');
      $('.index_item a img', '.index_container').each(function() {
        img_urls.img_url = $(this).attr('src');
      });
      $('.index_item a', '.index_container').each(function() {
        var url = $(this).attr('href');
        var title = $(this).attr('title');
        if(url.startsWith('/watch-')) {
          //urls.push('https://primewire.unblocked.ink' + url);
          db.run("INSERT INTO main (title, film_page_links, img_url) VALUES (?, ?, ?)",
                  title.replace("Watch ", ""),
                  pw_url + url,
                  "https:" + img_urls.img_url);
        };
      });
      console.log('[+] Processed page:' + page_number);
    }
  });
}

for (var i = 1; i < 5; i++) {
    iter_pages(i);
}

Here is my console.log:

[+] Creating database: storage.db
[+] Creating table with rows...
[+] The request response status code is: 200
[+] Inserting values into database.
[+] Processed page:4
[+] The request response status code is: 200
[+] Inserting values into database.
[+] Processed page:1
[+] The request response status code is: 200
[+] Inserting values into database.
[+] Processed page:3
[+] The request response status code is: 200
[+] Inserting values into database.
[+] Processed page:2

As you can see it goes in order 4, 1, 3, 2 which confuses me.

The image url it returns is always the 21st item of each page.

I am new to JavaScript so please be kind, I have tried moving the method the fetch the image url around within the iter_pages function which either breaks the code or returns the same thing.

Even a link to a more advanced tutorial would suffice, I learn things very quick but the problem is all tutorials I have found are only very basic techniques.

Answer 1

First problem:

This is how you set the image url: img_urls.img_url = ... .

What's happening is that everytime you set, you put it in the same property and overwrite what was there, so that's why it's always the last one from the page. You could try to fix it by pushing into an array, but since you have two loops, it makes things much more complicated, instead try to do both in the same loop:

 $('.index_item a', '.index_container').each(function() {
    var url = $(this).attr('href');
    var title = $(this).attr('title');
    var img_url = $('img', this).attr('src');
    if(url.startsWith('/watch-')) {
      //urls.push('https://primewire.unblocked.ink' + url);
      db.run("INSERT INTO main (title, film_page_links, img_url) VALUES (?, ?, ?)",
              title.replace("Watch ", ""),
              pw_url + url,
              "https:" + img_url);
    };
  });

Second problem:

What you have to realize couple of things. request(...) is making an asynchronous network request. It means this function immediately finishes, which the result is not have arrived yet. So the loop keeps going and all the network request start at the same time, but then thanks to a lot of different variables and luck some of those network requests finish at different times. Some might be faster, some slower. Since they were all nearly started at the same time, the order that they were started wouldn't matter much. Here's your problem simplified:

const request = require('request');

for (let i = 0; i < 5; i++) { 
  makeRequest(i);
}

function makeRequest(i) {
  console.log('Starting', i);
  console.time(i);
  request('http://google.com', () => console.timeEnd(i));
}

And here's the logs:

$ node a.js
Starting 0
Starting 1
Starting 2
Starting 3
Starting 4
1: 8176.111ms
2: 8176.445ms
3: 8206.300ms
0: 8597.458ms
4: 9112.237ms

Running it again yields this:

$ node a.js
Starting 0
Starting 1
Starting 2
Starting 3
Starting 4
3: 8255.378ms
1: 8260.633ms
2: 8259.134ms
0: 8268.859ms
4: 9230.929ms

So you can see the order is not deterministic. Just some finishes faster than the others.

If you really want them to happen in order, I suggest using a control flow library. async.js is one of the most popular ones.