node.js Web搜尋器圖像/視頻

Question

最近，我進入了Web爬蟲，已經找到了可以搜索文本的方法，但是有什么辦法可以讓我的Web爬蟲搜索其他內容，例如視頻和圖像，然后下載並組織它們。

到目前為止，這是我的網絡抓取工具：

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');

var START_URL = "https://www.google.com";
var SEARCH_WORD = "apples";
var MAX_PAGES_TO_VISIT = 200;

var pagesVisited = {};
var numPagesVisited = 0;
var pagesToVisit = [];
var url = new URL(START_URL);
var baseUrl = url.protocol + "//" + url.hostname;

pagesToVisit.push(START_URL);
 crawl();

 function crawl() {
  if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
    console.log("Reached max limit of number of pages to visit.");
     return;
  }
  var nextPage = pagesToVisit.pop();
  if (nextPage in pagesVisited) {
    // We've already visited this page, so repeat the crawl
    crawl();
  } else {
    // New page we haven't visited
    visitPage(nextPage, crawl);
  }
}

function visitPage(url, callback) {
  // Add page to our set
  pagesVisited[url] = true;
 numPagesVisited++;

  // Make the request
 console.log("Visiting page " + url);
  request(url, function(error, response, body) {
     // Check status code (200 is HTTP OK)
     console.log("Status code: " + response.statusCode);
     if(response.statusCode !== 200) {
       callback();
       return;
    }
    // Parse the document body
      var $ = cheerio.load(body);
     var isWordFound = searchForWord($, SEARCH_WORD);
     if(isWordFound) {
      console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
     } else {
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();
     }
  });
 }

function searchForWord($, word) {
  var bodyText = $('html > body').text().toLowerCase();
  return(bodyText.indexOf(word.toLowerCase()) !== -1);
}

 function collectInternalLinks($) {
     var relativeLinks = $("a[href^='/']");
console.log("Found " + relativeLinks.length + " relative links on page");
relativeLinks.each(function() {
    pagesToVisit.push(baseUrl + $(this).attr('href'));
});
}

我已經從在線教程中獲得了大部分代碼，以幫助我入門，但是我需要更多幫助，這些代碼可以正常工作，我只是想知道是否以及如何進行網絡抓取圖像和視頻。

較新的代碼：

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');

var START_URL = "http://moetube.net";
//var SEARCH_WORD = "anime";
 var MAX_PAGES_TO_VISIT = 200;

 var pagesVisited = {};
 var numPagesVisited = 0;
 var pagesToVisit = [];
  var url = new URL(START_URL);
 var baseUrl = url.protocol + "//" + url.hostname;

 pagesToVisit.push(START_URL);
 crawl();

 function crawl() {
   if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
     console.log("Reached max limit of number of pages to visit.");
     return;
   }
   var nextPage = pagesToVisit.pop();
   if (nextPage in pagesVisited) {
     // We've already visited this page, so repeat the crawl
     crawl();
    } else {
       // New page we haven't visited
       visitPage(nextPage, crawl);
     }
   }

   function visitPage(url, callback) {
    // Add page to our set
    pagesVisited[url] = true;
    numPagesVisited++;

     // Make the request
     console.log("Visiting page " + url);
    request(url, function(error, response, body) {
    var $ = cheerio.load(body);
      // Check status code (200 is HTTP OK)
      console.log("Status code: " + response.statusCode);
      collectImages($);
      if(response.statusCode !== 200) {
        callback();

          return;
       }
      // Parse the document body

     // var isWordFound = searchForWord($, SEARCH_WORD);

    // if(isWordFound) {
     //   console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
    // } else {
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();
   //  }
   });
}

   function searchForWord($, word) {
   var bodyText = $('html > body').text().toLowerCase();
  return(bodyText.indexOf(word.toLowerCase()) !== -1);
    }

function collectImages($) {

   return $("img").map(function() {
        return $(this).text();
         console.log((this).text() + "JHJHHHHHHHHHHHHHHHHHHHH");
     }).get();
      }

 function collectInternalLinks($) {

     var relativeLinks = $("a[href^='/']");
     console.log("Found " + relativeLinks.length + " relative links on page");
     relativeLinks.each(function() {
         pagesToVisit.push(baseUrl + $(this).attr('href'));
     });
 }

Answer 1

就像您使用cheerio在正文中搜索鏈接一樣，您也可以在正文中搜索<img>或<video>標簽。 找到這些標簽時，您沒有確切說出要做什么，但是您可以創建一個類似於collectInternalLinks()的函數，該函數將收集媒體對象以進行進一步處理：

// return array of image URLs (these may be page-relative URLS)
function collectImages($) {
    return $("img").map(function() {
        return $(this).prop("src");
    }).get();
}

// return collection of video elements
function collectVideos($) {
    let videoUrls = [];
    $("video").each(function() {
        let src = $(this).prop("src");
        if (src) {
            videoUrls.push(src);
        } else {
            let subElements = $(this).find("track, source");
            subElements.each(function() {
                let src = $(this).prop("src");
                if (src) {
                    videoUrls.push(src);
                }
            });
        }
    });
    return videoUrls;
}

收集視頻URL涉及更多，因為可以用許多不同的方式指定這些URL（ .src屬性，嵌入式<track>標簽，嵌入式<source>標簽等），因此您必須解析每種URL每個<video>標簽的可能方法。