node.js Web搜寻器图像/视频

Question

Recently I have been getting into web crawlers and I have figured out ow to search for text but is there any way that I can have my web crawler search for something else such as videos and images and then download them and organize them.

here is my web-crawler so far:

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');

var START_URL = "https://www.google.com";
var SEARCH_WORD = "apples";
var MAX_PAGES_TO_VISIT = 200;

var pagesVisited = {};
var numPagesVisited = 0;
var pagesToVisit = [];
var url = new URL(START_URL);
var baseUrl = url.protocol + "//" + url.hostname;

pagesToVisit.push(START_URL);
 crawl();

 function crawl() {
  if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
    console.log("Reached max limit of number of pages to visit.");
     return;
  }
  var nextPage = pagesToVisit.pop();
  if (nextPage in pagesVisited) {
    // We've already visited this page, so repeat the crawl
    crawl();
  } else {
    // New page we haven't visited
    visitPage(nextPage, crawl);
  }
}

function visitPage(url, callback) {
  // Add page to our set
  pagesVisited[url] = true;
 numPagesVisited++;

  // Make the request
 console.log("Visiting page " + url);
  request(url, function(error, response, body) {
     // Check status code (200 is HTTP OK)
     console.log("Status code: " + response.statusCode);
     if(response.statusCode !== 200) {
       callback();
       return;
    }
    // Parse the document body
      var $ = cheerio.load(body);
     var isWordFound = searchForWord($, SEARCH_WORD);
     if(isWordFound) {
      console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
     } else {
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();
     }
  });
 }

function searchForWord($, word) {
  var bodyText = $('html > body').text().toLowerCase();
  return(bodyText.indexOf(word.toLowerCase()) !== -1);
}

 function collectInternalLinks($) {
     var relativeLinks = $("a[href^='/']");
console.log("Found " + relativeLinks.length + " relative links on page");
relativeLinks.each(function() {
    pagesToVisit.push(baseUrl + $(this).attr('href'));
});
}

I have gotten most of this code from an online tutorial to help me get started but I need more help the code works I just wanted to know if and how would it be possible to web crawl images and video.

Newer Code:

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');

var START_URL = "http://moetube.net";
//var SEARCH_WORD = "anime";
 var MAX_PAGES_TO_VISIT = 200;

 var pagesVisited = {};
 var numPagesVisited = 0;
 var pagesToVisit = [];
  var url = new URL(START_URL);
 var baseUrl = url.protocol + "//" + url.hostname;

 pagesToVisit.push(START_URL);
 crawl();

 function crawl() {
   if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
     console.log("Reached max limit of number of pages to visit.");
     return;
   }
   var nextPage = pagesToVisit.pop();
   if (nextPage in pagesVisited) {
     // We've already visited this page, so repeat the crawl
     crawl();
    } else {
       // New page we haven't visited
       visitPage(nextPage, crawl);
     }
   }

   function visitPage(url, callback) {
    // Add page to our set
    pagesVisited[url] = true;
    numPagesVisited++;

     // Make the request
     console.log("Visiting page " + url);
    request(url, function(error, response, body) {
    var $ = cheerio.load(body);
      // Check status code (200 is HTTP OK)
      console.log("Status code: " + response.statusCode);
      collectImages($);
      if(response.statusCode !== 200) {
        callback();

          return;
       }
      // Parse the document body

     // var isWordFound = searchForWord($, SEARCH_WORD);

    // if(isWordFound) {
     //   console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
    // } else {
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();
   //  }
   });
}

   function searchForWord($, word) {
   var bodyText = $('html > body').text().toLowerCase();
  return(bodyText.indexOf(word.toLowerCase()) !== -1);
    }

function collectImages($) {

   return $("img").map(function() {
        return $(this).text();
         console.log((this).text() + "JHJHHHHHHHHHHHHHHHHHHHH");
     }).get();
      }

 function collectInternalLinks($) {

     var relativeLinks = $("a[href^='/']");
     console.log("Found " + relativeLinks.length + " relative links on page");
     relativeLinks.each(function() {
         pagesToVisit.push(baseUrl + $(this).attr('href'));
     });
 }

Answer 1

Just like you use cheerio to search the body for links, you can also search the body for either <img> or <video> tags. You don't say exactly what you want to do when you find those tags, but you could create a function similar to your collectInternalLinks() that would collect media objects for further processing:

// return array of image URLs (these may be page-relative URLS)
function collectImages($) {
    return $("img").map(function() {
        return $(this).prop("src");
    }).get();
}

// return collection of video elements
function collectVideos($) {
    let videoUrls = [];
    $("video").each(function() {
        let src = $(this).prop("src");
        if (src) {
            videoUrls.push(src);
        } else {
            let subElements = $(this).find("track, source");
            subElements.each(function() {
                let src = $(this).prop("src");
                if (src) {
                    videoUrls.push(src);
                }
            });
        }
    });
    return videoUrls;
}

Collecting video URLs is a bit more involved because those URLs can be specified a number of different ways ( .src property, embedded <track> tags, embedded <source> tags, etc...) so you'd have to parse out each possible way for each <video> tag.