Cheerio WebScraping Node JS

Question

I want to scrape a webpage and get all the links out with (if present) a thumbnail or image within the 'a' tag.

I'm able to get the links, but not sure how to grab the img > src value within the current a tag im iterating over.

const cheerio = require('cheerio')
const request = require('request')
const throttledRequest = require('throttled-request')(request)
throttledRequest.configure({ requests: 18, milliseconds: 1000 })

let o = {
  linksOut: []
}

const scrapeLinksOut = (o, body) => {
  if (body) {
    let $ = cheerio.load(body)

    $('a').map(function () {
      let link = $(this).attr('href')
      // I want to get the img url within the a tag for the current iteration
      let thumbnail = $(this).//img > src

      o.linksOut.push( {
        link: link,
        thumbnail: thumbnail
      })
    })
  } else {
   // something else
  }
}

const scrape = (() => {
  return new Promise((resolve, reject) => {
    throttledRequest({
      url: 'https://www.ibm.com/us-en',
      followAllRedirects: true,
      timeout: 30000
    }, (err, res, body) => {
      scrapeLinksOut(o, body)
      return resolve(o)
    })
  }) 
})

scrape()
  .then((res) => {
    res.linksOut.forEach((obj) => {
     console.log(obj);
   })
  })
  .catch((err) => console.log(err))

Answer 1

您可以使用find()获取img元素，然后使用attr()获取其 src。

$(this).find('img').attr('src')

Answer 2

I wrote up a quick solution that hopefully solves your problem. I tested it out on my machine and it works for me.

$('a').map(function () {

...
    let thumbnail = $(this).html();
    thumbnail = cheerio.parseHTML(thumbnail);

    if (thumbnail!== null && thumbnail.length > 0 && //Ensure link has an image
    thumbnail[0].attribs !== undefined && 
    thumbnail[0].attribs.src !== undefined) {
        thumbnail = thumbnail[0].attribs.src; //The image URL
    }

...

});

const scrapeLinksOut = (o, body) => {
  if (body) {
    let $ = cheerio.load(body)

    $('a').map(function () {
      let link = $(this).attr('href')
     
      let thumbnail = $(this).html();
      thumbnail = cheerio.parseHTML(thumbnail)


      if (thumbnail !== null && thumbnail.length > 0 && thumbnail[0].attribs !== undefined && thumbnail[0].attribs.src !== undefined) {
        thumbnail = thumbnail[0].attribs.src
      } else {
        thumbnail = null
      }

      o.linksOut.push( {
        link: link,
        thumbnail: thumbnail
      })
    })
  } else {
    o.linksOut.push(links)
  }
}

this logs:

null
null
null
[object Object]
[object Object]
[object Object]
[object Object]
[object Object]
[object Object]
[object Object]
[object Object]
null
null
null
null
null
null
null

Answer 3

This seemed to work out. Links and images match up.

const scrapeLinksOut = (o, body) => {
  if (body) {
    let $ = cheerio.load(body);

    $('a').map(function () {
      let thumbnail;
      let link = $(this).attr('href');

      $(this)
        .find('img')
        .map(function (i, img) {
          thumbnail = $(img).attr('src');
        });

      if (thumbnail !== undefined) {
        o.linksOut.push({
          link: link,
          thumbnail: thumbnail,
        });
      }
    });
  } else {
    // something else
  }
};