Web用BS4刮

Question

我在从 imdb.com 中抓取有关电影的一些基本信息时遇到问题。 我希望我的程序从给定的 URL 中获取电影的标题和描述。 标题部分正在完成它的工作，但是我不知道如何获取描述。 这是我的代码：

import requests

from bs4 import BeautifulSoup as bs

def get_data(url):
    r = requests.get(url, headers={'Accept-Language': 'en-US,en;q=0.5'})
    if not r or 'https://www.imdb.com/title' not in url:
        return print('Invalid movie page!')
    return r.content

if __name__ == '__main__':
    # print('Input the URL:')
    # link = input()
    link = 'https://www.imdb.com/title/tt0111161'
    data = get_data(link)
    soup = bs(data, 'html.parser')
    title = ' '.join(soup.find('h1').text.split()[:-1])
    desc = soup.find('p', {'data-testid':"plot", 'class':"GenresAndPlot__Plot-cum89p-8 kmrpno"}).text
    movie_info = {'title': title, 'description': desc}
    print(movie_info)

当我运行它时，我得到一个错误：

Exception has occurred: AttributeError
'NoneType' object has no attribute 'text'
  File "movie-scraper.py", line 18, in <module>
    desc = soup.find('p', {'data-testid':"plot", 'class':"GenresAndPlot__Plot-cum89p-8 kmrpno"}).text

如何正确访问描述？

Answer 1

要获取 plot 摘要，请将选择器更改为 find class="plot_summary" ：

import requests
from bs4 import BeautifulSoup as bs


def get_data(url):
    r = requests.get(url, headers={"Accept-Language": "en-US,en;q=0.5"})
    if not r or "https://www.imdb.com/title" not in url:
        return print("Invalid movie page!")
    return r.content


if __name__ == "__main__":
    link = "https://www.imdb.com/title/tt0111161"
    data = get_data(link)
    soup = bs(data, "html.parser")
    title = " ".join(soup.find("h1").text.split()[:-1])
    desc = soup.find("div", class_="plot_summary").get_text(strip=True)  # <-- change this to find class="plot_summary"
    movie_info = {"title": title, "description": desc}
    print(movie_info)

印刷：

{'title': 'The Shawshank Redemption', 'description': 'Two imprisoned men bond over a number of years, finding solace and eventual redemption through acts of common decency.Director:Frank DarabontWriters:Stephen King(short story "Rita Hayworth and Shawshank Redemption"),Frank Darabont(screenplay)Stars:Tim Robbins,Morgan Freeman,Bob Gunton|See full cast & crew»'}