python 抓取 - 我如何在 OddsPortal 中的日期之间循环

Question

我正在尝试抓取 Oddsportal 和下面的代码，但是当我运行它时，我得到了IndexError

代码：

import threading
from datetime import datetime
from math import nan
from multiprocessing.pool import ThreadPool
import pandas as pd
from bs4 import BeautifulSoup as bs
from selenium import webdriver



class Driver:
    def __init__(self):
        options = webdriver.ChromeOptions()
        options.add_argument("--headless")
        # Un-comment next line to supress logging:
        options.add_experimental_option('excludeSwitches', ['enable-logging'])
        self.driver = webdriver.Chrome(options=options)

    def __del__(self):
        self.driver.quit()  # clean up driver when we are cleaned up
        # print('The driver has been "quitted".')


threadLocal = threading.local()


def create_driver():
    the_driver = getattr(threadLocal, 'the_driver', None)
    if the_driver is None:
        the_driver = Driver()
        setattr(threadLocal, 'the_driver', the_driver)
    return the_driver.driver


class GameData:
    def __init__(self):
        self.date = []
        self.time = []
        self.game = []
        self.score = []
        self.home_odds = []
        self.draw_odds = []
        self.away_odds = []
        self.country = []
        self.league = []


def generate_matches(pgSoup, defaultVal=None):
    evtSel = {
        'time': 'p.whitespace-nowrap',
        'game': 'a div:has(>a[title])',
        'score': 'a:has(a[title])+div.hidden',
        'home_odds': 'a:has(a[title])~div:not(.hidden)',
        'draw_odds': 'a:has(a[title])~div:not(.hidden)+div:nth-last-of-type(3)',
        'away_odds': 'a:has(a[title])~div:nth-last-of-type(2)',
    }

    events, current_group = [], {}
    for evt in pgSoup.select('div[set]>div:last-child'):
        if evt.parent.select(f':scope>div:first-child+div+div'):
            cgVals = [v.get_text(' ').strip() if v else defaultVal for v in [
                evt.parent.select_one(s) for s in
                [':scope>div:first-child+div>div:first-child',
                 ':scope>div:first-child>a:nth-of-type(2):nth-last-of-type(2)',
                 ':scope>div:first-child>a:nth-of-type(3):last-of-type']]]
            current_group = dict(zip(['date', 'country', 'league'], cgVals))

        evtRow = {'date': current_group.get('date', defaultVal)}
        for k, v in evtSel.items():
            v = evt.select_one(v).get_text(' ') if evt.select_one(v) else defaultVal
            evtRow[k] = ' '.join(v.split()) if isinstance(v, str) else v
        evtRow['country'] = current_group.get('country', defaultVal)
        evtRow['league'] = current_group.get('league', defaultVal)

        events.append(evtRow)
    return events


def parse_data(url, return_urls=False):
    browser = create_driver()
    browser.get(url)
    browser.implicitly_wait(25)
    soup = bs(browser.page_source, "lxml")

    game_data = GameData()
    game_keys = [a for a, av in game_data.__dict__.items() if isinstance(av, list)]
    for row in generate_matches(soup, defaultVal=nan):
        for k in game_keys: getattr(game_data, k).append(row.get(k, nan))
    if return_urls:
        span = soup.find('span', {'class': 'next-games-date'})
        a_tags = soup.select('span.next-games-date a[href]') # Instead of a_tags = soup.select('span.next-games-date a[href]')
        urls = ['https://www.oddsportal.com' + a_tag['href'] for a_tag in a_tags]
        return game_data, urls
    return game_data


if __name__ == '__main__':
    games = None
    pool = ThreadPool(5)  # We will be getting, however, 7 URLs
    # Get today's data and the Urls for the other days:
    game_data_today, urls = pool.apply(parse_data, args=('https://www.oddsportal.com/matches/soccer', True))
    urls.pop(1)  # Remove url for today: We already have the data for that
    game_data_results = pool.imap(parse_data, urls)
    for i in range(8):
        try:
            game_data = game_data_today if i == 1 else next(game_data_results)
            result = pd.DataFrame(game_data.__dict__)
            if games is None:
                games = result
            else:
                games = games.append(result, ignore_index=True)
        except ValueError:
            game_data = game_data_today if i == 1 else next(game_data_results)
            result = pd.DataFrame(game_data.__dict__)
            if games is None:
                games = result
            else:
                games = games.append(result, ignore_index=True)
        finally:
            pass
    # print(games)
    # ensure all the drivers are "quitted":
    del threadLocal
    import gc

    gc.collect()  # a little extra insurance

错误：

  File "C:\Users\User\AppData\Roaming\JetBrains\PyCharmCE2022.1\scratches\scratch_1.py", line 114, in <module>
    urls.pop(1)  # Remove url for today: We already have the data for that
IndexError: pop from empty list

虽然循环是在可用日期之间抓取，但我无法让它完整地工作。

我不想检查a_tags = [] if span is None else span.find_all('a')因为它不符合我的目的

这部分代码：

def generate_matches(pgSoup, defaultVal=None):
    evtSel = {
        'time': 'p.whitespace-nowrap',
        'game': 'a div:has(>a[title])', 
        'score': 'a:has(a[title])+div.hidden', 
        'home_odds': 'a:has(a[title])~div:not(.hidden)',
        'draw_odds': 'a:has(a[title])~div:not(.hidden)+div:nth-last-of-type(3)',
        'away_odds': 'a:has(a[title])~div:nth-last-of-type(2)', 
    }

    events, current_group = [], {}
    for evt in pgSoup.select('div[set]>div:last-child'):
        if evt.parent.select(f':scope>div:first-child+div+div'):
            cgVals = [v.get_text(' ').strip() if v else defaultVal for v in [
              evt.parent.select_one(s) for s in 
              [ ':scope>div:first-child+div>div:first-child',
                ':scope>div:first-child>a:nth-of-type(2):nth-last-of-type(2)',
                ':scope>div:first-child>a:nth-of-type(3):last-of-type' ]]]
            current_group = dict(zip(['date', 'country', 'league'], cgVals))
        
        evtRow = {'date': current_group.get('date', defaultVal)}
        for k, v in evtSel.items():
            v = evt.select_one(v).get_text(' ') if evt.select_one(v) else defaultVal
            evtRow[k] = ' '.join(v.split()) if isinstance(v, str) else v
        evtRow['country'] = current_group.get('country', defaultVal)
        evtRow['league'] = current_group.get('league', defaultVal)

        events.append(evtRow)
    return events

收集

编辑：将a_tags = soup.select('span.next-games-date a[href]')更改为a_tags = soup.select('span.next-games-date a[href]')

Answer 1

 span = soup.find('span', {'class': 'next-games-date'}) a_tags = span.findAll('a')

 a_tags = span.findAll('a') AttributeError: 'NoneType' object has no attribute 'findAll'

_{您可以通过检查None来避免引发错误，例如a_tags = [] if span is None else span.findAll('a') [这是您在使用.find / .select_one btw 时应该习惯性做的事情]，但是您会由于下面解释的原因，总是得到一个空列表。}

你为什么要使用.find('span', {'class': 'next-games-date'})当 [如你 q 的最后一个屏幕截图所示] 链接不包含在span标签中，并且 [as据我从检查中可以看出] next-games-date class 根本没有任何元素？ 考虑到这一点， .find...返回None并且下一行引发错误也就不足为奇了。

---

鉴于检查选项卡的屏幕截图显示其中一个祖先是带有tabs class 的div ，并且所有a标签都有一个h-8 class，您可以使用类似

    if return_urls:
        a_cont = soup.find('div', {'class': 'tabs'})
        if a_cont is None: a_tags = [] # check for None to be safe
        else: a_tags =  a_cont.find_all('a', {'class':'h-8', 'href':True}) 
        urls = [
            'https://www.oddsportal.com'+a_tag['href'] for a_tag in a_tags
            if not a_tag['href'].startswith('#') # sections in current page 
            and 'active-item-calendar' not in a_tag['class'] # current page
        ]

或者，使用.select ：

    if return_urls:
        a_sel = 'div.tabs a.h-8:not([href^="#"]):not(.active-item-calendar)'
        urls = ['https://www.oddsportal.com'+a['href'] 
                for a in soup.select(a_sel)]

[由于您已经过滤掉了当前页面（应该是当天的页面），其中'active-item-calendar' not in a_tag['class']或:not(.active-item-calendar)中，因此不需要pop任何项目...]

Answer 2

Traceback 从字面上说明了您的代码出错的位置和原因。 在 parse_data fnc 中， soup.find('span', {'class': 'next-games-date'})找不到给定的 class。根据 BeautifulSoup 文档，“如果 find() 找不到任何东西，它返回 None:"，因此span被分配为 NoneType obj。 当运行以下行 -- a_tags = span.findAll('a') -- 时，会发生 AtributeError，因为 NoneType 对象没有findAll方法。

代码还有 2 个错误：
1.) 应该是span.find_all('a)而不是span.findAll('a')
2.) 网站上没有 class 属性 next-games-date（至少我找不到）。

出于某种原因，select 方法没有找到带有 class .flex gap-2 py-3 overflow-x-auto text-xs no-scrollbar的div ，但我能够使用find方法找到它并制作一个 url 列表. 继承人的代码：

from selenium import webdriver
from bs4 import BeautifulSoup
from time import sleep

def get_page_source(url):
    try:
        driver = webdriver.Chrome()
        driver.get(url)
        sleep(3)
        return driver.page_source
    finally: driver.quit()

def grab_button_urls(html, tag, attr):
    soup = BeautifulSoup(html, 'html.parser')
    a_tags = soup.find(tag, attr)\
        .find_all('a')
    return [f"https://www.oddsportal.com{tag['href']}" for tag in a_tags]

if __name__ == '__main__':
    website = 'https://www.oddsportal.com/matches/soccer'
    # get page source
    pg_html = get_page_source(website)
    # get urls from date buttons
    tag, class_attr = 'div', {'class':'flex gap-2 py-3 overflow-x-auto text-xs no-scrollbar'}
    tags = grab_button_urls(pg_html, tag, class_attr)
    print(tags)