从网页表格中抓取价值

Question

我想从以下网站检索匹配数据：

https://understat.com/match/81

我写了以下脚本：

import sys
import time 
import os
import io
import csv

from selenium import webdriver
import selenium.webdriver.support.expected_conditions as ec
import selenium.webdriver.support.ui as ui
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By

driver = None
cnx = None
currentDir = sys.path[0]

def scrap_understat():
   init_browser('firefox')

   for i in range(80, 10080):
      try:
         driver.get('https://understat.com/match/' + str(i))
         time.sleep(1)
         if try_find_Element(driver, By.CLASS_NAME, 'error-code') is not None:
            continue

         data = get_match_data()
         save_data(data)

      except Exception as ex:
         log_this(ex)
         print(str(ex))

   close_browser()


def get_match_data():
   data = []

   teams = driver.find_elements(By.NAME, 'team')

   for team in teams:
       team.find_element(By.XPATH, 'following-sibling::*').click()
       time.sleep(1)
       players = driver.find_element(By.ID, 'match-rosters').find_element(By.TAG_NAME, 'tbody').find_elements(By.TAG_NAME, 'tr')
       for player in players:
           tds = player.find_elements(By.TAG_NAME, 'td')
           record = [tds[1].text, tds[2].text, tds[3].text, tds[4].text,
                  tds[5].text, tds[6].text, tds[7].text, tds[8].text, tds[9].text]
           data.append(record)

   return data


def save_data(data):
# save CSV
  csv_file_path = currentDir + '/output' + current_time + '.csv'
  file = None
  writer = None
  if not os.path.exists(csv_file_path):
     file = io.open(csv_file_path, 'w', newline='', encoding='ISO-8859-1')
     writer = csv.writer(file)
     writer.writerow(
        ['player', 'pos', 'min', 'sh', 'g', 'kp', 'a', 'xG',
         'xA'])
  else:
      file = io.open(csv_file_path, 'a', newline='', encoding='ISO-8859-1')
      writer = csv.writer(file)
  for record in data:
      writer.writerow(record)
      file.close()

我的脚本的输出如下所示：

因此，xG和xA列存在问题。 我只需要Lowerscript-part，而脚本将td中的所有文本都包含在内。如何更改脚本，使其仅包含第一部分？ 通过检查页面元素，我看到不需要的部分称为子类

第二个问题：如何将球队名称声明为变量（曼彻斯特联队/托特纳姆热刺队）

Answer 1

尝试这一操作以避免匹配子文本：

record = [tds[1].text, tds[2].text, tds[3].text, tds[4].text,
                  tds[5].text, tds[6].text, tds[7].text,
          driver.execute_script('return arguments[0].firstChild.textContent', tds[8]), 
          driver.execute_script('return arguments[0].firstChild.textContent', tds[9])]

要获取队名，您可以使用

home = driver.find_element_by_xpath('//label[@for="team-home"]').text
away = driver.find_element_by_xpath('//label[@for="team-away"]').text

PS考虑使用Waits而不是time.sleep

Answer 2

看来您只需要删除sup ：

driver.execute_script("$('sup').remove()")