如何使用 selenium python 從動態表中提取數據？

Question

我正在嘗試從網站中提取數據。 我需要在搜索框中輸入值，然后查找詳細信息。 它將生成一個表。 生成表后，需要將詳細信息寫入文本文件或將它們插入數據庫。 我正在嘗試以下事情。

網站： https ://commtech.byu.edu/noauth/classSchedule/index.php 搜索文本：“CS 142”

示例代碼

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys

from selenium.webdriver.chrome.service import Service

from selenium.webdriver.chrome.options import Options
c_options = Options()
c_options.add_experimental_option("detach", True)

s = Service('C:/Users/sidat/OneDrive/Desktop/python/WebDriver/chromedriver.exe')



URL = "http://saasta.byu.edu/noauth/classSchedule/index.php"
driver = webdriver.Chrome(service=s, options=c_options)
driver.get(URL)
element = driver.find_element("id", "searchBar")
element.send_keys("C S 142", Keys.RETURN)
search_button = driver.find_element("id", "searchBtn")
search_button.click()

table = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//*[@id='sectionTable']")))

rows = table.find_elements("xpath", "//tr")

for row in rows:
    cells = row.find_elements(By.TAG_NAME, "td")
    for cell in cells:
        print(cell.text)

我正在使用 PyCharm 2022.3 編碼和測試結果。 我的代碼沒有打印任何內容。 請幫我解決這個問題，將數據提取到文本文件和 SQL 數據庫表中。

Answer 1

試試這個：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options

c_options = Options()
c_options.add_experimental_option("detach", True) 
s = Service('C:/Users/sidat/OneDrive/Desktop/python/WebDriver/chromedriver.exe')

driver = webdriver.Chrome()
URL = "http://saasta.byu.edu/noauth/classSchedule/index.php"
driver.get(URL)
driver.maximize_window()
element = driver.find_element("id", "searchBar")
element.send_keys("C S 142", Keys.RETURN)
search_button = driver.find_element("id", "searchBtn")
search_button.click()

header = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//table[@id='sectionTable']/thead/tr/th")))
for th in header:
    print(f"{th.get_attribute('textContent')}")


rows = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//table[@id='sectionTable']/tbody/tr")))
for i in range(0, len(rows)):
    cells = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, f"(//table[@id='sectionTable']/tbody/tr)[{i+1}]//td")))
    for cell in cells:
        print(cell.get_attribute('textContent'))

您正在等待表格，這是正確的，但表格已完全加載（ td尚未加載）。

    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//*[@id='sectionTable']//td")))

然后你至少等待將任何內容放入td元素

Answer 2

以下代碼打印您要求的表格的內容。
如果您要單擊它們或向它們發送文本，則需要等待元素可單擊，或者如果您想閱讀其文本內容，則需要等待可見性。

from selenium import webdriver
from selenium.webdriver import Keys
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

options = Options()
options.add_argument("start-maximized")

webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 30)

url = "http://saasta.byu.edu/noauth/classSchedule/index.php"
driver.get(url)

wait.until(EC.element_to_be_clickable((By.ID, "searchBar"))).send_keys("C S 142", Keys.RETURN)
wait.until(EC.element_to_be_clickable((By.ID, "searchBtn"))).click()

table = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//*[@id='sectionTable']")))
headers = table.find_elements("xpath", ".//thead//th")
cells = table.find_elements("xpath", ".//tbody//td")

headers_text = ""
for header in headers:
    cell_text = header.text
    headers_text = headers_text + cell_text.ljust(10)

cells_text = ""
for cell in cells:
    c_text = cell.text
    cells_text = cells_text + c_text.ljust(10)

print(headers_text)
print(cells_text)

輸出是：

Section   Type      Mode      InstructorCredits   Term      Days      Start     End       Location  Available Waitlist  
002       DAY       Classroom           3.00                                              TBA       0/0       0