硒无法刮桌子

Question

I'm unable to scrape the table from https://solanabeach.io/validators . For some reason, I can't access it using the following code snippet. Does anyone have an idea why I'm unable to scrape the table?

from bs4 import BeautifulSoup
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
import pandas as pd
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException

options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument("--enable-javascript")
options.add_argument('--no-sandbox')
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)

driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)

driver.get(f"https://solanabeach.io/validators")


driver.implicitly_wait(10)
api = BeautifulSoup(driver.find_element_by_xpath("//*").get_attribute("outerHTML"), 'html.parser')


table = api.findAll('tbody')

print(table)

driver.quit()

Answer 1

You dont need to use BeautifulSoup here. You can simply use selenium methods.

from selenium.webdriver.support import expected_conditions as EC
import re
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait

PATH = r"C:\Users\deepak.mathpal\IdeaProjects\Selenium4\src\main\resources\chromedriver_chrome_95\chromedriver.exe"
driver = webdriver.Chrome(PATH)

url = 'https://solanabeach.io/validators'

driver.get(url)
driver.maximize_window()
WebDriverWait(driver, 20).until(
    EC.visibility_of_element_located((By.XPATH, "//table[@class='table table-bordered maintable "
                                                "table-striped-even']//tbody/tr")))

columnHeader = driver.find_element(By.XPATH, "//table[@class='table table-bordered maintable "
                                             "table-striped-even']//thead")
print("---------------------------------------------------------------------------------------")
print(re.sub(r"\s+", '|', columnHeader.text.strip()))
print("---------------------------------------------------------------------------------------")
textInPage = driver.find_elements(By.XPATH, "//table[@class='table table-bordered maintable "
                                            "table-striped-even']//tbody/tr")
del textInPage[:2]
for element in textInPage:
    print(element.text)
    print("---------------------------------------------------------------------------------------")

driver.quit()

Output:

--------------------------------------------------------------------------------
#|VALIDATOR|STAKE|CUMULATIVE|STAKE|COMMISSION|LAST|VOTE
--------------------------------------------------------------------------------
20
VymD
1.8.3
3,896,651(81)
0.98 %
34.5 %
100 %
106,485,715
---------------------------------------------------------------------------------------
21
5KAX...PRuw
1.7.14
3,577,932(59)
0.90 %
35.4 %
100 %
106,485,711

And so on till 1270 items.