如何使用 Selenium 和 python 从 Tradingview 网站单击“加载更多”按钮？

Question

I am trying to query all the rows from the tradingview website.I need just the ticker and name of stock. I am not able to click on the 'load more' button at the bottom of the page to load all the rows. Does anyone have a solution to this?

This is the code that I wrote to get the ticker and name of stock(It works till the 'load more' button).

from bs4 import BeautifulSoup

URL = 'https://www.tradingview.com/markets/stocks-usa/sectorandindustry-industry/biotechnology/'
page = requests.get(URL)

soup = BeautifulSoup(page.content, 'html.parser')`

for tr in soup.find_all('tr'):
    #tds = tr.find_all('td')
    ticker = tr.find('a', class_='tv-screener__symbol')
    stock_name = tr.find('span', class_='tv-screener__description')
    if None in (ticker, stock_name):
        continue
    print(ticker.text.strip())
    print(stock_name.text.strip())
    print("\n\n")

Here is the selenium code that I tried to write for the 'Load More' button but wasn't successful -

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

options = Options()
#options.headless = True
options.add_argument("--window-size=1920,1200")

driver = webdriver.Chrome(options=options, executable_path=DRIVER_PATH)
driver.get("https://www.tradingview.com/markets/stocks-usa/sectorandindustry-industry/biotechnology/")
loadMoreButton = driver.find_elements_by_xpath("//span[@class='tv-load-more__btn']")
loadMoreButton.click()
driver.quit()

And this is the error I got:

AttributeError                            Traceback (most recent call last)
<ipython-input-80-cf801ef16cdd> in <module>
      9 driver.get("https://www.tradingview.com/markets/stocks-usa/sectorandindustry-industry/biotechnology/")
     10 loadMoreButton = driver.find_elements_by_xpath("//div[@class='tv-load-more tv-load-more--screener js-screener-load-more']")
---> 11 loadMoreButton.click()
     12 driver.quit()

AttributeError: 'list' object has no attribute 'click'

Answer 1

The issue is find_elements_by_xpath this returns a list. See below using find_element_by_xpath .

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

options = Options()
#options.headless = True
options.add_argument("--window-size=1920,1200")

driver = webdriver.Chrome(options=options, executable_path=DRIVER_PATH)
driver.get("https://www.tradingview.com/markets/stocks-usa/sectorandindustry-industry/biotechnology/")
loadMoreButton = driver.find_element_by_xpath("//span[@class='tv-load-more__btn']")
loadMoreButton.click()
driver.quit()

---

---

I would suggest using By , WebDriverWait , expected_conditions , and element_to_be_clickable :

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options

options = Options()
#options.headless = True
options.add_argument("--window-size=1920,1200")

driver = webdriver.Chrome(options=options, executable_path=DRIVER_PATH)
driver.get("https://www.tradingview.com/markets/stocks-usa/sectorandindustry-industry/biotechnology/")
loadMoreButton = WebDriverWait(driver, 3).until(EC.element_to_be_clickable((By.XPATH, "//span[@class='tv-load-more__btn']")))
loadMoreButton.click()
driver.quit()

Answer 2

Written this piece of code just to click on the Load More button till the full page is loaded. Moreover, I always prefer using XPath over class names to find the element.

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
import time

driver = webdriver.Chrome()
wait = WebDriverWait(driver, 5)
action = ActionChains(driver)

driver.get("https://www.tradingview.com/markets/stocks-usa/sectorandindustry-industry/biotechnology/")
# Used to get a click count
i = 0

while True:
    try:
        # Given some timeout so that data will be loaded in the page
        time.sleep(2)
        Load_More = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[text()='Load More']")))
        action.move_to_element(Load_More).click().perform()
        i += 1
        print(f"Clicked {i} time.")
    except:
        print("Reached End of the Page")
        break