如何使用 python 从动态表中抓取内容？

Question

I'm trying to extract RSI indicator present on this page under the 'Oscillators' tab.

URL: https://in.tradingview.com/markets/stocks-india/market-movers-active/

I know that I'll have to use something like Selenium to access the tab first, but how do I access the 'oscilators' div.

I'll need to use selenium, and then I could use beautiful-soup to find the right tags and data, right?

Edit -

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from time import sleep
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import pandas as pd

# create object for chrome options
chrome_options = Options()
base_url = 'https://in.tradingview.com/markets/stocks-india/market-movers-active/'


# To disable the message, "Chrome is being controlled by automated test software"
chrome_options.add_argument("disable-infobars")
# Pass the argument 1 to allow and 2 to block
chrome_options.add_experimental_option("prefs", { 
    "profile.default_content_setting_values.notifications": 2
    })
# invoke the webdriver
browser = webdriver.Chrome(executable_path = r'/Users/judhjitganguli/Downloads/chromedriver',
                          options = chrome_options)

browser.get('chrome://settings/')
browser.execute_script('chrome.settingsPrivate.setDefaultZoom(0.5);')
browser.get(base_url)

delay = 5 #seconds

while True:
    try:
  # find tab/button
        osiButton = browser.find_element_by_css_selector('.tv-screener-toolbar__favorites div div div:nth-child(8)')
        print('button text: ' + osiButton.text)
        osiButton.click()
        WebDriverWait(browser, 9).until(EC.text_to_be_present_in_element((By.CSS_SELECTOR, 'th:nth-child(2) .js-head-title'), "OSCILLATORS RATING"))
  
  # table updated, get the data
        for row in browser.find_elements_by_css_selector(".tv-data-table__tbody tr"):
            print(row.text)
           
        #for cell in browser.find_elements_by_css_selector('td'):
         #   print(cell.text)

        
        
    except Exception as ex:
        print(ex)
    

# close the automated browser
browser.close()

In the output, I get the required data but it is an infinite loop. How do I get it into a pandas df?

Answer 1

after Oscillators clicked, wait and monitor element th:nth-child(2).js-head-title for change, from Last to Oscillators Rating using WebDriverWait

# if running headless make sure to add this argument
# or the oscillators tab will not visible or can't be clicked
#chrome_options.add_argument("window-size=1980,960");

try:
  # find tab/button
  osiButton = driver.find_element_by_css_selector('.tv-screener-toolbar__favorites div div div:nth-child(8)')
  print('button text: ' + osiButton.text)
  osiButton.click()
  WebDriverWait(driver, 9).until(
      EC.text_to_be_present_in_element((By.CSS_SELECTOR, 'th:nth-child(2) .js-head-title'), "OSCILLATORS RATING"))
  
  # table updated, get the data
  for row in driver.find_elements_by_css_selector('.tv-data-table__tbody tr'):
      print(row.text)
      #for cell in driver.find_elements_by_css_selector('td'):
         #print(cell.text)

except Exception as ex:
  print(ex)