下一頁 python beautifulsoup

Question

我是 Python 的新手，並且堅持使用“下一頁”邏輯。

我嘗試了 while loop & selenium 與 chrome 沒有任何效果。

請對此有所了解-

import requests
from bs4 import BeautifulSoup
import csv 

pages = [ 0 , 25 , 50 , 75]
for page in pages:
    source = requests.get('https://finance.yahoo.com/screener/predefined/day_gainers?count=25&offset={}'.format(page)).text

soup = BeautifulSoup(source , 'lxml') 

for link in soup.find_all("a"):
    table = soup.find("table",{"class":"W(100%)"})
    thead = table.find("thead").find_all("th")
    table_head = [th.text for th in thead]
    #print(table_head)

    table_body = table.find ("tbody").find_all("tr")
        
with open("report.csv" , "a" , newline="") as csv_file:
        csv_write = csv.writer(csv_file)
        csv_write.writerow(table_head)
        
        for tr in table_body:
            table_data = [td.text.strip() for td in tr.find_all('td') ]
            csv_write.writerow(table_data)

Answer 1

我認為需要縮進你的代碼並且它的工作正常。 這是代碼：

import requests
from bs4 import BeautifulSoup
import csv

pages = [ 0 , 25 , 50 , 75]
for page in pages:
    
    source = requests.get('https://finance.yahoo.com/screener/predefined/day_gainers?count=25&offset={}'.format(page)).text
    

    soup = BeautifulSoup(source , 'lxml')


    for link in soup.find_all("a"):
        table = soup.find("table",{"class":"W(100%)"})
        thead = table.find("thead").find_all("th")
        table_head = [th.text for th in thead]
        #print(table_head)

        table_body = table.find ("tbody").find_all("tr")

        with open("report.csv" , "a" , newline="") as csv_file:
                csv_write = csv.writer(csv_file)
                csv_write.writerow(table_head)

                for tr in table_body:
                    table_data = [td.text.strip() for td in tr.find_all('td') ]
                    csv_write.writerow(table_data)

編輯第二個 for 循環獲取重復值。 所以刪除第二個 for 循環。 這是編輯后的代碼。

import requests
from bs4 import BeautifulSoup
import csv

pages = [ 0,25,50,75 ]
for page in pages:
    source = requests.get('https://finance.yahoo.com/screener/predefined/day_gainers?count=25&offset={}'.format(page)).text
    soup = BeautifulSoup(source , 'lxml')

    table = soup.find("table",{"class":"W(100%)"})
    thead = table.find("thead").find_all("th")
    table_head = [th.text for th in thead]
    table_body = table.find ("tbody").find_all("tr")
    with open("report.csv" , "a" , newline="") as csv_file:
            csv_write = csv.writer(csv_file)
            csv_write.writerow(table_head)
            for tr in table_body:
                table_data = [td.text.strip() for td in tr.find_all('td') ]
                csv_write.writerow(table_data)