![](/img/trans.png)
[英]How to get all links from website using Beautiful Soup (python) Recursively
[英]How to save all links from all pages to csv using python beautiful soup
我正在嘗試將從多個分頁頁面收集的所有鏈接保存到 csv。 從 print(links) 我可以看到我想從多個頁面保存的所有鏈接,但不幸的是,當我打開 csv 文件時,我只找到一個 URL 保存。 如何將我從終端看到的所有 URL(打印(鏈接))保存到 csv?
下面是我的代碼:
def scrape_pages(url) -> 無:
#max_pages = 10
max_pages = 5 # doing 3 pages for examples sake
current_page = 1
# Loop through all pages dynamically and build the url using the page number suffix the website uses
while current_page <= max_pages:
print(f'{url}page/{current_page}')
# Get each page's html
raw_html1 = requests.get(f'{url}page/{current_page}')
soup1 = BeautifulSoup(raw_html1.text, 'html.parser')
current_page += 1
# Find all table rows and from each table row get the needed data
#root = 'https://www.myjobmag.com'
for link1 in soup1.find_all('li',{'class':'mag-b'}):
link2 = link1.find('a',href=True)
link3 = 'https://www.myjobmag.com'+(link2['href'])
links = []
[links.append(link3) for link2 in link1 ]
for link2 in links:
raw_html = urlopen(link3)
soup = BeautifulSoup(raw_html.read(), 'html.parser')
def getTitle(soup):
return soup.find('h2', class_="mag-b").text.strip()
def getCompany(soup):
return soup.find('li', class_="job-industry").text.strip()
def getInfo(soup):
return soup.find('ul', class_="job-key-info").text.strip()
def getDescription(soup):
return soup.find('div', class_="job-details").text.strip()
def getApplication(soup):
return soup.find('div', class_="mag-b bm-b-30").text.strip()
with open('output.csv', 'w', encoding='utf8', newline='') as
f_output:
csv_output = csv.writer(f_output)
csv_output.writerow(['Title', 'Info', 'Desc', 'Application'])
row = [getTitle(soup), getCompany(soup), getInfo(soup),
getDescription(soup), getApplication(soup)]
print(row)
for f_output in row:
csv_output.writerow(row)
# print(product, row, Title, Company, Info, Description, Application)
time.sleep(5) # sleep before scraping next page to not send too
many requests at once
print('\n\n') # Clearing console up
def main() -> int:
URL = 'https://www.myjobmag.com/'
scrape_pages(URL)
return 0
如果名稱== '主':退出(主())
import requests
from bs4 import BeautifulSoup
import pandas as pd
import time as t
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'}
s = requests.Session()
s.headers.update(headers)
links_list = []
for x in range(1, 3):
r = s.get(f'https://www.myjobmag.com/page/{x}')
soup = BeautifulSoup(r.text, 'html.parser')
links = soup.select_one('ul.job-list').select('li.job-list-li')
for link in links:
try:
title = link.select_one('h2').text.strip()
url = link.select_one('h2').select_one('a').get('href')
r = s.get(f'https://www.myjobmag.com{url}')
soup = BeautifulSoup(r.text, 'html.parser')
key_info = soup.select_one('ul.job-key-info').text.strip()
description = soup.select_one('div.job-details').text.strip()
application_method = soup.select_one('div.mag-b.bm-b-30').text.strip()
links_list.append((title, key_info, description, application_method, url))
print(f'done {title} -- {url}')
t.sleep(5)
except Exception as e:
print(e)
df = pd.DataFrame(links_list, columns = ['title', 'key_info', 'description', 'application_method', 'url'])
df.to_csv('my_wonderful_jobs_list.csv')
這將返回一個包含職位名稱、關鍵信息、描述、申請方法和網址的 csv 文件。
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.