[英]Writing text to csv after web-scraping
我正在通过 python 中的抓取来提取房地产数据。 我希望这些数据在 CSV 文件中。
当我将数据写入 CSV 时,如果第一个抓取的项目没有我需要的值,它只会跳过所有行(但其他项目具有该值),它是空的并且不创建任何行,即使是空值也不行。
我的网页抓取代码块:
from selenium import webdriver
from bs4 import BeautifulSoup
import re
import csv
import time
PATH = 'C:\Program Files (x86)\chromedriver.exe'
driver = webdriver.Chrome(PATH)
data = []
def get_dl(soup):
d_list = {}
for dl in soup.findAll("dl", {"class": "obj-details"}):
for el in dl.find_all(["dt", "dd"]):
if el.name == 'dt':
key = el.get_text(strip=True)
elif key in ['Plotas:', 'Buto numeris:', 'Metai:', 'Namo numeris:', 'Kambarių sk.:', 'Aukštas:', 'Aukštų sk.:', 'Pastato tipas:', 'Šildymas:', 'Įrengimas:', 'Pastato energijos suvartojimo klasė:', 'Ypatybės:', 'Papildomos patalpos:', 'Papildoma įranga:', 'Apsauga:']:
d_list[key] = ' '.join(el.text.strip().replace("\n", ", ").split('NAUDINGA')[0].split('m²')[0].split())
return d_list
for puslapis in range(1, 2):
driver.get(f'https://www.aruodas.lt/butai/kaune/puslapis/{puslapis}')
response = driver.page_source
soup = BeautifulSoup(response, 'html.parser')
blocks = soup.find_all('tr', class_='list-row')
stored_urls = []
for url in blocks:
try:
stored_urls.append(url.a['href'])
except:
pass
for link in stored_urls:
driver.get(link)
response = driver.page_source
soup = BeautifulSoup(response, 'html.parser')
h1 = soup.find('h1', 'obj-header-text')
price = soup.find('div', class_ = 'price-left')
try:
address1 = h1.get_text(strip=True)
address2 = re.findall(r'(.*),[^,]*$', address1)
address = ''.join(address2)
city, district, street = address.split(',')
except:
city, district, street = 'NaN'
try:
full_price = price.find('span', class_ = 'price-eur').text.strip()
full_price1 = full_price.replace('€', '').replace(' ','').strip()
except:
full_price1 = 'NaN'
try:
price_sq_m = price.find('span', class_ = 'price-per').text.strip()
price_sq_m1 = price_sq_m.replace('€/m²)', '').replace('(domina keitimas)', '').replace('(', '').replace(' ','').strip()
except:
price_sq_m1 = 'NaN'
try:
price_change = price.find('div', class_ = 'price-change').text.strip()
price_change1 = price_change.replace('%', '').strip()
except:
price_change1 = 'NaN'
data.append({'city': city, 'district': district, 'street': street, 'full_price': full_price1, 'price_sq_m': price_sq_m1, 'price_change': price_change1, **get_dl(soup)})
例如在键列表中有值:
['Ypatybės:']:
但是在页面中,我正在刮第一个平面的地方没有那个价值,而且根本没有创建行,这不是我需要的。
用于写入 csv 的代码块:
with open('output_kaunas.csv', 'w', encoding='utf-8', newline='') as f_output:
csv_output = csv.DictWriter(f_output, fieldnames=data[0].keys(), extrasaction='ignore')
csv_output.writeheader()
csv_output.writerows(data)
所以,我的问题是,如何创建具有我需要的功能的行,即使该功能在第一个抓取的项目中也不存在。
要将数据存储在 csv 文件中,您可以使用 pandas Dataframe
df = pd.DataFrame(data).to_csv('output_kaunas.csv',index=False)
根据您的完整代码:
from selenium import webdriver
from bs4 import BeautifulSoup
import re
import pandas as pd
import time
PATH = 'C:\Program Files (x86)\chromedriver.exe'
driver = webdriver.Chrome(PATH)
data = []
def get_dl(soup):
d_list = {}
for dl in soup.findAll("dl", {"class": "obj-details"}):
for el in dl.find_all(["dt", "dd"]):
if el.name == 'dt':
key = el.get_text(strip=True)
elif key in ['Plotas:', 'Buto numeris:', 'Metai:', 'Namo numeris:', 'Kambarių sk.:', 'Aukštas:', 'Aukštų sk.:', 'Pastato tipas:', 'Šildymas:', 'Įrengimas:', 'Pastato energijos suvartojimo klasė:', 'Ypatybės:', 'Papildomos patalpos:', 'Papildoma įranga:', 'Apsauga:']:
d_list[key] = ' '.join(el.text.strip().replace("\n", ", ").split('NAUDINGA')[0].split('m²')[0].split())
return d_list
for puslapis in range(1, 2):
driver.get(f'https://www.aruodas.lt/butai/kaune/puslapis/{puslapis}')
response = driver.page_source
soup = BeautifulSoup(response, 'html.parser')
blocks = soup.find_all('tr', class_='list-row')
stored_urls = []
for url in blocks:
try:
stored_urls.append(url.a['href'])
except:
pass
for link in stored_urls:
driver.get(link)
response = driver.page_source
soup = BeautifulSoup(response, 'html.parser')
h1 = soup.find('h1', 'obj-header-text')
price = soup.find('div', class_ = 'price-left')
try:
address1 = h1.get_text(strip=True)
address2 = re.findall(r'(.*),[^,]*$', address1)
address = ''.join(address2)
city, district, street = address.split(',')
except:
city, district, street = 'NaN'
try:
full_price = price.find('span', class_ = 'price-eur').text.strip()
full_price1 = full_price.replace('€', '').replace(' ','').strip()
except:
full_price1 = 'NaN'
try:
price_sq_m = price.find('span', class_ = 'price-per').text.strip()
price_sq_m1 = price_sq_m.replace('€/m²)', '').replace('(domina keitimas)', '').replace('(', '').replace(' ','').strip()
except:
price_sq_m1 = 'NaN'
try:
price_change = price.find('div', class_ = 'price-change').text.strip()
price_change1 = price_change.replace('%', '').strip()
except:
price_change1 = 'NaN'
data.append({'city': city, 'district': district, 'street': street, 'full_price': full_price1, 'price_sq_m': price_sq_m1, 'price_change': price_change1, **get_dl(soup)})
df = pd.DataFrame(data).to_csv('output_kaunas.csv',index=False)
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.