熊猫导出到csv

Question

I have the code bellow:

from xlsxwriter import Workbook
import os,shutil
import requests
import pandas
from bs4 import BeautifulSoup
MAX_RETRIES = 20
base_url='https://pagellapolitica.it/politici/sfoggio/9/matteo-renzi?page='

for page in range(1,32,1):
    l=[]
    session = requests.Session()
    adapter = requests.adapters.HTTPAdapter(max_retries=MAX_RETRIES)
    session.mount('https://', adapter)
    session.mount('http://', adapter)

    site=(base_url+str(page)+".html")
   # print(site)
    c=session.get(site)
    r=c.content
    soup=BeautifulSoup(r,'html.parser')
    all=soup.find_all("div",{"class":"clearfix"})


for d in all:
    links=d.find_all("a")
    len(links)
    l=[]
   # workbook = Workbook('bbb.xlsx')
   # worksheet = workbook.add_worksheet()
   # row +=0
   # worksheet.write(row,0,'Link')
   # worksheet.write(row,1,'Name ')
   # row+=1
    for a in links[5:17]:

        d={}
        href=(a["href"])
        basic_url=('https://pagellapolitica.it/')
        site =basic_url + href
        #print(site)

        c=requests.get(site)
        r=c.content
        soup=BeautifulSoup(r,'html.parser')
        Name=soup.find("h3",{"class":"pull-left"}).text
        Fact_checking=soup.find("label",{"class":"verdict-analisi"}).text
        quote=soup.find("div",{"class":"col-xs-12 col-sm-6 col-smm-6 col-md-5 col-lg-5"}).text
        all=soup.find_all("span",{"class":"item"})
        Topic=all[0].text
        Date=all[2].text
        a=all[3].find("a",{"class":"" ""})
        Link=a["href"]
        Text=soup.find("div",{"class":"col-xs-12 col-md-12 col-lg-12"}).text
        d["Name"]=Name
        d["Fact_checking"]=Fact_checking
        d["Quote"]=quote
        d["Economic_topic"]=Topic
        d["Date"]=Date
        d["Link"]=Link
        d["Text"]=Text
        l.append(d)

        df=pandas.DataFrame(l) 
        df.to_csv("outing.csv")

The problem that when I export the data in csv I only get 6 rows of results.When I do print(df) and print(l) it prints all the data that I havein the list,however when i check len(l) I only get a lenght of 6. Any ideas why this is happening?? Thank you in advance!

Answer 1

Consider building a list of dictionaries binded into individual dataframes with pandas.DataFrame() and then concatenate a list of individual dataframes with pandas.concat() for final dataframe.

df_list = [] 

for d in all:
    links=d.find_all("a")
    len(links)
    l=[]

    for a in links[5:17]:    
        d={}
        ...
        d["Name"]=Name
        d["Fact_checking"]=Fact_checking
        d["Quote"]=quote
        d["Economic_topic"]=Topic
        d["Date"]=Date
        d["Link"]=Link
        d["Text"]=Text
        l.append(d)

    df_list.append(pandas.DataFrame(l))

final_df = pandas.concat(df_list)
final_df.to_csv("outing.csv")