只写第一行到csv

Question

I am trying to scrape page. I can get it to pull all the data and save it to array objects but cannot get my for loop to iterate over every index of the arrays and output those to CSV. It will write the headers and the first object. Novice to writing code so any help is appreciated.

from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as soup

my_url = 'https://www.sports-reference.com/cfb/schools/air-force/'

# Open Connection & Grabbing the Page
uClient = uReq(my_url)

#Creating variable to Save the Page
page_html = uClient.read()

#Closing the connection
uClient.close()

#Parse the data to HTML
page_soup = soup(page_html, "html.parser")

#Grab container info from the DOM
containers = page_soup.findAll("div",{"class":"overthrow table_container"})

filename = "airforce.csv"
f = open(filename, "w")

headers = "year, wins, losses, ties, wl, sos\n"

f.write(headers)

for container in containers:
 #Find all years
 year_container = container.findAll("td",{"data-stat":"year_id"})
 year = year_container[0].text

 #Find number of Wins
 wins_container = container.findAll("td",{"data-stat":"wins"})
 wins = wins_container[0].text

 #Find number of Wins
 losses_container = container.findAll("td",{"data-stat":"losses"})
 losses = losses_container[0].text

 #Number of Ties if any
 ties_container = container.findAll("td",{"data-stat":"ties"})
 ties = ties_container[0].text

 #Win-Loss as a percentage
 wl_container = container.findAll("td",{"data-stat":"win_loss_pct"})
 wl = wl_container[0].text


 #Strength of Schedule. Can be +/- w/0 being average
 sos_container = container.findAll("td",{"data-stat":"sos"})
 sos = sos_container[0].text

 f.write(year + "," + wins + "," + losses + "," + ties + "," + wl + "," + 
 sos + "\n")

f.close()

Answer 1

You want to find the table (body) and then iterate over the table rows that are not header rows, ie all rows that don't have a class.

For writing (and reading) CSV files there is a csv module in the standard library.

import csv
from urllib.request import urlopen

import bs4


def iter_rows(html):
    headers = ['year_id', 'wins', 'losses', 'ties', 'win_loss_pct', 'sos']
    yield headers

    soup = bs4.BeautifulSoup(html, 'html.parser')
    table_body_node = soup.find('table', 'stats_table').tbody
    for row_node in table_body_node('tr'):
        if not row_node.get('class'):
            yield [
                row_node.find('td', {'data-stat': header}).text
                for header in headers
            ]


def main():
    url = 'https://www.sports-reference.com/cfb/schools/air-force/'
    with urlopen(url) as response:
        html = response.read()

    with open('airforce.csv', 'w') as csv_file:
        csv.writer(csv_file).writerows(iter_rows(html))   


if __name__ == '__main__':
    main()

Answer 2

Pulling up the html source code, there is only one container to be put into your container list. Which means that your for loop is trying to access the wrong information.

You should use a range() generator to access the different elements of td that reside inside of the one item in your containers list.

try this

#number of records to iterate over
num = len(list(containers.findAll("td",{"data-stat":"year_id"})))

for i in range(num):
    #Find all years
    year_container = containers.findAll("td",{"data-stat":"year_id"})
    year = year_containers[i].text