Parsing tags using Beautiful Soup and Python

Question

This is my code so far:

# URL page we will scraping (see image above)
url = "https://www.basketball-reference.com/leagues/NBA_2019_per_game.html"
# this is the HTML from the given URL
html = urlopen(url)
soup = BeautifulSoup(html)
soup.findAll('tr', limit=10)

It returns

<th aria-label="Personal Fouls Per Game" class=" poptip hide_non_quals center" data-stat="pf_per_g" data-tip="Personal Fouls Per Game" scope="col">PF</th>
 <th aria-label="Points Per Game" class=" poptip hide_non_quals center" data-stat="pts_per_g" data-tip="Points Per Game" scope="col">PTS</th>
 </tr>,
 <tr class="full_table"><th class="right " csk="1" data-stat="ranker" scope="row">1</th><td class="left " csk="Abrines,Álex" data-append-csv="abrinal01" data-stat="player"><a href="/players/a/abrinal01.html">Álex Abrines</a></td><td class="center " data-stat="pos">SG</td><td class="right " data-stat="age">25</td><td class="left " data-stat="team_id"><a href="/teams/OKC/2019.html">OKC</a></td><td class="right " data-stat="g">31</td><td class="right " data-stat="gs">2</td><td class="right non_qual" data-stat="mp_per_g">19.0</td><td class="right non_qual" data-stat="fg_per_g">1.8</td><td class="right non_qual" data-stat="fga_per_g">5.1</td><td class="right non_qual" data-stat="fg_pct">.357</td><td class="right non_qual" data-stat="fg3_per_g">1.3</td><td class="right non_qual" data-stat="fg3a_per_g">4.1</td><td class="right non_qual" data-stat="fg3_pct">.323</td><td class="right non_qual" data-stat="fg2_per_g">0.5</td><td class="right non_qual" data-stat="fg2a_per_g">1.0</td><td class="right non_qual" data-stat="fg2_pct">.500</td><td class="right non_qual" data-stat="efg_pct">.487</td><td class="right non_qual" data-stat="ft_per_g">0.4</td><td class="right non_qual" data-stat="fta_per_g">0.4</td><td class="right non_qual" data-stat="ft_pct">.923</td><td class="right non_qual" data-stat="orb_per_g">0.2</td><td class="right non_qual" data-stat="drb_per_g">1.4</td><td class="right non_qual" data-stat="trb_per_g">1.5</td><td class="right non_qual" data-stat="ast_per_g">0.6</td><td class="right non_qual" data-stat="stl_per_g">0.5</td><td class="right non_qual" data-stat="blk_per_g">0.2</td><td class="right non_qual" data-stat="tov_per_g">0.5</td><td class="right non_qual" data-stat="pf_per_g">1.7</td><td class="right non_qual" data-stat="pts_per_g">5.3</td></tr>,
 <tr class="full_table"><th class="right " csk="2" data-stat="ranker" scope="row">2</th><td class="left " csk="Acy,Quincy" data-append-csv="acyqu01" data-stat="player"><a href="/players/a/acyqu01.html">Quincy Acy</a></td><td class="center " data-stat="pos">PF</td><td class="right " data-stat="age">28</td><td class="left " data-stat="team_id"><a href="/teams/PHO/2019.html">PHO</a></td><td class="right " data-stat="g">10</td><td class="right iz" data-stat="gs">0</td><td class="right non_qual" data-stat="mp_per_g">12.3</td><td class="right non_qual" data-stat="fg_per_g">0.4</td><td class="right non_qual" data-stat="fga_per_g">1.8</td><td class="right non_qual" data-stat="fg_pct">.222</td><td class="right non_qual" data-stat="fg3_per_g">0.2</td><td class="right non_qual" data-stat="fg3a_per_g">1.5</td><td class="right non_qual" data-stat="fg3_pct">.133</td><td class="right non_qual" data-stat="fg2_per_g">0.2</td><td class="right non_qual" data-stat="fg2a_per_g">0.3</td><td class="right non_qual" data-stat="fg2_pct">.667</td><td class="right non_qual" data-stat="efg_pct">.278</td><td class="right non_qual" data-stat="ft_per_g">0.7</td><td class="right non_qual" data-stat="fta_per_g">1.0</td><td class="right non_qual" data-stat="ft_pct">.700</td><td class="right non_qual" data-stat="orb_per_g">0.3</td><td class="right non_qual" data-stat="drb_per_g">2.2</td><td class="right non_qual" data-stat="trb_per_g">2.5</td><td class="right non_qual" data-stat="ast_per_g">0.8</td><td class="right non_qual" data-stat="stl_per_g">0.1</td><td class="right non_qual" data-stat="blk_per_g">0.4</td><td class="right non_qual" data-stat="tov_per_g">0.4</td><td class="right non_qual" data-stat="pf_per_g">2.4</td><td class="right non_qual" data-stat="pts_per_g">1.7</td></tr>,
 <tr class="full_table"><th class="right " csk="3" data-stat="ranker" scope="row">3</th><td class="left " csk="Adams,Jaylen" data-append-csv="adamsja01" data-stat="player"><a href="/players/a/adamsja01.html">Jaylen Adams</a></td><td class="center " data-stat="pos">PG</td><td class="right " data-stat="age">22</td><td class="left " data-stat="team_id"><a href="/teams/ATL/2019.html">ATL</a></td><td class="right " data-stat="g">34</td><td class="right " data-stat="gs">1</td><td class="right non_qual" data-stat="mp_per_g">12.6</td><td class="right non_qual" data-stat="fg_per_g">1.1</td><td class="right non_qual" data-stat="fga_per_g">3.2</td><td class="right non_qual" data-stat="fg_pct">.345</td><td class="right non_qual" data-stat="fg3_per_g">0.7</td><td class="right non_qual" data-stat="fg3a_per_g">2.2</td><td class="right non_qual" data-stat="fg3_pct">.338</td><td class="right non_qual" data-stat="fg2_per_g">0.4</td><td class="right non_qual" data-stat="fg2a_per_g">1.1</td><td class="right non_qual" data-stat="fg2_pct">.361</td><td class="right non_qual" data-stat="efg_pct">.459</td><td class="right non_qual" data-stat="ft_per_g">0.2</td><td class="right non_qual" data-stat="fta_per_g">0.3</td><td class="right non_qual" data-stat="ft_pct">.778</td><td class="right non_qual" data-stat="orb_per_g">0.3</td><td class="right non_qual" data-stat="drb_per_g">1.4</td><td class="right non_qual" data-stat="trb_per_g">1.8</td><td class="right non_qual" data-stat="ast_per_g">1.9</td><td class="right non_qual" data-stat="stl_per_g">0.4</td><td class="right non_qual" data-stat="blk_per_g">0.1</td><td class="right non_qual" data-stat="tov_per_g">0.8</td><td class="right non_qual" data-stat="pf_per_g">1.3</td><td class="right non_qual" data-stat="pts_per_g">3.2</td></tr>,
 <tr class="full_table"><th class="right " csk="4" data-stat="ranker" scope="row">4</th><td class="left " csk="Adams,Steven" data-append-csv="adamsst01"

I want to know how for each tr class, I can get the a href and data-append-csv. So for example the first tr class, the data-append-csv is abrinal01.

Answer 1

For quick solution you can try something like:

import re

tags = page_soup.find_all('tr', limit=10)

for tag in tags:
    m = re.match('.+" data-append-csv="([^\"]+)" ', str(tag))
    if m:
        ge = m.groups()
        print(ge[0])

Same approach with href. For global / reuse solution you need more accurate code with soup parsing or more accurate regular expression

Answer 2

If you want only the data-append-csv and href values, then you may use my code. I am using list comprehensions with find .

Code

from bs4 import BeautifulSoup
import requests

txt = '''
<th aria-label="Personal Fouls Per Game" class=" poptip hide_non_quals center" data-stat="pf_per_g" data-tip="Personal Fouls Per Game" scope="col">PF</th>
 <th aria-label="Points Per Game" class=" poptip hide_non_quals center" data-stat="pts_per_g" data-tip="Points Per Game" scope="col">PTS</th>
 </tr>,
 <tr class="full_table"><th class="right " csk="1" data-stat="ranker" scope="row">1</th><td class="left " csk="Abrines,Álex" data-append-csv="abrinal01" data-stat="player"><a href="/players/a/abrinal01.html">Álex Abrines</a></td><td class="center " data-stat="pos">SG</td><td class="right " data-stat="age">25</td><td class="left " data-stat="team_id"><a href="/teams/OKC/2019.html">OKC</a></td><td class="right " data-stat="g">31</td><td class="right " data-stat="gs">2</td><td class="right non_qual" data-stat="mp_per_g">19.0</td><td class="right non_qual" data-stat="fg_per_g">1.8</td><td class="right non_qual" data-stat="fga_per_g">5.1</td><td class="right non_qual" data-stat="fg_pct">.357</td><td class="right non_qual" data-stat="fg3_per_g">1.3</td><td class="right non_qual" data-stat="fg3a_per_g">4.1</td><td class="right non_qual" data-stat="fg3_pct">.323</td><td class="right non_qual" data-stat="fg2_per_g">0.5</td><td class="right non_qual" data-stat="fg2a_per_g">1.0</td><td class="right non_qual" data-stat="fg2_pct">.500</td><td class="right non_qual" data-stat="efg_pct">.487</td><td class="right non_qual" data-stat="ft_per_g">0.4</td><td class="right non_qual" data-stat="fta_per_g">0.4</td><td class="right non_qual" data-stat="ft_pct">.923</td><td class="right non_qual" data-stat="orb_per_g">0.2</td><td class="right non_qual" data-stat="drb_per_g">1.4</td><td class="right non_qual" data-stat="trb_per_g">1.5</td><td class="right non_qual" data-stat="ast_per_g">0.6</td><td class="right non_qual" data-stat="stl_per_g">0.5</td><td class="right non_qual" data-stat="blk_per_g">0.2</td><td class="right non_qual" data-stat="tov_per_g">0.5</td><td class="right non_qual" data-stat="pf_per_g">1.7</td><td class="right non_qual" data-stat="pts_per_g">5.3</td></tr>,
 <tr class="full_table"><th class="right " csk="2" data-stat="ranker" scope="row">2</th><td class="left " csk="Acy,Quincy" data-append-csv="acyqu01" data-stat="player"><a href="/players/a/acyqu01.html">Quincy Acy</a></td><td class="center " data-stat="pos">PF</td><td class="right " data-stat="age">28</td><td class="left " data-stat="team_id"><a href="/teams/PHO/2019.html">PHO</a></td><td class="right " data-stat="g">10</td><td class="right iz" data-stat="gs">0</td><td class="right non_qual" data-stat="mp_per_g">12.3</td><td class="right non_qual" data-stat="fg_per_g">0.4</td><td class="right non_qual" data-stat="fga_per_g">1.8</td><td class="right non_qual" data-stat="fg_pct">.222</td><td class="right non_qual" data-stat="fg3_per_g">0.2</td><td class="right non_qual" data-stat="fg3a_per_g">1.5</td><td class="right non_qual" data-stat="fg3_pct">.133</td><td class="right non_qual" data-stat="fg2_per_g">0.2</td><td class="right non_qual" data-stat="fg2a_per_g">0.3</td><td class="right non_qual" data-stat="fg2_pct">.667</td><td class="right non_qual" data-stat="efg_pct">.278</td><td class="right non_qual" data-stat="ft_per_g">0.7</td><td class="right non_qual" data-stat="fta_per_g">1.0</td><td class="right non_qual" data-stat="ft_pct">.700</td><td class="right non_qual" data-stat="orb_per_g">0.3</td><td class="right non_qual" data-stat="drb_per_g">2.2</td><td class="right non_qual" data-stat="trb_per_g">2.5</td><td class="right non_qual" data-stat="ast_per_g">0.8</td><td class="right non_qual" data-stat="stl_per_g">0.1</td><td class="right non_qual" data-stat="blk_per_g">0.4</td><td class="right non_qual" data-stat="tov_per_g">0.4</td><td class="right non_qual" data-stat="pf_per_g">2.4</td><td class="right non_qual" data-stat="pts_per_g">1.7</td></tr>,
 <tr class="full_table"><th class="right " csk="3" data-stat="ranker" scope="row">3</th><td class="left " csk="Adams,Jaylen" data-append-csv="adamsja01" data-stat="player"><a href="/players/a/adamsja01.html">Jaylen Adams</a></td><td class="center " data-stat="pos">PG</td><td class="right " data-stat="age">22</td><td class="left " data-stat="team_id"><a href="/teams/ATL/2019.html">ATL</a></td><td class="right " data-stat="g">34</td><td class="right " data-stat="gs">1</td><td class="right non_qual" data-stat="mp_per_g">12.6</td><td class="right non_qual" data-stat="fg_per_g">1.1</td><td class="right non_qual" data-stat="fga_per_g">3.2</td><td class="right non_qual" data-stat="fg_pct">.345</td><td class="right non_qual" data-stat="fg3_per_g">0.7</td><td class="right non_qual" data-stat="fg3a_per_g">2.2</td><td class="right non_qual" data-stat="fg3_pct">.338</td><td class="right non_qual" data-stat="fg2_per_g">0.4</td><td class="right non_qual" data-stat="fg2a_per_g">1.1</td><td class="right non_qual" data-stat="fg2_pct">.361</td><td class="right non_qual" data-stat="efg_pct">.459</td><td class="right non_qual" data-stat="ft_per_g">0.2</td><td class="right non_qual" data-stat="fta_per_g">0.3</td><td class="right non_qual" data-stat="ft_pct">.778</td><td class="right non_qual" data-stat="orb_per_g">0.3</td><td class="right non_qual" data-stat="drb_per_g">1.4</td><td class="right non_qual" data-stat="trb_per_g">1.8</td><td class="right non_qual" data-stat="ast_per_g">1.9</td><td class="right non_qual" data-stat="stl_per_g">0.4</td><td class="right non_qual" data-stat="blk_per_g">0.1</td><td class="right non_qual" data-stat="tov_per_g">0.8</td><td class="right non_qual" data-stat="pf_per_g">1.3</td><td class="right non_qual" data-stat="pts_per_g">3.2</td></tr>,
 <tr class="full_table"><th class="right " csk="4" data-stat="ranker" scope="row">4</th><td class="left " csk="Adams,Steven" data-append-csv="adamsst01"
 '''

#main scrape
bs = BeautifulSoup(txt, 'lxml')

#you may uncomment the following three lines to scrape directly from your url, the print results will be different
#url = 'https://www.basketball-reference.com/leagues/NBA_2019_per_game.html'
#html = requests.get(url)
#bs = BeautifulSoup(html.content, 'lxml')

tr = bs.find_all('tr')

#data-append-csv is part of <td class='left', ..., data-append-csv=...>
dacsv = [_.find('td', {'class':'left'})['data-append-csv'] if _.find('td') is not None else None for _ in tr]

#href is part of <a href=...>
href = [_.find('a')['href'] if _.find('a') is not None else None for _ in tr]

print(list(zip(dacsv, href)))

#[('abrinal01', '/players/a/abrinal01.html'), ('acyqu01', '/players/a/acyqu01.html'), ('adamsja01', '/players/a/adamsja01.html'), ('adamsst01', None)]

Note: If you want to see all attributes from an ID, you can do the following (then call the attribute you want)

temp = [_.find('td', {'class':'left'}).attrs if _.find('td') is not None else None for _ in tr]

print(temp)
#[{'class': ['left'], 'csk': 'Abrines,Álex', 'data-append-csv': 'abrinal01', 'data-stat': 'player'}, {'class': ['left'], 'csk': 'Acy,Quincy', 'data-append-csv': 'acyqu01', 'data-stat': 'player'}, {'class': ['left'], 'csk': 'Adams,Jaylen', 'data-append-csv': 'adamsja01', 'data-stat': 'player'}, {'class': ['left'], 'csk': 'Adams,Steven', 'data-append-csv': 'adamsst01'}]

Answer 3

If you simply want to extract those data_append_csv and href you could zip the two matched lists then extract in loop. I would investigate whether, with full html, you can remove the .left class selector.

from bs4 import BeautifulSoup as bs

html = '''
<html>
 <head></head>
 <body>
  <table> 
   <tbody>
    <tr> 
     <th aria-label="Personal Fouls Per Game" class=" poptip hide_non_quals center" data-stat="pf_per_g" data-tip="Personal Fouls Per Game" scope="col">PF</th> 
     <th aria-label="Points Per Game" class=" poptip hide_non_quals center" data-stat="pts_per_g" data-tip="Points Per Game" scope="col">PTS</th> 
    </tr> 
    <tr class="full_table"> 
     <th class="right " csk="1" data-stat="ranker" scope="row">1</th> 
     <td class="left " csk="Abrines,Álex" data-append-csv="abrinal01" data-stat="player"><a href="/players/a/abrinal01.html">Álex Abrines</a></td> 
     <td class="center " data-stat="pos">SG</td> 
     <td class="right " data-stat="age">25</td> 
     <td class="left " data-stat="team_id"><a href="/teams/OKC/2019.html">OKC</a></td> 
     <td class="right " data-stat="g">31</td> 
     <td class="right " data-stat="gs">2</td> 
     <td class="right non_qual" data-stat="mp_per_g">19.0</td> 
     <td class="right non_qual" data-stat="fg_per_g">1.8</td> 
     <td class="right non_qual" data-stat="fga_per_g">5.1</td> 
     <td class="right non_qual" data-stat="fg_pct">.357</td> 
     <td class="right non_qual" data-stat="fg3_per_g">1.3</td> 
     <td class="right non_qual" data-stat="fg3a_per_g">4.1</td> 
     <td class="right non_qual" data-stat="fg3_pct">.323</td> 
     <td class="right non_qual" data-stat="fg2_per_g">0.5</td> 
     <td class="right non_qual" data-stat="fg2a_per_g">1.0</td> 
     <td class="right non_qual" data-stat="fg2_pct">.500</td> 
     <td class="right non_qual" data-stat="efg_pct">.487</td> 
     <td class="right non_qual" data-stat="ft_per_g">0.4</td> 
     <td class="right non_qual" data-stat="fta_per_g">0.4</td> 
     <td class="right non_qual" data-stat="ft_pct">.923</td> 
     <td class="right non_qual" data-stat="orb_per_g">0.2</td> 
     <td class="right non_qual" data-stat="drb_per_g">1.4</td> 
     <td class="right non_qual" data-stat="trb_per_g">1.5</td> 
     <td class="right non_qual" data-stat="ast_per_g">0.6</td> 
     <td class="right non_qual" data-stat="stl_per_g">0.5</td> 
     <td class="right non_qual" data-stat="blk_per_g">0.2</td> 
     <td class="right non_qual" data-stat="tov_per_g">0.5</td> 
     <td class="right non_qual" data-stat="pf_per_g">1.7</td> 
     <td class="right non_qual" data-stat="pts_per_g">5.3</td> 
    </tr> 
    <tr class="full_table"> 
     <th class="right " csk="2" data-stat="ranker" scope="row">2</th> 
     <td class="left " csk="Acy,Quincy" data-append-csv="acyqu01" data-stat="player"><a href="/players/a/acyqu01.html">Quincy Acy</a></td> 
     <td class="center " data-stat="pos">PF</td> 
     <td class="right " data-stat="age">28</td> 
     <td class="left " data-stat="team_id"><a href="/teams/PHO/2019.html">PHO</a></td> 
     <td class="right " data-stat="g">10</td> 
     <td class="right iz" data-stat="gs">0</td> 
     <td class="right non_qual" data-stat="mp_per_g">12.3</td> 
     <td class="right non_qual" data-stat="fg_per_g">0.4</td> 
     <td class="right non_qual" data-stat="fga_per_g">1.8</td> 
     <td class="right non_qual" data-stat="fg_pct">.222</td> 
     <td class="right non_qual" data-stat="fg3_per_g">0.2</td> 
     <td class="right non_qual" data-stat="fg3a_per_g">1.5</td> 
     <td class="right non_qual" data-stat="fg3_pct">.133</td> 
     <td class="right non_qual" data-stat="fg2_per_g">0.2</td> 
     <td class="right non_qual" data-stat="fg2a_per_g">0.3</td> 
     <td class="right non_qual" data-stat="fg2_pct">.667</td> 
     <td class="right non_qual" data-stat="efg_pct">.278</td> 
     <td class="right non_qual" data-stat="ft_per_g">0.7</td> 
     <td class="right non_qual" data-stat="fta_per_g">1.0</td> 
     <td class="right non_qual" data-stat="ft_pct">.700</td> 
     <td class="right non_qual" data-stat="orb_per_g">0.3</td> 
     <td class="right non_qual" data-stat="drb_per_g">2.2</td> 
     <td class="right non_qual" data-stat="trb_per_g">2.5</td> 
     <td class="right non_qual" data-stat="ast_per_g">0.8</td> 
     <td class="right non_qual" data-stat="stl_per_g">0.1</td> 
     <td class="right non_qual" data-stat="blk_per_g">0.4</td> 
     <td class="right non_qual" data-stat="tov_per_g">0.4</td> 
     <td class="right non_qual" data-stat="pf_per_g">2.4</td> 
     <td class="right non_qual" data-stat="pts_per_g">1.7</td> 
    </tr> 
    <tr class="full_table"> 
     <th class="right " csk="3" data-stat="ranker" scope="row">3</th> 
     <td class="left " csk="Adams,Jaylen" data-append-csv="adamsja01" data-stat="player"><a href="/players/a/adamsja01.html">Jaylen Adams</a></td> 
     <td class="center " data-stat="pos">PG</td> 
     <td class="right " data-stat="age">22</td> 
     <td class="left " data-stat="team_id"><a href="/teams/ATL/2019.html">ATL</a></td> 
     <td class="right " data-stat="g">34</td> 
     <td class="right " data-stat="gs">1</td> 
     <td class="right non_qual" data-stat="mp_per_g">12.6</td> 
     <td class="right non_qual" data-stat="fg_per_g">1.1</td> 
     <td class="right non_qual" data-stat="fga_per_g">3.2</td> 
     <td class="right non_qual" data-stat="fg_pct">.345</td> 
     <td class="right non_qual" data-stat="fg3_per_g">0.7</td> 
     <td class="right non_qual" data-stat="fg3a_per_g">2.2</td> 
     <td class="right non_qual" data-stat="fg3_pct">.338</td> 
     <td class="right non_qual" data-stat="fg2_per_g">0.4</td> 
     <td class="right non_qual" data-stat="fg2a_per_g">1.1</td> 
     <td class="right non_qual" data-stat="fg2_pct">.361</td> 
     <td class="right non_qual" data-stat="efg_pct">.459</td> 
     <td class="right non_qual" data-stat="ft_per_g">0.2</td> 
     <td class="right non_qual" data-stat="fta_per_g">0.3</td> 
     <td class="right non_qual" data-stat="ft_pct">.778</td> 
     <td class="right non_qual" data-stat="orb_per_g">0.3</td> 
     <td class="right non_qual" data-stat="drb_per_g">1.4</td> 
     <td class="right non_qual" data-stat="trb_per_g">1.8</td> 
     <td class="right non_qual" data-stat="ast_per_g">1.9</td> 
     <td class="right non_qual" data-stat="stl_per_g">0.4</td> 
     <td class="right non_qual" data-stat="blk_per_g">0.1</td> 
     <td class="right non_qual" data-stat="tov_per_g">0.8</td> 
     <td class="right non_qual" data-stat="pf_per_g">1.3</td> 
     <td class="right non_qual" data-stat="pts_per_g">3.2</td> 
    </tr>
   </tbody>
  </table>
 </body>
</html>
'''
soup = bs(html, 'lxml')

for name, link in zip(soup.select('[data-append-csv].left'),soup.select('[data-append-csv].left a')): #you may wish to add td in
    print(name['data-append-csv'], link['href'])