Read column from a web page table

Question

I want to be able to read only the column of "Daily solar radiation - horizontal" from the NASA web page. How do I go about it? Here is my code:

# Horizontal radiation values (kwh/m**2)
import urllib.parse
import html5lib
import pandas as pd

url = "https://eosweb.larc.nasa.gov/cgi-bin/sse/retscreen.cgi?email=rets%40nrcan.gc.ca&step=1&lat=49.4&lon=7.3&submit=Submit"

params = {'lat':a,'lon':b}

url_parts = list(urllib.parse.urlparse(url))

query = dict(urllib.parse.parse_qsl(url_parts[4]))

query.update(params)

url_parts[4] = urllib.parse.urlencode(query)

print(urllib.parse.urlunparse(url_parts))

webresult = pd.read_html(urllib.parse.urlunparse(url_parts))
webresult[3]

It only displays the full table.

Answer 1

This can be done easily using BeautifulSoup . Explanation is given in the code comments.

import bs4, requests

def getColumn(url):
    # get the page
    resp = requests.get(url)

    # create a BeautifulSoup object that represents the page
    # and use lxml parser to parse the html
    soup = bs4.BeautifulSoup(resp.text, 'lxml')

    # get all the tables in the page
    tables= soup.findAll('table')

    # all data of interest will be collected here
    data = []

    #we only want to process the 4th table, so we store it in table
    table = tables[3]

    # for each row in this table, get the 4th column and add it in data
    for row in table.findAll('tr'):
        row_data= row.findAll('td')

        if not row_data: continue    #skip empty lists

        column4= row.findAll('td')[3].string    # read the 4th column

        data.append(column4)

    # data is in string so we need to convert it to float

    # discard the first and last two elements in the list (we don't want them)
    # then convert the remaining from string to float
    data = [ float(x.strip()) for x in data[1:-2]]

    return data


def main():
    url= 'https://eosweb.larc.nasa.gov/cgi-bin/sse/retscreen.cgi?email=rets%40nrcan.gc.ca&step=1&lat=49.4&lon=7.3&submit=Submit'
    lst = getColumn(url)

    print(lst)

if __name__ == '__main__':
    main()