Python requests.get() loop returns nothing

Question

When trying to scrape multiple pages of this website, I get no content in return. I usually check to make sure all the lists I'm creating are of equal length, but all are coming back as len = 0 .

I've used similar code to scrape other websites, so why does this code not work correctly?

Some solutions I've tried, but haven't worked for my purposes: requests.Session() solutions as suggested in this answer , .json as suggested here .

for page in range(100, 350):

    page = requests.get("https://www.ghanaweb.com/GhanaHomePage/election2012/parliament.constituency.php?ID=" + str(page) + "&res=pm")

    page.encoding = page.apparent_encoding

    if not page:
        pass

    else:

        soup = BeautifulSoup(page.text, 'html.parser')

        ghana_tbody = soup.find_all('tbody')

        sleep(randint(2,10))

         for container in ghana_tbody:

            #### CANDIDATES ####
            candidate = container.find_all('div', class_='can par')
            for data in candidate:
                cand = data.find('h4')
                for info in cand:
                    if cand is not None:
                        can2 = info.get_text()
                        can.append(can2)

            #### PARTY NAMES ####
            partyn = container.find_all('h5')
            for data in partyn:
                if partyn is not None:
                    partyn2 = data.get_text()
                    pty_n.append(partyn2)

            #### CANDIDATE VOTES ####
            votec = container.find_all('td', class_='votes')
            for data in votec:
                if votec is not None:
                    votec2 = data.get_text()
                    cv1.append(votec2)

            #### CANDIDATE VOTE SHARE ####
            cansh = container.find_all('td', class_='percent')
            for data in cansh:
                if cansh is not None:
                    cansh2 = data.get_text()
                    cvs1.append(cansh2)

        #### TOTAL  VOTES ####`
        tfoot = soup.find_all('tr', class_='total')
        for footer in tfoot:
            fvote = footer.find_all('td', class_='votes')
            for data in fvote:
                if fvote is not None:
                    fvote2 = data.get_text()
                    fvoteindiv = [fvote2]
                    fvotelist = fvoteindiv * (len(pty_n) - len(vot1))
                    vot1.extend(fvotelist)

Thanks in advance for your help!

Answer 1

I've made some simplification changes. The major changes that needed to be changed were:

1. ghana_tbody = soup.find_all('table', class_='canResults')
2. can2 = info # not info.get_text()

I have only tested this against page 112; life is too short.

import requests
from bs4 import BeautifulSoup
from random import randint
from time import sleep

can = []
pty_n = []
cv1 = []
cvs1 = []
vot1 = []

START_PAGE = 112
END_PAGE = 112

for page in range(START_PAGE, END_PAGE + 1):
    page = requests.get("https://www.ghanaweb.com/GhanaHomePage/election2012/parliament.constituency.php?ID=112&res=pm")
    page.encoding = page.apparent_encoding
    if not page:
        pass
    else:
        soup = BeautifulSoup(page.text, 'html.parser')
        ghana_tbody = soup.find_all('table', class_='canResults')
        sleep(randint(2,10))
        for container in ghana_tbody:

            #### CANDIDATES ####
            candidate = container.find_all('div', class_='can par')
            for data in candidate:
                cand = data.find('h4')
                for info in cand:
                    can2 = info # not info.get_text()
                    can.append(can2)

            #### PARTY NAMES ####
            partyn = container.find_all('h5')
            for data in partyn:
                partyn2 = data.get_text()
                pty_n.append(partyn2)


            #### CANDIDATE VOTES ####
            votec = container.find_all('td', class_='votes')
            for data in votec:
                votec2 = data.get_text()
                cv1.append(votec2)

            #### CANDIDATE VOTE SHARE ####
            cansh = container.find_all('td', class_='percent')
            for data in cansh:
                cansh2 = data.get_text()
                cvs1.append(cansh2)

        #### TOTAL  VOTES ####`
        tfoot = soup.find_all('tr', class_='total')
        for footer in tfoot:
            fvote = footer.find_all('td', class_='votes')
            for data in fvote:
                fvote2 = data.get_text()
                fvoteindiv = [fvote2]
                fvotelist = fvoteindiv * (len(pty_n) - len(vot1))
                vot1.extend(fvotelist)

print('can = ', can)
print('pty_n = ', pty_n)
print('cv1 = ', cv1)
print('cvs1 = ', cvs1)
print('vot1 = ', vot1)

Prints:

can =  ['Kwadwo Baah Agyemang', 'Daniel Osei', 'Anyang - Kusi Samuel', 'Mary Awusi']
pty_n =  ['NPP', 'NDC', 'IND', 'IND']
cv1 =  ['14,966', '9,709', '8,648', '969', '34292']
cvs1 =  ['43.64', '28.31', '25.22', '2.83', '\xa0']
vot1 =  ['34292', '34292', '34292', '34292']

Be sure to first change START_PAGE and END_PAGE to 100 and 350 respecively.