parsing google news using beautiful soup python

Question

I have python code as below. It searches a google news page and prints hyperlinks and titles for each news. My problem is that googlenews groups news that are similar in one bucket and below script prints only 1st news in each bucket. How can I print all new from all buckets?

from bs4 import BeautifulSoup
import requests

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}

#r = requests.get('http://www.aflcio.org/Legislation-and-Politics/Legislative-Alerts', headers=headers)
r = requests.get('https://www.google.com/search?q=%22eric+bledsoe%22&tbm=nws&tbs=qdr:d', headers=headers)
r = requests.get('https://www.google.com/search?q=%22lebron+james%22&tbm=nws&tbs=qdr:y', headers=headers)

soup = BeautifulSoup(r.text, "html.parser")

letters = soup.find_all("div", class_="_cnc")
#print soup.prettify() 
#print letters
print type(letters)
print len(letters)
print("\n")

for x in range(0, len(letters)):
    print x
    print letters[x].a["href"]


print("\n")

letters2 = soup.find_all("a", class_="l _HId")
for x in range(0, len(letters2)):
    print x
    print letters2[x].get_text()

print ("\n----------content")
#print letters[0]

By bucketing news I mean that in the below image, first few news are grouped together. The news "LeBron James compares one of his teammates to Denn" is part of another group.

Answer 1

I'm not sure what you mean by bucket? If you mean you're trying to parse multiple websites then I can tell you you're overwriting r by sending it multiple news requests.get()

Here's a loop to process all the URLs you have in the urls array.

import bs4
import requests

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}


urls = ["https://www.google.com/search?q=%22eric+bledsoe%22&tbm=nws&tbs=qdr:d",
        "https://www.google.com/search?q=%22lebron+james%22&tbm=nws&tbs=qdr:y"]

ahrefs = []
titles = []

for url in urls:
    req = requests.get(url, headers=headers)
    soup = bs4.BeautifulSoup(req.text, "html.parser")

    #you don't even have to process the div container
    #just go strait to <a> and using indexing get "href"
    #headlines
    ahref  = [a["href"] for a in soup.find_all("a", class_="_HId")]
    #"buckets"
    ahref += [a["href"] for a in soup.find_all("a", class_="_sQb")]
    ahrefs.append(ahref)

    #or get_text() will return the array inside the hyperlink
    #the title you want
    title =  [a.get_text() for a in soup.find_all("a", class_="_HId")]
    title += [a.get_text() for a in soup.find_all("a", class_="_sQb")]
    titles.append(title)

#print(ahrefs)
#print(titles)

My google search for lebron turns up 18 results, including subheadlines, and len(ahrefs[1]) == 18

Answer 2

With a whole new shift to kill I decided to take a more efficient stab at the problem, this way you just have to append queries to search for new players. I'm not sure what you want for an end result but this will return a list of dictionaries.

import bs4
import requests

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}


#just add to this list for each new player
#player name : url
queries = {"bledsoe":"https://www.google.com/search?q=%22eric+bledsoe%22&tbm=nws&tbs=qdr:d",
           "james":"https://www.google.com/search?q=%22lebron+james%22&tbm=nws&tbs=qdr:y"}


total = []

for player in queries: #keys

    #request the google query url of each player
    req  = requests.get(queries[player], headers=headers)
    soup = bs4.BeautifulSoup(req.text, "html.parser")

    #look for the main container
    for each in soup.find_all("div"):
        results = {player: { \
            "link": None,    \
            "title": None,   \
            "source": None,  \
            "time": None}    \
        }

        try:
          #if <div> doesn't have class="anything"
          #it will throw a keyerror, just ignore

          if "_cnc" in each.attrs["class"]: #mainstories
            results[player]["link"] = each.find("a")["href"]
            results[player]["title"] = each.find("a").get_text()
            sourceAndTime = each.contents[1].get_text().split("-")
            results[player]["source"], results[player]["time"] = sourceAndTime
            total.append(results)

          elif "card-section" in each.attrs["class"]: #buckets
            results[player]["link"] = each.find("a")["href"]
            results[player]["title"] = each.find("a").get_text()
            results[player]["source"] = each.contents[1].contents[0].get_text()
            results[player]["time"] = each.contents[1].get_text()
            total.append(results)

        except KeyError:
            pass