writing to second column of a csv file python

Question

I have a Python program that gets the google rating for restaurants after reading the names of the restaurants from a CSV file (has only a single column). The names are in a single column (first one). it gets the rating for each restaurant and appends it to a list. Now i need to make a second column and pass that list into the second column. Here is my CSV file (it doesn't have headers, ie names of columns)

Barrafina london
palomar london
fateema london
five guys london
10 greek street london
edition hotel london

And here is the code:

import requests
from bs4 import BeautifulSoup
import csv


def main():
    global rate
    ratings = []
    res = []
    with open('CSV restaurants example.csv', newline='', encoding='utf-8') as f:
        reader = csv.reader(f)

        for row1 in reader:
            for row in row1:
                res.append(row)
    for restaurant in res:
        try:
            re = requests.get('https://www.google.com/search?q=' + restaurant)
        except requests.exceptions.ChunkedEncodingError:
            re = requests.get('https://www.google.com/search?q=' + restaurant)

        soup = BeautifulSoup(re.content, 'html.parser')

        rating = soup.select_one('.oqSTJd')
        try:
            rate = rating.text
            ratings.append(rate)
        except AttributeError:
            print("google search is asking for captcha, use a proxy or change your wifi network")
            exit()

        print('\nThe rating for ' + restaurant + ' is ' + str(rate) + ' out of 5 ' + '\n')


if __name__ == '__main__':
    main()

Any suggestions on how can i append that list to second column?

Answer 1

Just create a csv.writer to compliment your csv.reader . You want to save the output as a new file, I recommend never overwriting your raw data, even if you are just adding data to it.

I will say that you probably want to be careful about the delimiter, as the location and restaurant fields seem to be merged, that looks like two columns in an ill-formed csv to me.

Here you can solve it in the same for-loop:

import requests
from bs4 import BeautifulSoup
import csv


def main():
    # global rate
    # ratings = []
    # res = []
    with open('CSV restaurants example.csv', newline='', encoding='utf-8') as infd, \
             open('restaurants_with_ratings.csv') as outfd:
        reader = csv.reader(infd)

        # create a csv writer, these are defaults, but I'm showing that 
        # you can choose formatting
        writer = csv.writer(outfd, delimiter=',', quotechar='"')

        # if there's only one column, the comma works here.
        for restaurant, in reader:
            # res.append(row)

            try:
                re = requests.get('https://www.google.com/search?q=' + restaurant)
            except requests.exceptions.ChunkedEncodingError:
                re = requests.get('https://www.google.com/search?q=' + restaurant)

            soup = BeautifulSoup(re.content, 'html.parser')

            rating = soup.select_one('.oqSTJd')
            try:
                rate = rating.text
                writer.writerow([restauant, rate])  # add a column
                # ratings.append(rate)
            except AttributeError:
                print("google search is asking for captcha, use a proxy or change your wifi network")
                exit()

            print('\nThe rating for ' + restaurant + ' is ' + str(rate) + ' out of 5 ' + '\n')


if __name__ == '__main__':
    main()

Answer 2

Try this, it creates a pandas object with a second column of ratings and you can convert it to csv:

import requests
import pandas as pd
from bs4 import BeautifulSoup
import csv


def main():
    global rate
    ratings = []
    res = []
    with open('CSV restaurants example.csv', newline='', encoding='utf-8') as f:
        reader = csv.reader(f)

        for row1 in reader:
            for row in row1:
                res.append(row)
    for restaurant in res:
        try:
            re = requests.get('https://www.google.com/search?q=' + restaurant)
        except requests.exceptions.ChunkedEncodingError:
            re = requests.get('https://www.google.com/search?q=' + restaurant)

        soup = BeautifulSoup(re.content, 'html.parser')

        rating = soup.select_one('.oqSTJd')
        try:
            rate = rating.text
            ratings.append(rate)
        except AttributeError:
            print("google search is asking for captcha, use a proxy or change your wifi network")
            exit()

        print('\nThe rating for ' + restaurant + ' is ' + str(rate) + ' out of 5 ' + '\n')
    df2 = pd.read_csv('CSV restaurants example.csv', header=None, names=['Restaurants']) 
    df2['ratings'] = ratings
    print(df2)
    print("")
    print(df2.to_csv())
    return df2

if __name__ == '__main__':
    df2 = main()

You can rewrite back to csv without headers like this:

df2.set_index("Restaurants").to_csv(header=False)

# 'Barrafina london,4.5\npalomar london,4.6\nfateema london,4.8\nfive guys london,4.2\n10 greek street london,4.5\nedition hotel london,4.6\n'