Google API Multi-Processing

Question

I'm trying to grab specific information from emails under my Gmail account (Subject, From, Date, Message Body) and was able to do so succesfully using the Google API and relevant libraries, however, I've noticed the more emails you have the longer it takes to parse, so much so that parsing 34 emails takes nearly 15 seconds, which is bad if you tried to scale that to parsing 1000 emails. My aim was to utilise concurrency/multi-processing on the parse_messages() function, however, I've had no luck and keep returning an empty list. The aim is to process all the emails, then append them all to a the combined list.

Apologies for the sloppyness, it's yet to be cleaned up, there's less than 100 lines in total.

from __future__ import print_function
import os.path
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from concurrent.futures import ProcessPoolExecutor
import base64
import re

combined = []

def authenticate():
    # If modifying these scopes, delete the file token.json.
    SCOPES = ['https://www.googleapis.com/auth/gmail.readonly']

    creds = None

    if os.path.exists('token.json'):
        creds = Credentials.from_authorized_user_file('token.json', SCOPES)

    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                'creds.json', SCOPES)
            creds = flow.run_local_server(port=0)

        with open('token.json', 'w') as token:
            token.write(creds.to_json())
    return creds

def get_messages(creds):
    # Get the messages
    days = 31
    service = build('gmail', 'v1', credentials=creds)
    results = service.users().messages().list(userId='me', q=f'newer_than:{days}d, in:inbox').execute()
    messages = results.get('messages', [])
    message_count = len(messages)
    print(f"You've received {message_count} email(s) in the last {days} days")
    if not messages:
        print(f'No Emails found in the last {days} days.')
    return messages


def parse_message(msg):
    # Call the Gmail API
    service = build('gmail', 'v1', credentials=creds)
    txt = service.users().messages().get(userId='me', id=msg['id']).execute()
    payload = txt['payload']
    headers = payload['headers']

    #Grab the Subject Line, From and Date from the Email
    for d in headers:
        if d['name'] == 'Subject':
            subject = d['value']
        if d['name'] == 'From':
            sender = d['value']
            try:
                match = re.search(r'<(.*)>', sender).group(1)
            except:
                match = sender
        if d['name'] == "Date":
            date_received = d['value']

    def get_body(payload):
        if 'body' in payload and 'data' in payload['body']:
            return payload['body']['data']
        elif 'parts' in payload:
            for part in payload['parts']:
                data = get_body(part)
                if data:
                    return data
        else:
            return None

    data = get_body(payload)

    data = data.replace("-","+").replace("_","/")
    decoded_data = base64.b64decode(data).decode("UTF-8")
    decoded_data = (decoded_data.encode('ascii', 'ignore')).decode("UTF-8")
    decoded_data = decoded_data.replace('\n','').replace('\r','').replace('\t', '')

    # Append parsed message to shared list
    return combined.append([date_received, subject, match, decoded_data])

if __name__ == '__main__':
    creds = authenticate()
    messages = get_messages(creds)
    # Create a process pool with 4 worker processes
    with ProcessPoolExecutor(max_workers=4) as executor:
        # Submit the parse_message function for each message in the messages variable
        executor.map(parse_message, messages)
   
    print(f"Combined: {combined}")

When running the script, my output is normally.

You've received 34 email(s) in the last 31 days
combined: []

Answer 1

Thanks to the help of simpleApp, I made their changes along with a few others to get this working.

    # Append parsed message to shared list
    return [date_received, subject, match, decoded_data]

if __name__ == '__main__':
    creds = authenticate()
    messages, service = get_messages(creds)
    # Create a process pool with default worker processes
    with ProcessPoolExecutor() as executor:
        combined = []
        # Submit the parse_message function for each message in the messages variable
        all_pools = executor.map(parse_message, messages, [service]*len(messages))
        for e_p in all_pools:
            combined.append(e_p)