谷歌 API 多处理

Question

我正在尝试从我的 Gmail 帐户（主题、发件人、日期、邮件正文）下的电子邮件中获取特定信息，并且能够使用 Google API 和相关库成功地做到这一点，但是，我注意到您拥有的电子邮件越多解析所需的时间越长，以至于解析 34 封电子邮件需要将近 15 秒，如果您试图将其扩展到解析 1000 封电子邮件，这就很糟糕了。 我的目标是在parse_messages() function 上利用并发/多处理，但是，我没有运气，一直返回一个空列表。 目的是处理所有电子邮件，然后 append 将它们全部合并到一个combined列表中。

草率的道歉，它还没有被清理，总共不到100行。

from __future__ import print_function
import os.path
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from concurrent.futures import ProcessPoolExecutor
import base64
import re

combined = []

def authenticate():
    # If modifying these scopes, delete the file token.json.
    SCOPES = ['https://www.googleapis.com/auth/gmail.readonly']

    creds = None

    if os.path.exists('token.json'):
        creds = Credentials.from_authorized_user_file('token.json', SCOPES)

    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                'creds.json', SCOPES)
            creds = flow.run_local_server(port=0)

        with open('token.json', 'w') as token:
            token.write(creds.to_json())
    return creds

def get_messages(creds):
    # Get the messages
    days = 31
    service = build('gmail', 'v1', credentials=creds)
    results = service.users().messages().list(userId='me', q=f'newer_than:{days}d, in:inbox').execute()
    messages = results.get('messages', [])
    message_count = len(messages)
    print(f"You've received {message_count} email(s) in the last {days} days")
    if not messages:
        print(f'No Emails found in the last {days} days.')
    return messages


def parse_message(msg):
    # Call the Gmail API
    service = build('gmail', 'v1', credentials=creds)
    txt = service.users().messages().get(userId='me', id=msg['id']).execute()
    payload = txt['payload']
    headers = payload['headers']

    #Grab the Subject Line, From and Date from the Email
    for d in headers:
        if d['name'] == 'Subject':
            subject = d['value']
        if d['name'] == 'From':
            sender = d['value']
            try:
                match = re.search(r'<(.*)>', sender).group(1)
            except:
                match = sender
        if d['name'] == "Date":
            date_received = d['value']

    def get_body(payload):
        if 'body' in payload and 'data' in payload['body']:
            return payload['body']['data']
        elif 'parts' in payload:
            for part in payload['parts']:
                data = get_body(part)
                if data:
                    return data
        else:
            return None

    data = get_body(payload)

    data = data.replace("-","+").replace("_","/")
    decoded_data = base64.b64decode(data).decode("UTF-8")
    decoded_data = (decoded_data.encode('ascii', 'ignore')).decode("UTF-8")
    decoded_data = decoded_data.replace('\n','').replace('\r','').replace('\t', '')

    # Append parsed message to shared list
    return combined.append([date_received, subject, match, decoded_data])

if __name__ == '__main__':
    creds = authenticate()
    messages = get_messages(creds)
    # Create a process pool with 4 worker processes
    with ProcessPoolExecutor(max_workers=4) as executor:
        # Submit the parse_message function for each message in the messages variable
        executor.map(parse_message, messages)
   
    print(f"Combined: {combined}")

运行脚本时，我的output是正常的。

You've received 34 email(s) in the last 31 days
combined: []

Answer 1

多亏了 simpleApp 的帮助，我和其他一些人一起做出了他们的改变来让它工作。

    # Append parsed message to shared list
    return [date_received, subject, match, decoded_data]

if __name__ == '__main__':
    creds = authenticate()
    messages, service = get_messages(creds)
    # Create a process pool with default worker processes
    with ProcessPoolExecutor() as executor:
        combined = []
        # Submit the parse_message function for each message in the messages variable
        all_pools = executor.map(parse_message, messages, [service]*len(messages))
        for e_p in all_pools:
            combined.append(e_p)