Python 多处理 output 结果

Question

Given a list of data to process and a 64-core CPU (plus 500 GB RAM). The list should sort strings and store data in a result set of millions of records, which runs just fine, takes a few seconds with multiprocessing. But I'd also need to store the result somehow, either in a txt, csv output or a database. So far I haven't found a viable solution, because after the first part (process), the insert method either gives an error with trying it with MySQL pooling, or takes an insanely long time giving the txt output. What Ive tried so far: simple txt output, print out to txt file, using csv, pandas and numpy libs. Nothing seems to speed it up. Any help would be greatly appreciated: My code right now:

import os
import re
import datetime
import time
import csv

import mysql.connector as connector
from mysql.connector.pooling import MySQLConnectionPool

import mysql

import numpy as np
from tqdm import tqdm
from time import sleep
import multiprocessing as mp

import numpy

pool = MySQLConnectionPool( pool_name="sql_pool",
                            pool_size=32,
                            pool_reset_session=True,
                            host="localhost",
                            port="3306",
                            user="homestead",
                            password="secret",
                            database="homestead")

# # sql connection
db = mysql.connector.connect(
  host="localhost",
  port="3306",
  user="homestead",
  password="secret",
  database="homestead"
)

sql_cursor = db.cursor()
delete_statement = "DELETE FROM statistics"
sql_cursor.execute(delete_statement)

db.commit()

sql_statement = "INSERT INTO statistics (name, cnt) VALUES (%s, %s)"

list = []
domains = mp.Manager().list()
unique_list = mp.Manager().list()
invalid_emails = mp.Manager().list()
result = mp.Manager().list()
regex_email = '^(\w|\.|\_|\-)+[@](\w|\_|\-|\.)+[.]\w{2,3}$'

# check email validity
def check(list, email):
    if(re.search(regex_email, email)):
        domains.append(email.lower().split('@')[1])
        return True
    else:
        invalid_emails.append(email)
        return False
#end of check email validity

# execution time converter
def convertTime(seconds):
    seconds = seconds % (24 * 3600)
    hour = seconds // 3600
    seconds %= 3600
    minutes = seconds // 60
    seconds %= 60

    if(hour == 0):
        if(minutes == 0):
            return "{0} sec".format(seconds)
        else:
            return "{0}min {1}sec".format(minutes, seconds)
    else:
        return "{0}hr {1}min {2}sec".format(hour, minutes, seconds)
# execution time converter end

#process
def process(list):
    for item in tqdm(list):
        if(check(list, item)):
            item = item.lower().split('@')[1]
            if item not in unique_list:
                unique_list.append(item)
# end of process

def insert(list):
    global sql_statement

    # Add to db
    con = pool.get_connection()
    cur = con.cursor()

    print("PID %d: using connection %s" % (os.getpid(), con))
    #cur.executemany(sql_statement, sorted(map(set_result, list)))
    for item in list:

        cur.execute(sql_statement, (item, domains.count(item)))
    con.commit()
    cur.close()
    con.close()

# def insert_into_database(list):
    #sql_cursor.execute(sql_statement, (unique_list, 1), multi=True)

    # sql_cursor.executemany(sql_statement, sorted(map(set_result, list)))
    # db.commit()

# statistics
def statistics(list):
    for item in tqdm(list):
        if(domains.count(item) > 0):
            result.append([domains.count(item), item])
# end of statistics

params = sys.argv
filename = ''
process_count = -1
for i, item in enumerate(params):
    if(item.endswith('.txt')):
        filename = item
    if(item == '--top'):
        process_count = int(params[i+1])


def set_result(item):
    return item, domains.count(item)

# main
if(filename):
    try:
        start_time = time.time()
        now = datetime.datetime.now()
        dirname = "email_stats_{0}".format(now.strftime("%Y%m%d_%H%M%S"))
        os.mkdir(dirname)

        list = open(filename).read().split()

        if(process_count == -1):
            process_count = len(list)

        if(process_count > 0):
            list = list[:process_count]

        #chunking list
        n = int(len(list) /  mp.cpu_count())
        chunks = [list[i:i + n] for i in range(0, len(list), n)]

        processes = []
        print('Processing list on {0} cores...'.format(mp.cpu_count()))
        for chunk in chunks:
            p = mp.Process(target=process, args=[chunk])
            p.start()
            processes.append(p)

        for p in processes:
            p.join()

        # insert(unique_list)

        ## step 2 - write sql

        ##  Clearing out db before new data insert
        con = pool.get_connection()
        cur = con.cursor()

        delete_statement = "DELETE FROM statistics"
        cur.execute(delete_statement)

        u_processes = []

        #Maximum pool size for sql is 32, so maximum chunk number should be that too.
        if(mp.cpu_count() < 32):
            n2 = int(len(unique_list) /  mp.cpu_count())
        else:
            n2 = int(len(unique_list) /  32)

        u_chunks = [unique_list[i:i + n2] for i in range(0, len(unique_list), n2)]
        for u_chunk in u_chunks:
            p = mp.Process(target=insert, args=[u_chunk])
            p.start()
            u_processes.append(p)

        for p in u_processes:
            p.join()

        for p in u_processes:
            p.close()

        # sql_cursor.executemany(sql_statement, sorted(map(set_result, unique_list)))
        # db.commit()
        # for item in tqdm(unique_list):
        #     sql_val = (item, domains.count(item))
        #     sql_cursor.execute(sql_statement, sql_val)
        #
        #     db.commit()

        ## numpy.savetxt('saved.txt', sorted(map(set_result, unique_list)), fmt='%s')


        # with(mp.Pool(mp.cpu_count(), initializer = db) as Pool:
        #     Pool.map_async(insert_into_database(),set(unique_list))
        #     Pool.close()
        #     Pool.join()

        print('Creating statistics for {0} individual domains...'.format(len(unique_list)))

        # unique_list = set(unique_list)
        # with open("{0}/result.txt".format(dirname), "w+") as f:
        #     csv.writer(f).writerows(sorted(map(set_result, unique_list), reverse=True))

        print('Writing final statistics...')
        print('OK.')
        f = open("{0}/stat.txt".format(dirname),"w+")
        f.write("Number of processed emails: {0}\r\n".format(process_count))
        f.write("Number of valid emails: {0}\r\n".format(len(list) - len(invalid_emails)))
        f.write("Number of invalid emails: {0}\r\n".format(len(invalid_emails)))
        f.write("Execution time: {0}".format(convertTime(int(time.time() - start_time))))
        f.close()

    except FileNotFoundError:
        print('File not found, path or file broken.')
else:
    print('Wrong file format, should be a txt file.')
# main

Answer 1

See my comments regarding some changes you might wish to make, one of which might improve performance. But I think one area of performance which could really be improved is in your use of managed lists. These are represented by proxies and each operation on such a list is essentially a remote procedure call and thus very slow. You cannot avoid this given that you need to have multiple processes updating a common, shared lists (or dict if you take my suggestion). But in the main process you might be trying, for example, to construct a set from a shared list as follows:

Pool.map_async(insert_into_database(),set(unique_list))

(by the way, that should be Pool.map(insert_into_database, set(unique_list)) , ie you have an extra set of () and you can then get rid of the calls to pool.close() and pool.join() if you wish)

The problem is that you are iterating every element of unique_list through a proxy, which might be what is taking a very long time. I say "might" because I would think the use of managed lists would prevent the code as is, ie without outputting the results, from completing in "a few seconds" if we are talking about "millions" of records and thus millions of remote procedure calls. But this number could certainly be reduced if you could somehow get the underlying list as a native list.

First, you need to heed my comment about having declared a variable named list thus making it impossible to create native lists or subclasses of list . Once your have renamed that variable to something more reasonable, we can create our own managed class MyList that will expose the underlying list on which it is built. Note that you can do the same thing with a MyDict class that subclasses dict . I have defined both classes for you. Here is a benchmark showing the difference between constructing a native list from a managed list versus creating a native list from a MyList :

import multiprocessing as mp
from multiprocessing.managers import BaseManager
import time

class MyManager(BaseManager):
    pass

class MyList(list):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def get_underlying_list(self):
        return self

class MyDict(dict):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def get_underlying_dict(self):
        return self

# required for windows, which I am running on:
if __name__ == '__main__':
    l = mp.Manager().list()
    for i in range(100_000):
        l.append(i)
    t = time.time()
    l2 = list(l)
    print(time.time() - t, l2[0:5], l2[-5:])


    MyManager.register('MyList', MyList)
    MyManager.register('MyDict', MyDict)
    my_manager = MyManager()
    # must explicitly start the manager or use: with MyManager() as manager:
    my_manager.start()
    l = my_manager.MyList()
    for i in range(100_000):
        l.append(i)
    t = time.time()
    l2 = list(l.get_underlying_list())
    print(time.time() - t, l2[0:5], l2[-5:])

Prints:

7.3949973583221436 [0, 1, 2, 3, 4] [99995, 99996, 99997, 99998, 99999]
0.007997751235961914 [0, 1, 2, 3, 4] [99995, 99996, 99997, 99998, 99999]