如何对包含多重处理的函数进行多重处理？

Question

我在线上发现了一个非常复杂的函数，我确定该函数使用python multiprocessing包本身。

现在，我有许多系列的数据，我也想对该函数使用多重处理。

这是该函数的简化版本：

import multiprocessing
import numpy as np
def main(number=10):
    print('ok')
    def segment():
        result=[]
        error='meet bad case'
        for i in range(number):
            result.append(i*np.random.rand())
            if result[-1] ==number/2:
                raise AssertionError(error)
        json.dump(result, codecs.open(os.getcwd()+'/data/result'+str(number), \
                'w', encoding='utf-8'), separators=(',', ':'), sort_keys=True, indent=4) 
    p = multiprocessing.Process(target=segment)
    p.daemon = True
    p.start()
    p.join()  

这个主要功能代表了我下载的复杂功能。 现在，如何使用并行多处理技术将该函数应用于10到100范围内的可变数字？

Answer 1

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import multiprocessing
# We must import this explicitly, it is not imported by the top-level
# multiprocessing module.
import multiprocessing.pool
import time

from random import randint


class NoDaemonProcess(multiprocessing.Process):
    # make 'daemon' attribute always return False
    def _get_daemon(self):
        return False
    def _set_daemon(self, value):
        pass
    daemon = property(_get_daemon, _set_daemon)

# We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool
# because the latter is only a wrapper function, not a proper class.
class MyPool(multiprocessing.pool.Pool):
    Process = NoDaemonProcess
def main(number):

    def segment():
        result=[]
        error='meet bad case'
        for i in range(number):
            result.append(i*np.random.rand())
            if result[-1] ==number/2:
                raise AssertionError(error)
        json.dump(result, codecs.open(os.getcwd()+'/data/result'+str(number), \
                'w', encoding='utf-8'), separators=(',', ':'), sort_keys=True, indent=4) 
    p = multiprocessing.Process(target=segment)
    p.start()
    p.join() 
    print('ok')

def test():
    print("Creating 5 (non-daemon) workers and jobs in main process.")
    pool = MyPool(5)

    pool.map(main, [5,6,7])

    pool.close()
    pool.join()

if __name__ == '__main__':
    test()