Python 多处理：如何从池中的池返回结果？

Question

所以我稍微改变了这段代码以返回值，因为我面临着类似的问题，而不是从外部池返回结果，而是立即返回生成器。

以下是更新后的代码：

from concurrent.futures import ProcessPoolExecutor as Pool

num_pool = 10


def main_pool(num):
    numbers = []
    print('MAIN POOL', num)
    strings_write = (f'{num}-{i}' for i in range(num))
    with Pool(num) as subp:
        numbers.extend(subp.map(sub_pool, strings_write))
    return numbers


def sub_pool(x):
    print('sub_pool', x)
    print(f'{x}')
    return x


if __name__ == "__main__":
    with Pool(num_pool) as p:
        result = p.map(main_pool, list(range(1, num_pool + 1)))
        print('Result Outside')
        print(result)

output 是：

Result Outside
<generator object _chain_from_iterable_of_lists at 0x7faafbcd2b88>
MAIN POOL 1
MAIN POOL 2
MAIN POOL 3
MAIN POOL 4
MAIN POOL 5
MAIN POOL 6
MAIN POOL 7
MAIN POOL 8
MAIN POOL 9
sub_pool 1-0
1-0
MAIN POOL 10
sub_pool 2-0
2-0
sub_pool 2-1
2-1
sub_pool 3-0
3-0
sub_pool 4-0
4-0
sub_pool 3-1
3-1
sub_pool 3-2
3-2
sub_pool 4-2
4-2
sub_pool 4-1
4-1
sub_pool 4-3
4-3
sub_pool 5-0
5-0
sub_pool 5-1
5-1
sub_pool 6-0
6-0
sub_pool 5-2
5-2
sub_pool 7-0
7-0
sub_pool 6-1
6-1
sub_pool 5-3
5-3
sub_pool 7-1
7-1
sub_pool 6-2
6-2
sub_pool 7-2
7-2
sub_pool 5-4
5-4
sub_pool 6-3
6-3
sub_pool 7-3
7-3
sub_pool 6-4
6-4
sub_pool 7-4
7-4
sub_pool 6-5
6-5
sub_pool 7-5
7-5
sub_pool 7-6
7-6
sub_pool 9-0
9-0
sub_pool 8-0
8-0
sub_pool 9-1
9-1
sub_pool 10-0
10-0
sub_pool 8-1
8-1
sub_pool 9-2
9-2
sub_pool 10-1
10-1
sub_pool 8-2
8-2
sub_pool 9-3
9-3
sub_pool 10-2
10-2
sub_pool 9-4
9-4
sub_pool 8-3
8-3
sub_pool 10-3
10-3
sub_pool 9-5
9-5
sub_pool 8-4
8-4
sub_pool 10-4
10-4
sub_pool 9-6
9-6
sub_pool 8-5
8-5
sub_pool 10-5
10-5
sub_pool 9-7
9-7
sub_pool 8-6
8-6
sub_pool 10-6
10-6
sub_pool 9-8
9-8
sub_pool 8-7
8-7
sub_pool 10-7
10-7
sub_pool 10-8
10-8
sub_pool 10-9
10-9

Answer 1

如果您阅读了concurrent.futures.Executor.map的文档，您会说“与map类似，除了可立即收集而不是延迟收集可迭代对象，并且func是异步执行的，并且可以同时对func进行多次调用。”

换句话说，它返回一个生成器，就像内置的map function 一样。 我通过您有些含糊的帖子假设您希望返回除生成器之外的其他东西。 如果是这样，您希望将生成器显式转换为列表：

from concurrent.futures import ProcessPoolExecutor as Pool

num_pool = 10


def main_pool(num):
    numbers = []
    print('MAIN POOL', num)
    strings_write = (f'{num}-{i}' for i in range(num))
    with Pool(num) as subp:
        # Note that list.extend will iterate the generator returned by
        # map, so there is no need to first convert the return value
        # from map into a list:
        numbers.extend(subp.map(sub_pool, strings_write))
    return numbers


def sub_pool(x):
    print('sub_pool', x)
    print(f'{x}')
    return x


if __name__ == "__main__":
    with Pool(num_pool) as p:
        # Convert returned generator to a list:
        result = list(p.map(main_pool, list(range(1, num_pool + 1))))
        print('Result Outside')
        print(result)

印刷：

MAIN POOL 1
MAIN POOL 2
MAIN POOL 4
MAIN POOL 5
MAIN POOL 3
MAIN POOL 6
MAIN POOL 7
MAIN POOL 8
MAIN POOL 9
MAIN POOL 10
sub_pool 7-0
7-0
sub_pool 7-1
7-1
sub_pool 7-2
7-2
sub_pool 1-0
1-0
sub_pool 7-3
7-3
sub_pool 7-4
7-4
sub_pool 7-5
7-5
sub_pool 7-6
7-6
sub_pool 4-0
4-0
sub_pool 4-1
4-1
sub_pool 3-0
3-0
sub_pool 3-1
sub_pool 4-2
sub_pool 4-3
sub_pool 8-0
8-0
sub_pool 8-1
8-1
sub_pool 8-2
4-2
sub_pool 5-1
4-3
sub_pool 5-0
sub_pool 6-0
sub_pool 8-3
8-3
sub_pool 8-4
5-1
5-0
6-0
3-1
sub_pool 5-2
5-2
8-4
sub_pool 8-5
sub_pool 6-1
sub_pool 5-4
sub_pool 3-2
8-2
sub_pool 8-6
8-6
8-5
6-1
5-4
3-2
sub_pool 8-7
sub_pool 5-3
sub_pool 6-3
8-7
5-3
sub_pool 6-2
6-2
sub_pool 10-0
10-0
sub_pool 10-1
10-1
6-3
sub_pool 10-2
10-2
sub_pool 10-4
10-4
sub_pool 9-0
sub_pool 6-5
sub_pool 2-0
sub_pool 10-5
10-5
sub_pool 10-6
9-0
2-0
sub_pool 9-1
sub_pool 6-4
sub_pool 10-3
10-3
sub_pool 10-8
10-8
sub_pool 10-9
10-9
6-4
6-5
10-6
sub_pool 9-2
sub_pool 9-3
sub_pool 2-1
sub_pool 10-7
9-1
sub_pool 9-4
sub_pool 9-5
9-5
9-4
9-2
9-3
2-1
10-7
sub_pool 9-6
9-6
sub_pool 9-7
9-7
sub_pool 9-8
9-8
Result Outside
[['1-0'], ['2-0', '2-1'], ['3-0', '3-1', '3-2'], ['4-0', '4-1', '4-2', '4-3'], ['5-0', '5-1', '5-2', '5-3', '5-4'], ['6-0', '6-1', '6-2', '6-3', '6-4', '6-5'], ['7-0', '7-1', '7-2', '7-3', '7-4', '7-5', '7-6'], ['8-0', '8-1', '8-2', '8-3', '8-4', '8-5', '8-6', '8-7'], ['9-0', '9-1', '9-2', '9-3', '9-4', '9-5', '9-6', '9-7', '9-8'], ['10-0', '10-1', '10-2', '10-3', '10-4', '10-5', '10-6', '10-7', '10-8', '10-9']]