Python 多处理通过Queue共享大数据？

Question

I would like to put output data into a queue in a multiprocessing computation. It seems that when the size of the return is too large, the program got stuck. To illustrate the problem, here is a minimal codes. Anyone can help to make this work?

from multiprocessing import Process, Queue
import numpy as np

def foo(q, qid):
    x = np.random.randint(0,5,7)
    y = np.random.random(100*10*10).reshape(100,10,10)
    q.put([qid,x,y])

def main():
    processes = []
    q = Queue()

    for qid in range(5):
        p = Process(target=foo, args=(q, qid))
        p.start()
        processes.append(p)

    for process in processes:
        process.join()

    for qid in range(5):
        [_, x, y] = q.get()
        print(x)
        print(y)

if __name__ == '__main__':
    main()

Answer 1

I figured out one solution is as below to switch the join and get . By default, the get method blocks.

from multiprocessing import Process, Queue
import numpy as np

def foo(q, qid):
    x = np.random.randint(0,5,7)
    y = np.random.random(100*10*10).reshape(100,10,10)
    q.put([qid,x,y])

def main():
    processes = []
    q = Queue()

    for qid in range(5):
        p = Process(target=foo, args=(q, qid))
        p.start()
        processes.append(p)

    for qid in range(5):
        [_, x, y] = q.get()
        print(x)
        print(y)

    for process in processes:
        process.join()

 if __name__ == '__main__':
    main()