在进程之间共享队列

Question

I am pretty new to multiprocessing in python and trying to achieve something which should be a rather common thing to do. But I cannot find an easy way when searching the web.

I want to put data in a queue and then make this queue available to different consumer functions. Of course when getting an element from the queue, all consumer functions should get the same element. The following example should make clear what I want to achieve:

from multiprocessing import Process, Queue

def producer(q):

    for i in range(10):
        
        q.put(i)
    
    q.put(None)

def consumer1(q):
    while True:
        data = q.get()
        
        if data is None:   
            break
   
        print(data)

def consumer2(q):

    while True:
        data = q.get()
        
        if data is None:   
            break
   
        print(data)

def main():
    q = Queue()
    p1 = Process(target=producer, args=(q,))
    p2 = Process(target=consumer1, args=(q,))
    p3 = Process(target=consumer2, args=(q,))
    p1.start()
    p2.start()
    p3.start()
    p1.join()
    p2.join()
    p3.join()


if __name__ == '__main__':
    main()

Since the script is not terminating and I only get the print output of one function I guess this is not the way to do it. I think sharing a queue implies some things to consider? It works fine when using only one consumer function. Appreciate the help!

Answer 1

Your question exemplifies the misunderstanding

"all consumer functions should get the same element"

That's just not how queues work. Queues are automatically managed (there's quite a lot under the hood) such if one item is put in, only one item can be taken out. That item is not duplicated to all consumers. It seems like you actually need two separate queues to guarantee that each consumer gets each input without competing against the other consumer:

from multiprocessing import Process, Queue

def producer(q1, q2):

    for i in range(10):
        
        q1.put(i)
        q2.put(i)
    
    q1.put(None)
    q2.put(None)

def consumer1(q):
    while True:
        data = q.get()
        
        if data is None:   
            break
   
        print(data)

def consumer2(q):

    while True:
        data = q.get()
        
        if data is None:   
            break
   
        print(data)

def main():
    q1 = Queue()
    q2 = Queue()
    p1 = Process(target=producer,  args=(q1, q2))
    p2 = Process(target=consumer1, args=(q1,))
    p3 = Process(target=consumer2, args=(q2,))
    p1.start()
    p2.start()
    p3.start()
    p1.join()
    p2.join()
    p3.join()


if __name__ == '__main__':
    main()

Answer 2

If the values you are storing can be represented by one of the fundamental data types defined in the ctypes module , then the following could work. Here we are implementing a "queue" that can hold int values or None:

from multiprocessing import Process, Condition
import ctypes
from multiprocessing.sharedctypes import RawArray, RawValue
from threading import local
import time

my_local = local()
my_local.current = 0

class StructuredInt(ctypes.Structure):
    """
    This class is necessary because we want to be able to store in the RawArray
    either an int or None, which requires using ctypes.c_void_p as the array type.
    But, infortunately, ctypes.c_void_p(0) is interpreted as None.
    So we need a way to represent 0. Field value 'value' is the
    actual int value being stored and we use an arbitrarty 'ptr'
    field value that will not be interpreted as None.

    To store a None value, we set 'ptr' to ctypes.c_void_p(None) and field
    'value' is irrelevant.

    To store an integer. we set 'ptr' to ctypes.c_void_p(1) and field
    'value' has the actual value.
    """
    _fields_ = [('ptr', ctypes.c_void_p), ('value', ctypes.c_int)]

class MultiIntQueue:
    """
    An integer queue that can be processed by multiple threads where each thread
    can retrieve all the values added to the queue.

    :param maxsize: The maximum queue capacity (defaults to 20 if specified as None)
    :type maxsize: int
    """

    def __init__(self, maxsize=None):
        if maxsize is None:
            maxsize = 20
        self.maxsize = maxsize
        self.q = RawArray(StructuredInt, maxsize)
        self.condition = Condition()
        self.size = RawValue(ctypes.c_int, 0)

    def get(self):
        with self.condition:
            while my_local.current >= self.size.value:
                self.condition.wait()
        i = self.q[my_local.current]
        my_local.current += 1
        return None if i.ptr is None else i.value

    def put(self, i):
        assert 0 <= self.size.value < self.maxsize
        with self.condition:
            self.q[self.size.value] = (ctypes.c_void_p(None), 0) if i is None else (ctypes.c_void_p(1), i)
            self.size.value += 1
            self.condition.notify_all()


def producer(q):

    for i in range(10):
        q.put(i)
        time.sleep(.3) # simulate processing

    q.put(None)

def consumer1(q):
    while True:
        data = q.get()

        if data is None:
            break
        time.sleep(.1) # simulate processing

        print('Consumer 1:', data)

def consumer2(q):

    while True:
        data = q.get()

        if data is None:
            break
        time.sleep(.1) # simulate processing

        print('Consumer 2:', data)

def main():
    q = MultiIntQueue()
    p1 = Process(target=producer,  args=(q,))
    p2 = Process(target=consumer1, args=(q,))
    p3 = Process(target=consumer2, args=(q,))
    p1.start()
    p2.start()
    p3.start()
    p1.join()
    p2.join()
    p3.join()


if __name__ == '__main__':
    main()

Prints:

Consumer 1: 0
Consumer 2: 0
Consumer 2: 1
Consumer 1: 1
Consumer 2: 2
Consumer 1: 2
Consumer 2: 3
Consumer 1: 3
Consumer 2: 4
Consumer 1: 4
Consumer 1: 5
Consumer 2: 5
Consumer 1: 6
Consumer 2: 6
Consumer 1: 7
Consumer 2: 7
Consumer 2: 8
Consumer 1: 8
Consumer 1: 9
Consumer 2: 9