在進程之間共享隊列

Question

我對 python 中的多處理非常陌生，並試圖實現一些應該是相當常見的事情。 但是我在搜索網絡時找不到簡單的方法。

我想將數據放入隊列中，然后使該隊列可用於不同的消費者功能。 當然，從隊列中獲取元素時，所有消費者函數都應該獲取相同的元素。 以下示例應該清楚我想要實現的目標：

from multiprocessing import Process, Queue

def producer(q):

    for i in range(10):
        
        q.put(i)
    
    q.put(None)

def consumer1(q):
    while True:
        data = q.get()
        
        if data is None:   
            break
   
        print(data)

def consumer2(q):

    while True:
        data = q.get()
        
        if data is None:   
            break
   
        print(data)

def main():
    q = Queue()
    p1 = Process(target=producer, args=(q,))
    p2 = Process(target=consumer1, args=(q,))
    p3 = Process(target=consumer2, args=(q,))
    p1.start()
    p2.start()
    p3.start()
    p1.join()
    p2.join()
    p3.join()


if __name__ == '__main__':
    main()

由於腳本沒有終止，我只得到一個函數的打印輸出，我猜這不是這樣做的方法。 我認為共享隊列意味着需要考慮一些事情？ 僅使用一個消費者功能時它工作正常。 感謝幫助！

Answer 1

你的問題是誤解的例證

“所有消費者功能都應該獲得相同的元素”

這不是隊列的工作方式。 隊列是自動管理的（引擎蓋下有很多），如果放入一件物品，只能取出一件物品。 該項目不會復制給所有消費者。 看起來您實際上需要兩個單獨的隊列來保證每個消費者獲得每個輸入而不與另一個消費者競爭：

from multiprocessing import Process, Queue

def producer(q1, q2):

    for i in range(10):
        
        q1.put(i)
        q2.put(i)
    
    q1.put(None)
    q2.put(None)

def consumer1(q):
    while True:
        data = q.get()
        
        if data is None:   
            break
   
        print(data)

def consumer2(q):

    while True:
        data = q.get()
        
        if data is None:   
            break
   
        print(data)

def main():
    q1 = Queue()
    q2 = Queue()
    p1 = Process(target=producer,  args=(q1, q2))
    p2 = Process(target=consumer1, args=(q1,))
    p3 = Process(target=consumer2, args=(q2,))
    p1.start()
    p2.start()
    p3.start()
    p1.join()
    p2.join()
    p3.join()


if __name__ == '__main__':
    main()

Answer 2

如果您存儲的值可以由ctypes模塊中定義的一種基本數據類型表示，那么以下方法可以工作。 在這里，我們正在實現一個可以保存 int 值或 None 的“隊列”：

from multiprocessing import Process, Condition
import ctypes
from multiprocessing.sharedctypes import RawArray, RawValue
from threading import local
import time

my_local = local()
my_local.current = 0

class StructuredInt(ctypes.Structure):
    """
    This class is necessary because we want to be able to store in the RawArray
    either an int or None, which requires using ctypes.c_void_p as the array type.
    But, infortunately, ctypes.c_void_p(0) is interpreted as None.
    So we need a way to represent 0. Field value 'value' is the
    actual int value being stored and we use an arbitrarty 'ptr'
    field value that will not be interpreted as None.

    To store a None value, we set 'ptr' to ctypes.c_void_p(None) and field
    'value' is irrelevant.

    To store an integer. we set 'ptr' to ctypes.c_void_p(1) and field
    'value' has the actual value.
    """
    _fields_ = [('ptr', ctypes.c_void_p), ('value', ctypes.c_int)]

class MultiIntQueue:
    """
    An integer queue that can be processed by multiple threads where each thread
    can retrieve all the values added to the queue.

    :param maxsize: The maximum queue capacity (defaults to 20 if specified as None)
    :type maxsize: int
    """

    def __init__(self, maxsize=None):
        if maxsize is None:
            maxsize = 20
        self.maxsize = maxsize
        self.q = RawArray(StructuredInt, maxsize)
        self.condition = Condition()
        self.size = RawValue(ctypes.c_int, 0)

    def get(self):
        with self.condition:
            while my_local.current >= self.size.value:
                self.condition.wait()
        i = self.q[my_local.current]
        my_local.current += 1
        return None if i.ptr is None else i.value

    def put(self, i):
        assert 0 <= self.size.value < self.maxsize
        with self.condition:
            self.q[self.size.value] = (ctypes.c_void_p(None), 0) if i is None else (ctypes.c_void_p(1), i)
            self.size.value += 1
            self.condition.notify_all()


def producer(q):

    for i in range(10):
        q.put(i)
        time.sleep(.3) # simulate processing

    q.put(None)

def consumer1(q):
    while True:
        data = q.get()

        if data is None:
            break
        time.sleep(.1) # simulate processing

        print('Consumer 1:', data)

def consumer2(q):

    while True:
        data = q.get()

        if data is None:
            break
        time.sleep(.1) # simulate processing

        print('Consumer 2:', data)

def main():
    q = MultiIntQueue()
    p1 = Process(target=producer,  args=(q,))
    p2 = Process(target=consumer1, args=(q,))
    p3 = Process(target=consumer2, args=(q,))
    p1.start()
    p2.start()
    p3.start()
    p1.join()
    p2.join()
    p3.join()


if __name__ == '__main__':
    main()

印刷：

Consumer 1: 0
Consumer 2: 0
Consumer 2: 1
Consumer 1: 1
Consumer 2: 2
Consumer 1: 2
Consumer 2: 3
Consumer 1: 3
Consumer 2: 4
Consumer 1: 4
Consumer 1: 5
Consumer 2: 5
Consumer 1: 6
Consumer 2: 6
Consumer 1: 7
Consumer 2: 7
Consumer 2: 8
Consumer 1: 8
Consumer 1: 9
Consumer 2: 9