使用aa管理器更新Python多进程中的队列

Question

我正在设计一个Python多处理代码以在可能随着处理而更新的队列中工作。 以下代码有时会起作用，卡住或引发Empty错误。

import multiprocessing as mp

def worker(working_queue, output_queue):
    while True:
        if working_queue.empty() is True:
            break    
        else:
            picked = working_queue.get_nowait()
            if picked % 2 == 0: 
                    output_queue.put(picked)
            else:
                working_queue.put(picked+1)
    return

if __name__ == '__main__':
    manager = mp.Manager()
    static_input = xrange(100)    
    working_q = manager.Queue()
    output_q = mp.Queue()
    for i in static_input:
        working_q.put(i)
    processes = [mp.Process(target=worker,args=(working_q, output_q)) for i in range(mp.cpu_count())]
    for proc in processes:
        proc.start()
    for proc in processes:
        proc.join()
    results_bank = []
    while True:
       if output_q.empty() is True:
           break
       results_bank.append(output_q.get_nowait())
    print len(results_bank) # length of this list should be equal to static_input, which is the range used to populate the input queue. In other words, this tells whether all the items placed for processing were actually processed.
    results_bank.sort()
    print results_bank

我应该使用列表作为全局变量并锁定它，而不是使用manager.Queue（）吗？

Answer 1

我只是添加了一个try:和except Exception:来处理Empty错误。 现在的结果似乎是一致的。 请让我知道如果您发现在此解决方案中我忽略的问题。

import multiprocessing as mp

def worker(working_queue, output_queue):
    while True:
        try:
            if working_queue.empty() is True:
                break  
            else:
                picked = working_queue.get_nowait()
                if picked % 2 == 0: 
                        output_queue.put(picked)
                else:
                    working_queue.put(picked+1)
        except Exception:
            continue

    return

if __name__ == '__main__':
    #Manager seem to be unnecessary.
    #manager = mp.Manager()
    #working_q = manager.Queue()

    working_q = mp.Queue()
    output_q = mp.Queue()
    static_input = xrange(100)     
    for i in static_input:
        working_q.put(i)
    processes = [mp.Process(target=worker,args=(working_q, output_q)) for i in range(mp.cpu_count())]
    for proc in processes:
        proc.start()
    for proc in processes:
        proc.join()
    results_bank = []
    while True:
       if output_q.empty() is True:
           break
       results_bank.append(output_q.get_nowait())
    print len(results_bank) # length of this list should be equal to static_input, which is the range used to populate the input queue. In other words, this tells whether all the items placed for processing were actually processed.
    results_bank.sort()
    print results_bank

Answer 2

只需使用一个锁来保护对共享数据的访问，它就更安全了（并且可以保护您免受进程的怪异行为影响）：

import multiprocessing as mp

def worker(working_queue, output_queue, lock):
    while True:
        shouldBeak = False
        lock.acquire()
        if working_queue.empty() is True:
            shouldBeak = True    
        else:

            picked = working_queue.get_nowait()
            if picked % 2 == 0: 
                output_queue.put(picked)
            else:
                working_queue.put(picked+1)
        lock.release()
        if shouldBeak:
            break
    return

if __name__ == '__main__':
    manager = mp.Manager()
    static_input = xrange(1000)    
    working_q = manager.Queue()
    output_q = mp.Queue()
    lock = mp.Lock()
    for i in static_input:
        working_q.put(i)
    processes = [mp.Process(target=worker,args=(working_q, output_q,lock)) for i in range(mp.cpu_count())]
    for proc in processes:
        proc.start()
    for proc in processes:
        proc.join()
    results_bank = []
    while True:
       if output_q.empty() is True:
           break
       results_bank.append(output_q.get_nowait())
    print len(results_bank) # length of this list should be equal to static_input, which is the range used to populate the input queue. In other words, this tells whether all the items placed for processing were actually processed.
    results_bank.sort()
    print results_bank