Unable to write/read value to/from registered shared object with multiprocessing BaseManager server across multiple processes

Question

So I have been having some troubles with the multiprocessing module, and namely with the BaseManager .

To reproduce the issue, the following four files can be written:

1. The first file queue_manager.py just defines my BaseManager to be served, along with a convenience function to connect to the server.

# queue_manager.py

import multiprocessing
from multiprocessing.managers import BaseManager

ADDRESS = "127.0.0.1"
PORT = 50000
PASSWORD = "password"

class QueueManager(BaseManager):
    pass

def connect_to_manager(names):
    for name in names:
        QueueManager.register(name + "_input_tasks")

    manager = QueueManager(address=(ADDRESS, PORT), authkey=PASSWORD.encode('utf-8'))
    manager.connect()
    return multiprocessing.Manager().dict({name: getattr(manager, name + "_input_tasks")() for name in names})

2. The server logic itself with file server.py . Here, we want to be able to serve two queues, named 'foo' and 'bar', which we could obtain across processes by calling the connect_to_manager() function defined before.

# server.py

from queue_manager import QueueManager, ADDRESS, PORT, PASSWORD
import queue


class Server:

    @staticmethod
    def serve(names):
        print("Serving...")
        for name in names:
            QueueManager.register(name + "_input_tasks", callable=lambda: queue.Queue())

        m = QueueManager(address=(ADDRESS, PORT), authkey=PASSWORD.encode('utf-8'))
        s = m.get_server()
        s.serve_forever()


def main(names):
    Server.serve(names)

if __name__ == '__main__':
    main(['foo', 'bar'])

3. The third file workers.py would be composed of the multiprocessing processes that should do some tasks on these queues. Here, we only obtain the value in the queue and print it.

# workers.py

import multiprocessing
from queue_manager import connect_to_manager, PASSWORD
import time

class MyProcess(multiprocessing.Process):
    def __init__(self, input_queue):
        multiprocessing.Process.__init__(self)
        self.input_queue = input_queue

    def run(self):
        multiprocessing.current_process().authkey = PASSWORD.encode('utf-8')
        while True:
            time.sleep(1)
            print("Running...")
            if input_queue.empty():
                print("Empty")
            else:
                x = input_queue.get()
                print(x) 

def create_workers(input_queue, n_workers):
    p = MyProcess(input_queue)
    p.start()
    p.join()


if __name__ == '__main__':
    multiprocessing.current_process().authkey = PASSWORD.encode('utf-8')
    name_queue_mapping = connect_to_manager(['foo', 'bar'])
    
    # assume we focus on foo queue
    input_queue = name_queue_mapping['foo']
    create_workers(input_queue, 1)

4. Lastly, we create a client which places some values inside the queue with client.py

# client.py

from queue_manager import connect_to_manager, PASSWORD
import multiprocessing
import time

def client():
    multiprocessing.current_process().authkey = PASSWORD.encode('utf-8')
    names = ['foo', 'bar']
    name_queue_mapping = connect_to_manager(names)
    foo_queue = name_queue_mapping["foo"]
    bar_queue = name_queue_mapping["bar"]


    for i in range(5):
        time.sleep(1)
        foo_queue.put(i)


if __name__ == '__main__':
    client()

Let's say we run each of the last three files simultaneously in separate terminals (and in this order):

• python server.py
• python client.py
• python workers.py

The issue is that the client successfully places values in the foo queue. HOWEVER, they are not picked up by the workers .

Now one way to circumvent this is NOT to pass a lambda function to the QueueManager.register() call in the serve() function. Ie like this:

QueueManager.register(name + "_input_tasks", callable=get_foo_queue)

# where get_foo_queue is defined globally in the server.py file as such

# server.py
my_foo_queue = queue.Queue()

def get_foo_queue():
    return my_foo_queue

class Server:
    ....

instead of:

QueueManager.register(name + "_input_tasks", callable=lambda: queue.Queue())

However, I don't want to do this because I want to be able to dynamically create the queues/functions, since I don't know their names nor how many I want before I run. Any Ideas?

Answer 1

First of all, in file workers.py method run , where you have references to variable input_queue , this needs to be changed to self.input_queue (see last listing).

I would then modify file file server.py to ensure that for a given queue name (such as foo , for example), that the QueueManager always serves up the same, singleton queue.Queue instance. Once you do that as follows in this updated server.py file...

Updated server.py

# server.py

from queue_manager import QueueManager, ADDRESS, PORT, PASSWORD
import queue

queues = {}

def get_queue(queue_name):
    def f():
        return queues[queue_name]
    return f

class Server:

    @staticmethod
    def serve(names):
        print("Serving...")
        for name in names:
            queue_name = name + "_input_tasks"
            q = queue.Queue()
            queues[queue_name] = q
            callable = get_queue(queue_name)
            QueueManager.register(queue_name, callable)

        m = QueueManager(address=(ADDRESS, PORT), authkey=PASSWORD.encode('utf-8'))
        s = m.get_server()
        s.serve_forever()


def main(names):
    Server.serve(names)

if __name__ == '__main__':
    main(['foo', 'bar'])

... then the updated queue_manager.py file becomes simplified as:

Updated queue_manager.py

# queue_manager.py

import multiprocessing
from multiprocessing.managers import BaseManager

ADDRESS = "127.0.0.1"
PORT = 50000
PASSWORD = "password"

class QueueManager(BaseManager):
    pass

def connect_to_manager(names):
    for name in names:
        QueueManager.register(name + "_input_tasks")

    manager = QueueManager(address=(ADDRESS, PORT), authkey=PASSWORD.encode('utf-8'))
    manager.connect()
    return {name: getattr(manager, name + "_input_tasks")() for name in names}

And for good measure, the corrected workers.py :

Corrected workers.py

# workers.py

import multiprocessing
from queue_manager import connect_to_manager, PASSWORD
import time

class MyProcess(multiprocessing.Process):
    def __init__(self, input_queue):
        multiprocessing.Process.__init__(self)
        self.input_queue = input_queue

    def run(self):
        multiprocessing.current_process().authkey = PASSWORD.encode('utf-8')
        while True:
            time.sleep(1)
            print("Running...")
            if self.input_queue.empty():
                print("Empty")
            else:
                x = self.input_queue.get()
                print(x)

def create_workers(input_queue, n_workers):
    p = MyProcess(input_queue)
    p.start()
    p.join()


if __name__ == '__main__':
    multiprocessing.current_process().authkey = PASSWORD.encode('utf-8')
    name_queue_mapping = connect_to_manager(['foo', 'bar'])

    # assume we focus on foo queue
    input_queue = name_queue_mapping['foo']
    create_workers(input_queue, 1)