Elegant solution for IPC in python with multiprocessing

Question

I have two independent processes on the same machine in need of IPC. As of now, I have this working solution:

server.py

#!/usr/bin/python3
from multiprocessing.managers import BaseManager
from multiprocessing import Process, Queue

def do_whatever():
    print('function do whatever, triggered by xyz')
    # do something

def start_queue_server(q):
    class QueueManager(BaseManager): pass

    QueueManager.register('get_queue', callable=lambda:q)
    m = QueueManager(address=('', 55555), authkey=b'tuktuktuk')
    s = m.get_server()
    s.serve_forever()

def main():
    queue = Queue()
    proc = Process(target=start_queue_server, args=(queue,))  
    proc.start()

    while True:
        command = queue.get()
        print('command from queue:', command)

        if command == 'xyz':
            do_whatever()

        # many more if, elif, else statements

if __name__ == "__main__":
    main()

client.py

#!/usr/bin/python3

from multiprocessing.managers import BaseManager

def communicator(command):
    class QueueManager(BaseManager): pass
    QueueManager.register('get_queue')

    m = QueueManager(address=('', 55555), authkey=b'tuktuktuk')
    m.connect()
    queue = m.get_queue()
    queue.put(command)

def main():
    command = ('xyz')
    communicator(command)

if __name__ == "__main__":
    main()

• Is there a more elegant way to call 'do_whatever' than parsing the commands passed on by the queue and then calling the target function?
• Can I somehow pass on a reference to 'do_whatever' and call it directly from the client?
• How is an answer from the server, eg True or False, communicated to the client? I tried passing a shared variable instead of a queue object but failed. Do I need to open another connection using a second socket to pass the answer?

I read the python documentation but couldn't find more options for unrelated processes. Inputs would be welcome! Cheers singultus

Answer 1

Finally, I settled for an additional Listener

server.py

#!/usr/bin/python3
from multiprocessing.managers import BaseManager
from multiprocessing import Process, Queue
from multiprocessing.connection import Client

def do_whatever():
    print('function do whatever, triggered by xyz')
    # do something

def start_queue_server(q):
    class QueueManager(BaseManager): pass

    QueueManager.register('get_queue', callable=lambda:q)
    m = QueueManager(address=('', 55555), authkey=b'tuktuktuk')
    s = m.get_server()
    s.serve_forever()

def talkback(msg, port):
    conn = Client(address=('', port), authkey=b'tuktuktuk')
    conn.send(msg)
    conn.close()

def main():
    queue = Queue()
    proc = Process(target=start_queue_server, args=(queue,))  
    proc.start()

    while True:
        command = queue.get()
        print('command from queue:', command)

        if command[0] == 'xyz':
            do_whatever()
            talkback('aaa', command[1])

        # many more if, elif, else statements

if __name__ == "__main__":
    main()

client.py

#!/usr/bin/python3

from multiprocessing.managers import BaseManager
from multiprocessing.connection import Listener

def communicator(command, talkback=False):
    if talkback:
        listener = Listener(address=('', 0), authkey=b'prusaprinter')
        return_port = listener.address[1]
        command = command + (return_port,)

    class QueueManager(BaseManager): pass
    QueueManager.register('get_queue')

    m = QueueManager(address=('', 55555), authkey=b'tuktuktuk')
    m.connect()
    queue = m.get_queue()
    queue.put(command)

    if talkback:
        conn = listener.accept()
        server_return = conn.recv()
        conn.close()
        listener.close()
        return server_return

def main():
    command = ('xyz')
    communicator(command, True)

if __name__ == "__main__":
    main()

The client opens an available port and starts listening on it. It then sends the command to the server together with the aforementioned port number. The server executes the command, then uses the port number to report back to the client. After receiving the answer, the client closes the port.

Answer 2

I have two independent processes on the same machine in need of IPC. As of now, I have this working solution:

server.py

#!/usr/bin/python3
from multiprocessing.managers import BaseManager
from multiprocessing import Process, Queue

def do_whatever():
    print('function do whatever, triggered by xyz')
    # do something

def start_queue_server(q):
    class QueueManager(BaseManager): pass

    QueueManager.register('get_queue', callable=lambda:q)
    m = QueueManager(address=('', 55555), authkey=b'tuktuktuk')
    s = m.get_server()
    s.serve_forever()

def main():
    queue = Queue()
    proc = Process(target=start_queue_server, args=(queue,))  
    proc.start()

    while True:
        command = queue.get()
        print('command from queue:', command)

        if command == 'xyz':
            do_whatever()

        # many more if, elif, else statements

if __name__ == "__main__":
    main()

client.py

#!/usr/bin/python3

from multiprocessing.managers import BaseManager

def communicator(command):
    class QueueManager(BaseManager): pass
    QueueManager.register('get_queue')

    m = QueueManager(address=('', 55555), authkey=b'tuktuktuk')
    m.connect()
    queue = m.get_queue()
    queue.put(command)

def main():
    command = ('xyz')
    communicator(command)

if __name__ == "__main__":
    main()

• Is there a more elegant way to call 'do_whatever' than parsing the commands passed on by the queue and then calling the target function?
• Can I somehow pass on a reference to 'do_whatever' and call it directly from the client?
• How is an answer from the server, eg True or False, communicated to the client? I tried passing a shared variable instead of a queue object but failed. Do I need to open another connection using a second socket to pass the answer?

I read the python documentation but couldn't find more options for unrelated processes. Inputs would be welcome! Cheers singultus