Python 多处理共享 memory； 一写多读

Question

SYSTEM

• Linux (Manjaro KDE)
• Python 3.8.3

PROGRAM:
I have incoming string data on a UDP port. The main loop spools up the processes prior to using selectors to monitor the UDP port. I want the UDP data, which is constantly updated, available for each process.

TRIED:

• Multiprocessing Queues with maxsize = 1 and that became a headache and quickly broke down.
• Multiprocessing Arrays (this is where I'm at now)

I have checked, and the Array at each location I'm looking at has the same memory address (I think). For whatever reason, when I try to access the contents of the Array in the child process, the process hangs.

NOT TRIED

• Pipes. I have a feeling this may be the way to go. But I'm already deep in uncharted territory; I've never used them before.

WHAT I WANT
I would like to access the UDP data from the child processes - these are the camera_view method.

Dummy UDP string

import socket
import random
import datetime
import time

conn = ('127.0.0.1', 6666)

def rand_value(f_val, t_val):
    result = round(random.uniform(f_val, t_val), 2)  
    result = random.uniform(f_val, t_val)
    return result

sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

while True:

    time.sleep(6)
    timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    overlay = timestamp

    for i in range(9):
        val = rand_value(i*10, i*10+10)
        if i == 8: val = 'TASK: Im the real Batman'
        overlay = overlay + "," + str(val)
    
    print(overlay)
    sock.sendto(overlay.encode(), conn)

My Program

import datetime
import selectors
import socket
import time
from multiprocessing import Lock, Process, Queue
from multiprocessing.sharedctypes import Array
from ctypes import c_char_p


REQUIRED_CAMERAS = 1
CAMERA_CONN = {'name':['Colour Camera'], 'ip':['127.0.0.1'], 'port':[9000]}
OVERLAY_CONN = ('0.0.0.0', 6666)
CONTROL_CONN = ('0.0.0.0', 6667)
NUMBER_OF_ITEMS_IN_OVERLAY = 10

class Camera():
    def __init__(self, cam_name, cam_ip, cam_port):
        self.ip = cam_ip
        self.port = cam_port
        self.capture = cv2.VideoCapture(0)
        self.frame_width = int(self.capture.get(3))
        self.frame_height = int(self.capture.get(4))
        self.name = cam_name


def get_overlay(data_packet):
        data = data_packet.decode()
        data = data.split(',')
        field0 = data[0]
        field1 = 'KP: ' + str(round(float(data[1]), 3))
        field2 = 'DCC: ' + str(round(float(data[2]), 2)) + 'm'
        field3 = 'E: ' + str(round(float(data[3]), 2)) + 'm'
        field4 = 'N: ' + str(round(float(data[4]), 2)) + 'm'
        field5 = 'D: ' + str(round(float(data[5]), 2)) + 'm'
        field6 = 'H: ' + str(round(float(data[6]), 2)) # + '°'
        field7 = 'R: ' + str(round(float(data[7]), 2)) # + '°'
        field8 = 'P: ' + str(round(float(data[8]), 2)) # + '°' 
        field9 = data[9]

        x = []
        for i in range(NUMBER_OF_ITEMS_IN_OVERLAY):
            x.append(eval('field' + str(i)).encode())
            # if i == 0:
            #     print(x[i])
                
        return x

def socket_reader(sock, mask, q, REQUIRED_CAMERAS, overlay):
    data_packet, sensor_ip = sock.recvfrom(1024)
    sensor_port = sock.getsockname()[1]
    print(f'SENSOR PORT {sensor_port} and SENSOR_IP {sensor_ip}')

    if sensor_port == OVERLAY_CONN[1]:
        x = get_overlay(data_packet)
        for i in range(len(x)):
            overlay[i] = x[i]
            print(f'Socket Reader {overlay}')

def camera_view(CAMERA_CONN, cam_name, camera, overlay_q, control_q, overlay):
    while True:
        print(f'PROCESS {camera} RUNNING FOR: {cam_name}')
        try:
            print(f'Camera View {overlay}')
            for i in range(len(overlay)):
                print(overlay[i])
        except:
            pass
        time.sleep(1)
        

def controller(REQUIRED_CAMERAS, CAMERA_CONN, OVERLAY_CONN, CONTROL_CONN):
    
    if REQUIRED_CAMERAS > len(CAMERA_CONN['name']):
        print(f'REQURIED_CAMERAS: {REQUIRED_CAMERAS} - more than connections in CAMERA_CONN ')
    else:
        # Set up a UDP connection for the overlay string and the control commands
        sock_overlay = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        sock_control = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        sock_overlay.bind(OVERLAY_CONN)
        sock_control.bind(CONTROL_CONN)
        
        # Set up the selector to watch over the socket
        # and trigger when data is ready for reading
        sel = selectors.DefaultSelector()
        sel.register(fileobj=sock_overlay, events=selectors.EVENT_READ, data=socket_reader)
        sel.register(fileobj=sock_control, events=selectors.EVENT_READ, data=socket_reader)
        
        # create shared memory
        overlay_q = Queue(maxsize=1)
        control_q = Queue(maxsize=1)     
        overlay = Array(c_char_p, range(NUMBER_OF_ITEMS_IN_OVERLAY))
        print(f'Init Overlay {overlay}')
        
        # Generate the processes; one per camera
        processes = []
        
        for camera in range(REQUIRED_CAMERAS):
            processes.append(Process(target=camera_view, args=(CAMERA_CONN, CAMERA_CONN['name'][camera], camera, overlay_q, control_q, overlay)))

        for process in processes:
            process.daemon = True
            process.start()
            
        # Spin over the selector
        while True:

            # Only have one connnection registered, so to stop
            # the loop spinning up the CPU, I have made it blocking 
            # with the timeout = 1 (sec) instead of =0.
            events = sel.select(timeout=None)

            for key, mask in events:
                # the selector callback is the data= from the register above
                callback = key.data
                # the callback gets the sock, mask and the sensor queues
                if key.fileobj == sock_overlay:
                    callback(key.fileobj, mask, overlay_q, REQUIRED_CAMERAS, overlay)
                else:
                    callback(key.fileobj, mask, control_q, REQUIRED_CAMERAS, overlay)


if __name__ == "__main__":
    
    controller(REQUIRED_CAMERAS, CAMERA_CONN, OVERLAY_CONN, CONTROL_CONN)

EDIT1:

from multiprocessing import Process, Array
from ctypes import c_char_p
import time

def worker(arr):
    count = 0
    while True:
        count += 1
        val = 'val' + str(count)
        arr[0] = val
        print(arr[:])
        time.sleep(2)

def main():
    arr = Array(c_char_p, 1)
    p = Process(target=worker, args=(arr,))
    p.daemon = True
    p.start()
    
    while True:
        print(arr[:])
        try:
            print(arr[:].decode('utf-8'))
        except :
            pass
        # try:
        #     val = arr[:]
        #     val = val.decode('utf-8')
        #     print(f'main {val}')
        # except:
        #     pass
        time.sleep(1)

if __name__ == "__main__":
    main()
    
    
'''
from multiprocessing import Process, Array
from ctypes import c_char_p
import time

def worker(arr):
    count = 0
    while True:
        count += 1
        val = 'val' + str(count)
        arr[0] = bytes(val, 'utf-8')
        print(arr[:])
        time.sleep(2)

def main():
    arr = Array(c_char_p, 1)
    p = Process(target=worker, args=(arr,))
    p.daemon = True
    p.start()
    
    while True:
        print(arr[:])
        try:
            print(arr[:].decode('utf-8'))
        except :
            pass

        time.sleep(1)

if __name__ == "__main__":
    main()

if __name__ == "__main__":
    main()
'''

EDIT2:
Thanks to @RolandSmith, I have persevered with Queues and I think I have got a template on how I can move forward. See below code. If I can't get this to work in program, I'll be back here.

from multiprocessing import Process, Queue
import time
import datetime

def worker(camera, q):
    val = ''
    while True:    
        if q.full() == True:
            val = q.get()
        else:
            val = val
        print(f'WORKER{camera} {val}')
        time.sleep(0.2)

def main():
    
    cameras = 2
    
    processes = []
    queues = []
    
    for camera in range(cameras):
        queues.append(Queue(maxsize=1))
        processes.append(Process(target=worker, args=(camera, queues[camera])))
        
    for process in processes:                     
        process.daemon = True
        process.start()

    while True:
        for q in queues:
            if not q.empty():
                try:
                    _ = q.get()
                except:
                    pass
            else:
                q.put(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        time.sleep(.5)

if __name__ == "__main__":
    main()

Answer 1

In my view, using Queue is a less error-prone solution than using an Array .

Here is your second example, converted to using a Queue :

from multiprocessing import Process, Queue
import time


def worker(q):
    count = 0 
    while True:
        count += 1
        val = 'val' + str(count)
        q.put(val)
        print('worker:', val)
        time.sleep(2)


def main():
    q = Queue()

    p = Process(target=worker, args=(q, ))
    p.daemon = True
    p.start()

    while True:
        if not q.empty():
            print('main:', q.get())
        time.sleep(1)


if __name__ == "__main__":
    main()

This yields:

> python3 test3.py
worker: val1
main: val1
worker: val2
main: val2
worker: val3
main: val3
worker: val4
main: val4
worker: val5

---

Here is the same example using a Pipe :

from multiprocessing import Process, Pipe
import time


def worker(p):
    count = 0 
    while True:
        count += 1
        val = 'val' + str(count)
        p.send(val)
        print('worker:', val)
        time.sleep(2)


def main():
    child, parent = Pipe()

    p = Process(target=worker, args=(child, ))
    p.daemon = True
    p.start()

    while True:
        if parent.poll():
            print('main:', parent.recv())
        time.sleep(1)


if __name__ == "__main__":
    main()

This produces the same result as the previous example.

Additionally, by default a pipe is bidirectional. So you could also send back data from the workers to the parent.