Send data to another already running python process via named pipe

Question

I am trying to find ways to send data to a running Python (3.7+) process via a named pipe. As an example the process is adding an enthusiastic prefix to a string and printing it. The process has its own things to do and runs indefinitely. Beside it own things it gets new taks. As there are multiple tasks ahead is had a queue:

from queue import Queue 
import time
import tkinter as tk 
import os
import asyncio
import win32pipe, win32file, pywintypes
import sys

q = Queue()

for i in range(5):
    q.put(i) #own tasks

class C_Window():
    def __init__(self, parent=None, windowname='Window'): 
        self.parent = parent
        self.root =  tk.Tk()
        self.sendBox = tk.Text(self.root, height=1, width=30)
        self.sendBox.pack()
        self.buttonCommit=tk.Button(self.root, height=1, width=10, text="Commit", 
                            command=lambda: self.commit_button())
        self.buttonCommit.pack()
        self.root.update()

    def commit_button(self):
        inputValue=self.sendBox.get("1.0","end-1c")
        self.sendBox.delete('1.0', tk.END)
        q.put(inputValue)

    async def async_tk(self):
        while True:
            self.root.update()
            await asyncio.sleep(0.25) 

async def async_main():
    while True:
        if q.empty():
            print ("relaxing")
        else: 
            print ("YEAY! ", q.get())
        await asyncio.sleep(1) 


if __name__ == '__main__':
    window_obj = C_Window()
    windowtask = asyncio.ensure_future(window_obj.async_tk()) # create_task() replaces ensure_future() in 3.7+ 
    maintask = asyncio.ensure_future(async_main()) 
    loop = asyncio.get_event_loop()
    loop.run_forever()

This works as expected.: Te queue get done, when something is added thorugh the interface it gets done and if nothing is in the queue it is relaxing.

Now i would like to add taks from outside to the queue via a named pipe. For that i made a pipe class:

class C_Pipe():
    def __init__(self): 
        self.pipe = win32pipe.CreateNamedPipe(r'\\.\pipe\mypipe',
                                    win32pipe.PIPE_ACCESS_DUPLEX,
                                    win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE | win32pipe.PIPE_WAIT,
                                    1,  # nMaxInstances
                                    65536,  # nOutBufferSize
                                    65536,  # nInBufferSize
                                    0, # 50ms timeout (the default)
                                    None) # securityAttributes

    async def async_pipe(self):
        win32pipe.ConnectNamedPipe(self.pipe)
        while True:
            try:
                msg = win32file.ReadFile(self.pipe, 65536)[1].decode('utf-8')
                print(msg)
                self.main_queue.put(msg)
            except pywintypes.error as e:
                if e.winerror == 109: #no process on other side OR Pipe closed
                    win32pipe.DisconnectNamedPipe(self.pipe)
                    print("Reconnecting pipe")
                    win32pipe.ConnectNamedPipe(self.pipe)
            else:
                raise
            await asyncio.sleep(0.25)   

and then try to run it with:

if __name__ == '__main__':
    window_obj = C_Window()
    windowtask = asyncio.ensure_future(window_obj.async_tk()) 
    maintask = asyncio.ensure_future(async_main()) 
    pipe_obj = C_Pipe()
    pipetask = asyncio.ensure_future(pipe_obj.async_pipe()) 
    loop = asyncio.get_event_loop()
    loop.run_forever()

Which doesn't work. As soon as it's the pipetasks turn it blocks while reading and everything freezes. That's why I tried to put it in a separat thread:

if __name__ == '__main__':

    loop2 = asyncio.new_event_loop()
    pipe_obj = C_Pipe()
    pipetask = asyncio.run_coroutine_threadsafe(pipe_obj.async_pipe(),loop2) 

    loop = asyncio.get_event_loop()
    window_obj = C_Window()
    windowtask = asyncio.ensure_future(window_obj.async_tk()) 
    maintask = asyncio.ensure_future(async_main()) 

    loop.run_forever()

but the pipe recieves 1 message and then fails without every putting data into the Queue. My questions are:

1. Is there a way to out data into a Queue from an outside process (and get rid of the named pipe)?
2. Is there a way to run the named pipe in its own thread so it can stay blocked?
3. Is asyncio really the right choice here? I have been reading a lot about asyncio and multiprocessing but I cant find a clear picture

Thank you very much!

Answer 1

2. Is there a way to run the named pipe in its own thread so it can stay blocked?

This is probably the easiest approach. Your attempt failed because you never actually created a different thread. You created two event loops and ran only one. The idea with run_coroutine_threadsafe is to allow non-asyncio threads to submit jobs to the (single) asyncio event loop. First, the communication relies on sync APIs, so it can remain sync:

    # calling it sync_pipe, since it's no longer async
    def sync_pipe(self, enqueue):
        win32pipe.ConnectNamedPipe(self.pipe)
        while True:
            try:
                msg = win32file.ReadFile(self.pipe, 65536)[1].decode('utf-8')
                print(msg)
                enqueue(msg)
            except pywintypes.error as e:
                if e.winerror == 109: #no process on other side OR Pipe closed
                    win32pipe.DisconnectNamedPipe(self.pipe)
                    print("Reconnecting pipe")
                    win32pipe.ConnectNamedPipe(self.pipe)
            else:
                raise
            time.sleep(0.25)   

Note how we've sent it an abstract function that defines how to enqueue something. With that in place the main block can look like this:

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    queue = asyncio.Queue()
    def enqueue(item):
        loop.call_soon_threadsafe(queue.put_nowait, item)
    pipe_obj = C_Pipe()
    pipethread = threading.Thread(target=pipe_obj.sync_pipe, args=(enqueue,))

    window_obj = C_Window()
    windowtask = asyncio.ensure_future(window_obj.async_tk()) 
    maintask = asyncio.ensure_future(async_main(queue)) 

    loop.run_forever()

async_main now receives the queue that it will share with sync_pipe .