Run an infinite process that checks multiprocessing queues for processing

Question

The program has three parts: Add data to queues, Process data in Queues and Check if queue's empty. Each except Add(), all parts having their own process. The program should function like this, When we start it,

It should keep checking queues, if anything in it run the function, if not keep running.

At the same time, an asynchronous data can be added to the queue and it should run the job like stated previously

Only one thing can be processed from the queue at a time.

I am using windows, and I keep getting

from multiprocessing import Queue,Process
from time import sleep
import threading
from multiprocessing.pool import ThreadPool
from multiprocessing import dummy as multithreading
import concurrent.futures

# import queue
class A(object):
    def __init__(self, *args, **kwargs):
        self.Q = Queue()

#Add data to queue: should be accessable all time
    def Add(self, i):
        # q = Queue()
        self.Q.put(threading.Thread(target=self.printAns(i)))

#Processes the data: runs only upon call
    def printAns(self,name):
        print("Name to print is: ",name)
        return 'completed'

#This function call printANS as a process
    def jobRun(self):
        # job = self.Q.get()
        # ans = Queue()
        jobThread = self.Q.get()
        async_result = jRPool.apply_async(jobThread)
        print(async_result.get())

#Checks if the queue has anything: checker functions needs to run constantly
    def checkQueue(self):
        while True:
            if self.Q.empty():
                pass
            else:
                return True

#should initiate call to checker upon success calls jobRun() as a process and go back to checking
    def run(self):
        with concurrent.futures.ProcessPoolExecutor() as executor:
            checkfunc = executor.map(self.checkQueue)
            while True:
                if checkfunc:
                    sleep(1)
                    executor.map(self.jobRun)
                    self.Q.close()  




if __name__ == '__main__':
    a = A()
    a.Add("test1")
    a.Add("test2")
    a.run()
    # a.Add("this")
    while True:
        data = input("Enter a string: ")
        a.Add(data)

Here is the code

 from multiprocessing import Queue,Process from time import sleep import threading from multiprocessing.pool import ThreadPool from multiprocessing import dummy as multithreading import concurrent.futures # import queue class A(object): def __init__(self, *args, **kwargs): self.Q = Queue() #Add data to queue: should be accessable all time def Add(self, i): # q = Queue() self.Q.put(threading.Thread(target=self.printAns(i))) #Processes the data: runs only upon call def printAns(self,name): print("Name to print is: ",name) return 'completed' #This function call printANS as a process def jobRun(self): # job = self.Q.get() # ans = Queue() jobThread = self.Q.get() async_result = jRPool.apply_async(jobThread) print(async_result.get()) #Checks if the queue has anything: checker functions needs to run constantly def checkQueue(self): while True: if self.Q.empty(): pass else: return True #should initiate call to checker upon success calls jobRun() as a process and go back to checking def run(self): with concurrent.futures.ProcessPoolExecutor() as executor: checkfunc = executor.map(self.checkQueue) while True: if checkfunc: sleep(1) executor.map(self.jobRun) self.Q.close() if __name__ == '__main__': a = A() a.Add("test1") a.Add("test2") a.run() # a.Add("this") while True: data = input("Enter a string: ") a.Add(data) 

Any Kind of help is deeply appreciated. My hunch is something has to do with locks or semaphores.

Answer 1

Something like this might work for you if I've understood your requirements correctly.

• compute is the function called in the remote process. Right now it just prints the PID of the worker processing things and prints out each letter of the string. This is so you can see the processes work in parallel.
• tasks is a list of task objects the poll_results thread should keep track of.
• poll_results is a threading.Thread target that runs in the main process; it busy-loops over the tasks in tasks , printing the result value as soon as they become ready.

import os
import time
import threading
from multiprocessing import Pool


def compute(value):
    for ch in value:
        print(os.getpid(), ch)
        time.sleep(0.05)
    return (value, len(value))


tasks = []


def poll_results():
    while True:
        for task in tasks[:]:
            if task.ready():
                print("Task finished:", task.get())
                tasks.remove(task)


def main():
    poller_thread = threading.Thread(target=poll_results)
    poller_thread.start()

    with Pool() as p:
        t1 = p.apply_async(compute, ("hello",))
        t2 = p.apply_async(compute, ("world",))
        # Wait for the first results before entering the loop
        t1.wait()
        t2.wait()

        while True:
            data = input("Enter a string: ")
            tasks.append(p.apply_async(compute, (data,)))


if __name__ == "__main__":
    main()

The output here is something like

65755 h
65756 w
65755 e
65756 o
65755 l
65756 r
65755 l
65756 l
65755 o
65756 d
Enter a string: Hello, world!
Enter a string: 65757 H
65757 e
65757 l
65757 l
65757 o
65757 ,
65757
65757 w
65757 o
65757 r
65757 l
65757 d
65757 !
Task finished: ('Hello, world!', 13)

Note how the IO is interleaved (you get the second "Enter string" prompt even before the task is being processed, and the two workers print out the letters from hello and world whenever they can).