Return value while executing paralel processes with python

Question

I have been using multiprocessing with python and I have used queues successfully but there are some variables that I need to monitor (from main) while the process is still being executed.

I know that it is not a good practice to use global variables, but not even this approach has worked.

Can anyone can point me in the right direction?

Thanks in advance,

GCCruz

Addendum: I am posting a simple example of what I would like to do:

import multiprocessing
import time

def sampleprocess(array, count):
    '''process with heavy image processing in a loop'''
    for i in range(count)
        # Do processing on this array that outputs a given variable
        sample_variable= count*10 # I would like to monitor this variable  

if __name__ == '__main__':

    p = multiprocessing.Process(target=sampleprocess, args=(array,1000,))
    p.start()

    # continuously monitor the dummy variable that is being computed on the process
    while sample_variable < 1000
        time.sleep(0.1)
        print ' Still less than 1000'

Answer 1

One option is to use multiprocessing Value and Array for shared data objects. https://docs.python.org/2/library/multiprocessing.html#multiprocessing-managers

A working example from your sample code. If you were to have more than one process a lock would be needed to synchronize the writes.

---

from multiprocessing import Process, Value, Array, Lock import time

def sampleprocess(s, count, lock): '''process with heavy image processing in a loop''' for i in range(count.value): # Do processing on this array that outputs a given variable

sample_variable= count*10 # I would like to monitor this variable

    with lock:
        s.value= i*10

if name == ' main ':

val = Value('i', 1000)
sample_variable = Value('i', 0)
lock = Lock()

p = Process(target=sampleprocess, args=(sample_variable, val, lock))
p.start()

# continuously monitor the dummy variable that is being computed on the process
while sample_variable.value < 1000:
    time.sleep(0.1)
    print ' Still less than 1000'

Answer 2

Here are a few programs that might demonstrate how you could implement a solution:

Example 1

from multiprocessing import *
import time


def main():
    array, loops = list(range(1000)), 1000
    variable = Value('I')
    p = Process(target=sample, args=(array, loops, variable))
    p.start()
    while variable.value < 1000:
        print('Still less than 1000')
        time.sleep(0.005)
    print('Must be at least 1000')
    p.join()
    print('Value is', variable.value)


def sample(array, loops, variable):
    for number in range(loops):
        variable.value = number * 10
    print('Sample is done')

if __name__ == '__main__':
    main()

Example 2

from multiprocessing import *
import time


def main():
    processes = 10
    array, loops = list(range(1000)), 1000
    shared = Array('I', processes)
    p_array = []
    for index in range(processes):
        p = Process(target=sample, args=(array, loops, shared, index))
        p.start()
        p_array.append(p)
    while True:
        less_than_1000 = [p for p in enumerate(shared[:]) if p[1] < 1000]
        if less_than_1000:
            print(less_than_1000)
            time.sleep(0.001)
        else:
            break
    print('No process in less than 1000')
    for p in p_array:
        p.join()
    print(shared[:])


def sample(array, loops, p_array, index):
    time.sleep(1)
    for number in range(loops):
        time.sleep(0.001)
        p_array[index] = number * 10
    print('Sample is done')

if __name__ == '__main__':
    main()

Example 3

from multiprocessing import *
import time


def main():
    array, loops = list(range(1000)), 1000
    with Manager() as manager:
        variable = manager.Value('I', 0)
        p = Process(target=sample, args=(array, loops, variable))
        p.start()
        while variable.value < 1000:
            print('Still less than 1000')
            time.sleep(0.005)
        print('Must be at least 1000')
        p.join()
        print('Value is', variable.value)


def sample(array, loops, variable):
    for number in range(loops):
        variable.value = number * 10
    print('Sample is done')

if __name__ == '__main__':
    main()

Example 4

from multiprocessing import *
import time


def main():
    array, loops = list(range(1000)), 1000
    event = Event()
    p = Process(target=sample, args=(array, loops, event))
    p.start()
    event.wait()
    print('Must be at least 1000')
    p.join()


def sample(array, loops, event):
    for number in range(loops):
        if number >= 100 and not event.is_set():
            event.set()
            time.sleep(0.001)
    print('Sample is done')

if __name__ == '__main__':
    main()

As you can see, there are a variety of ways to do what you are asking to accomplish.