How to append multiprocessed result that is in a loop?

Question

I want to utilize multiple processors to calculate a function for two lists of values. In the test case below, my idea is that I have two lists: c = [1,2,3], and c_shift = [2,3,4]. I want to evaluate a function for a single value in each list, and append two separate solution arrays.

import numpy as np
import multiprocessing as mp

def function(x,a,b,c):

    return a*x**2+b*x+c

def calculate(x,a,b,c):
    
    c_shift = c+1
    
    result = []
    result_shift = []
    
    for i in range(len(c)):
        process0 = mp.Process(target = function, args = (x,a,b,c[i]))
        process1 = mp.Process(target = function, args = (x,a,b,c_shift[i]))
        
        process0.start()
        process1.start()
        
        process0.join()
        process1.join()
        
        # After it finishes, how do I append each list?

    return np.array(result), np.array(result_shift)

if __name__ == '__main__':
    x = np.linspace(-1,1,50)
    a = 1
    b = 1
    c = np.array([1,2,3])
    
    calculate(x,a,b,c)

When each process finishes, and goes through join() , how do I append process0 to result = [] and process1 to result_shift = [] ?

The structure of the returned results should have the form:

result = [ [1 x 50], [1 x 50], [1 x 50] ]

result_shifted = [ [1 x 50], [1 x 50], [1 x 50] ]

Answer 1

Slightly different approach but I think this is what you were looking to do?

import multiprocessing
import numpy as np
from functools import partial


def your_func(c, your_x, a, b):
    results = []
    for c_value in c:
        results.append(a * your_x ** 2 + b * your_x + c_value)
    return results


def get_results(c_values):
    your_x = np.linspace(-1, 1, 50)
    a = 1
    b = 1
    with multiprocessing.Pool() as pool:
        single_arg_function = partial(your_func, your_x=your_x, a=a, b=b)
        out = pool.map(single_arg_function, c_values)
        return out


if __name__ == "__main__":
    c_values = [np.array([1, 2, 3]), np.array([1, 2, 3]) + 1]
    out = get_results(c_values)
    result_one = out[0]
    result_two = out[1]

Answer 2

I'm not sure exactly what you're trying to accomplish with the shifted results, but in terms of concurrency, you should check out concurrent.futures to execute parallel tasks. Also, take a look at functools.partial to create a partial object - essentially a function with pre-filled args / kwargs. Here's an example:

import concurrent.futures
from functools import partial

import numpy as np


def map_processes(func, _iterable):
    with concurrent.futures.ProcessPoolExecutor() as executor:
        result = executor.map(func, _iterable)
    return result

def function(x, a, b, c):
    return a * x**2 + b * (x + c)

if __name__ == "__main__":
    base_func = partial(function, np.linspace(-1, 1, 50), 1, 1)
    print(list(map_processes(base_func, np.array([1, 2, 3]))))