Python getattr 与 joblib 并行

Question

I have two classes, the first one is my base implementation, and the second class uses the class one and calls its methods using joblib's Parallel so that it would be something like this:

class MyClass:
    def __init__(self, param_list: list):
        self.param_list = param_list

    def method1(self, w, z):
        pass

    def method2(self, w, y, z):
        pass

Now, the second class is implemented like this:

from joblib import Parallel, delayed

class MyParallelClass:
    def __init__(self, param_list: list[dict]):
        self.param_list = param_list

    def method1(self, x: list[dict]):
        results = Parallel(n_jobs=-1)(delayed(MyClass(**params).method1)(**arguments)
                                      for params in self.param_list
                                      for arguments in x)
        return results

    def method2(self, x: list[dict]):
        results = Parallel(n_jobs=-1)(delayed(MyClass(**params).method2)(**arguments)
                                      for params in self.param_list
                                      for arguments in x)
        return results

This works, but as you can see, MyParallelClass methods 1 and 2 are essentially the same, the only thing that changes is the method is called from MyClass

So I'd like to simplify this having an internal method that handles the parallel execution, given a parameter that tells me which method to execute, so it would be something like this:

class MyParallelClass:
    def __init__(self, param_list: list[dict]):
        self.param_list = param_list

    def method1(self, x: list[dict]):
        return self._run(x, method="method1")

    def method2(self, x: list[dict]):
        return self._run(x, method="method2")
    
    def _run(self, x, method):
        pass

How can I write this _run method? I tried using the standard getatrr like this, but failed:

def _run(self, x: list[dict], method):
    results = Parallel(n_jobs=-1)(delayed(getattr(MyClass(**params), method))(**arguments)
                                  for params in self.param_list
                                  for arguments in x)
    return results 

but I get this error:

TypeError: cannot unpack non-iterable function object

Answer 1

Is this example explain you how to reuse class methods?

class Base:

  def __init__(self, init_arg):
    self.init_arg = init_arg

  def method1(self, x):
    print(x, "from method1,", "iteration_number is", self.init_arg)
    return f"lower value is {x.lower()} [iteration number is {self.init_arg}]"

  def method2(self, x, y):
    print(x, y, "from method2,", "iteration_number is", self.init_arg)
    return f"lower values are {x.lower()}, {y.lower()} [iteration number is {self.init_arg}]"

class Executor:

  def __init__(self, init_arg):
    self.base_instance = Base(init_arg)

  def _method_base(self, method_name, *args, **kwargs):
    return getattr(self.base_instance, method_name)(*args, **kwargs)

  def method1(self, x):
    return self._method_base('method1', x)

  # here x is *args and y=y is **kwargs
  def method2(self, x, y):
    return self._method_base('method2', x, y=y)

for idx, val in enumerate([("X1", "Y1"), ("X2", "Y2"), ("X3", "Y3")]):
  executor = Executor(idx)
  _1 = executor.method1(val[0])
  _2 = executor.method2(val[0], val[1])
  print("calculated values: ", _1, _2)

Output:

X1 from method1, iteration_number is 0
X1 Y1 from method2, iteration_number is 0
calculated values:  lower value is x1 [iteration number is 0] lower values are x1, y1 [iteration number is 0]
X2 from method1, iteration_number is 1
X2 Y2 from method2, iteration_number is 1
calculated values:  lower value is x2 [iteration number is 1] lower values are x2, y2 [iteration number is 1]
X3 from method1, iteration_number is 2
X3 Y3 from method2, iteration_number is 2
calculated values:  lower value is x3 [iteration number is 2] lower values are x3, y3 [iteration number is 2]