无法在我的PyQT5应用程序中使用joblib并行运行方法，

Question

I m quite new to PyQT5 parallel programming. I have 10 methods that I want to run parallelly in my GUI, I get an abort on the Joblib that I use.

I get TypeError: cannot unpack non-iterable function object . I try to use enumerator but still I get an abort.

---

My error message:

joblib.externals.loky.process_executor._RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 418, in _process_worker
    r = call_item()
  File "/usr/local/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 272, in __call__
    return self.fn(*self.args, **self.kwargs)
  File "/usr/local/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 567, in __call__
    return self.func(*args, **kwargs)
  File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 225, in __call__
    for func, args, kwargs in self.items]
  File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 225, in <listcomp>
    for func, args, kwargs in self.items]
TypeError: cannot unpack non-iterable function object
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "job_parallel.py", line 73, in button_click_parallel
    Parallel(n_jobs=8)(delayed(command_list[i]()) for i in range(10))
  File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 934, in __call__
    self.retrieve()
  File "/usr/local/lib/python3.7/site-packages/joblib/parallel.py", line 833, in retrieve
    self._output.extend(job.get(timeout=self.timeout))
  File "/usr/local/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 521, in wrap_future_result
    return future.result(timeout=timeout)
  File "/usr/local/Cellar/python/3.7.2/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/_base.py", line 432, in result
    return self.__get_result()
  File "/usr/local/Cellar/python/3.7.2/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
TypeError: cannot unpack non-iterable function object
Abort trap: 6

My code:

#!/usr/bin/env python3

import sys
from PyQt5 import QtCore, QtWidgets
from joblib import Parallel, delayed
import threading


class Window(QtWidgets.QMainWindow):

    #uart_list = [1,2,3,4,5]

    def __init__(self):
        super(Window, self).__init__()

        self.x_size = 800
        self.y_size = 650
        self.setFixedSize(self.x_size, self.y_size)
        self.setWindowTitle("Test")

        buttons = list()

        self.button1 = QtWidgets.QPushButton("ONE", self)
        self.button1.move(100,100)
        buttons.append(self.button1)

        self.button2 = QtWidgets.QPushButton("TWO", self)
        self.button2.move(100,150)
        buttons.append(self.button2)

        for button in buttons:
            button.clicked.connect(self.button_click_parallel)

        self.show()

    def run_iot_uart1(self):
        print("Job 1")

    def run_iot_uart2(self):
        print("Job 2")

    def run_iot_uart3(self):
        print("Job 3")

    def run_iot_uart4(self):
        print("Job 4")

    def run_iot_uart5(self):
        print("Job 5")

    def run_iot_uart6(self):
        print("Job 6")

    def run_iot_uart7(self):
        print("Job 7")

    def run_iot_uart8(self):
        print("Job 8")

    def run_iot_uart9(self):
        print("Job 9")

    def run_iot_uart10(self):
        print("Job 10")


    def button_click_parallel(self): # Click any button, process all UART in parallel
        command_list=[self.run_iot_uart1, self.run_iot_uart2, self.run_iot_uart3,
                      self.run_iot_uart4, self.run_iot_uart5, self.run_iot_uart6,
                      self.run_iot_uart7, self.run_iot_uart8, self.run_iot_uart9,
                      self.run_iot_uart10]
        Parallel(n_jobs=10)(delayed(command_list[i]()) for i in range(10))
       sys.stdout.flush()

def run():
    QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling, True)
    app = QtWidgets.QApplication(sys.argv)
    GUI = Window()
    sys.exit(app.exec_())

run()

Answer 1

The problem seems to be in your Parallel setup. I haven't used this library myself previously, but the documentation gives the following example, where collect is a function.

Parallel(n_jobs=2)(
    delayed(collect)(i) 
    for i in range(5)
)

Here the function is passed to delayed, and the parameters for that function are passed as arguments to the result of that call. That is

delayed_function = delayed(function)
delayed_function(arg)

In your example code you're getting the function via command_list[i] and then calling it immediately command_list[i]() then passing the result of that function (a string) to delayed .

Parallel(n_jobs=10)(
    delayed(command_list[i]()) 
    for i in range(10)
)

I think what you're looking for is

Parallel(n_jobs=10)(
    delayed(command_list[i])()
    for i in range(10)
)

Cannot pickle x

After making that change to your code I still receive the following error —

File "/usr/local/lib/python3.6/site-packages/joblib/parallel.py", line 183, in 
delayed
pickle.dumps(function)
TypeError: can't pickle Window objects

This occurs because Parallel uses pickle (a Python serialization format) to send functions and arguments to the executors. Unfortunately it looks as though QWindow (and probably other Qt objects?) are not pickleable. Because your window methods need access to self this needs to be pickled.

The solution is to move these methods either a) to a pickleable class, b) to functions. Below I've done the second.

#!/usr/bin/env python3

import sys
from PyQt5 import QtCore, QtWidgets
from joblib import Parallel, delayed
import threading

def run_iot_uart1():
    print("Job 1")

def run_iot_uart2():
    print("Job 2")

def run_iot_uart3():
    print("Job 3")

def run_iot_uart4():
    print("Job 4")

def run_iot_uart5():
    print("Job 5")

def run_iot_uart6():
    print("Job 6")

def run_iot_uart7():
    print("Job 7")

def run_iot_uart8():
    print("Job 8")

def run_iot_uart9():
    print("Job 9")

def run_iot_uart10():
    print("Job 10")


class Window(QtWidgets.QMainWindow):

    #uart_list = [1,2,3,4,5]

    def __init__(self):
        super(Window, self).__init__()

        self.x_size = 800
        self.y_size = 650
        self.setFixedSize(self.x_size, self.y_size)
        self.setWindowTitle("Test")

        buttons = list()

        self.button1 = QtWidgets.QPushButton("ONE", self)
        self.button1.move(100,100)
        buttons.append(self.button1)

        self.button2 = QtWidgets.QPushButton("TWO", self)
        self.button2.move(100,150)
        buttons.append(self.button2)

        for button in buttons:
            button.clicked.connect(self.button_click_parallel)

        self.show()

    def button_click_parallel(self): # Click any button, process all UART in parallel
        command_list=[run_iot_uart1, run_iot_uart2, run_iot_uart3,
                      run_iot_uart4, run_iot_uart5, run_iot_uart6,
                      run_iot_uart7, run_iot_uart8, run_iot_uart9,
                      run_iot_uart10]
        Parallel(n_jobs=10)(delayed(command_list[i])() for i in range(10))
        sys.stdout.flush()

def run():
    QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling, True)
    app = QtWidgets.QApplication(sys.argv)
    GUI = Window()
    sys.exit(app.exec_())

run()

Running this and pressing button "ONE" I get the following output.

martin@Martins-MacBook-Pro temp $ python3 parallel.py
Job 1
Job 3
Job 2
Job 4
Job 5
Job 6
Job 7
Job 8
Job 9
Job 10