Using multiprocessing, stopping a subprocess from an PyQt5 application, where the subprocess is not using an event loop

Question

I am building a GUI application running neural.networks. Starting them off works fine, but I need the ability to abort the calculation, while the.net is already running and working.

I built a small prototype, which shows you the main mechanics of my problem within my application.

Architecture

Contains all GUI Elements and is the main Object which initiates and controls everything. It contains a QThreadPool self.__pool and the QRunnable Object self.__runner . The QRunnable Object contains everything needed for the neural.net. The reason why I use a QRunnable object is, to not block the GUI while in another thread the neural.net is being handled. Also I need to have communication between the neural.net and my application.

The runner class handles communication between the Main Window and the neural.net itself. The neural.net is a Process Object from multiprocessing put into self._.net . For communication I use a Queue self.__queue . When the QRunnable object is being started, the process is starting with self._.net.start() . I observe the Queue with a infinite loop. The loop is terminated on certain signals. In this example I use only the Signal NetSignal.finished . When the.net is finished I send a signal to the Window object.

Since QRunnables cannot make use of Signals, this class is needed to package some Signals into the QRuannable Object.

The Mnist class is the Subprocess itself, inheriting from Process . In this example it runs an really easy example of the mnist dataset being processed in a neural.network. There is no Loop surrounding the process, which normally can be used to stop such a subprocess. When the neural.network is finished, the queue transmits a signal to the QRunnable object, to signal the process having finished calculating, which therefore sends a signal to the main window.

Question

I need to be able to stop the process somehow. I thought of trying to kill it off with os.kill , which does not really work well in my application. I tried also self._.net.terminate() , self._.net.kill() . I was thinking of somehow passing an object into the callback parameter of the neural.net, to maybe abort the processing there, but I am not really sure if it is the way to go.

Code

from sys import exit, argv from multiprocessing import Process, Queue from PyQt5.QtWidgets import QPushButton, QApplication, QHBoxLayout, QWidget, QLabel from PyQt5.QtCore import QRunnable, QObject, pyqtSignal, QThreadPool class Window(QWidget): def __init__(self): QWidget.__init__(self) self.__btn_run = QPushButton("Start") self.__btn_stp = QPushButton("Stop") self.__label = QLabel("Idle") self.__runner = Runner() self.__pool = QThreadPool.globalInstance() self.__btn_run.clicked.connect(self.__run.net) self.__btn_stp.clicked.connect(self.__stp.net) self.__runner.signals.finished.connect(self.__on_finished) self.__btn_stp.setDisabled(True) self.setLayout(QHBoxLayout()) self.layout().addWidget(self.__btn_run) self.layout().addWidget(self.__btn_stp) self.layout().addWidget(self.__label) def __run.net(self): self.__btn_run.setDisabled(True) self.__btn_stp.setEnabled(True) self.__label.setText("Running") self.__pool.start(self.__runner) def __stp.net(self): pass # What to do here? def __on_finished(self): self.__btn_run.setEnabled(True) self.__btn_stp.setDisabled(True) self.__label.setText("Finished") self.__runner = Runner() class Runner(QRunnable): def __init__(self): QRunnable.__init__(self) self.__queue = Queue() self._.net = Mnist(self.__queue) self.signals = RunnerSignal() def run(self): self._.net.start() while True: data = self.__queue.get() if data == NetSignal.finished: self.signals.finished.emit() break class RunnerSignal(QObject): finished = pyqtSignal() class Mnist(Process): def __init__(self, queue: Queue): Process.__init__(self) self.__queue = queue def run(self): mnist = tf.keras.datasets.mnist # 28x28 Bilder hangeschriebener Ziffern von 0-9 (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = tf.keras.utils.normalize(x_train, axis=1) model = tf.keras.models.Sequential() model.add(tf.keras.layers.Flatten()) model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu)) model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu)) model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax)) model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=['accuracy']) model.fit(x_train, y_train, epochs=8) self.__queue.put(NetSignal.finished) self.__queue.close() class NetSignal: finished = "finished" if __name__ == "__main__": main_thread = QApplication(argv) main_window = Window() main_window.show() exit(main_thread.exec())

Answer 1

Okay by now I have found out what the problem was. When I terminated the process with self._.net.terminate() there were still zombieprocesses remaining, why I recognized it not being the correct solution.

The zombieprocess is being caused by the QRunnable, which is not terminating, since it is waiting for the subprocess to send a signal in the queue.

Therefore I need a custom "terminate" function which calls terminate() and also transmits a signal to the QRunnable, that the process has finished. This way all processes and threads terminate accordingly and no zombie process remains.

import tensorflow as tf
from sys             import exit, argv
from multiprocessing import Process, Queue
from PyQt5.QtWidgets import QPushButton, QApplication, QHBoxLayout, QWidget, QLabel
from PyQt5.QtCore    import QRunnable, QObject, pyqtSignal, QThreadPool


class Window(QWidget):

    def __init__(self):
        QWidget.__init__(self)

        self.__btn_run = QPushButton("Start")
        self.__btn_stp = QPushButton("Stop")
        self.__label   = QLabel("Idle")

        self.__runner  = Runner()
        self.__pool    = QThreadPool.globalInstance()

        self.__btn_run.clicked.connect(self.__run_net)
        self.__btn_stp.clicked.connect(self.__stp_net)
        self.__runner.signals.finished.connect(self.__on_finished)

        self.__btn_stp.setDisabled(True)

        self.setLayout(QHBoxLayout())
        self.layout().addWidget(self.__btn_run)
        self.layout().addWidget(self.__btn_stp)
        self.layout().addWidget(self.__label)

    def __run_net(self):
        self.__btn_run.setDisabled(True)
        self.__btn_stp.setEnabled(True)
        self.__label.setText("Running")
        self.__pool.start(self.__runner)

    def __stp_net(self):
        self.__runner.close()
        # What to do here?

    def __on_finished(self):
        self.__btn_run.setEnabled(True)
        self.__btn_stp.setDisabled(True)
        self.__label.setText("Finished")
        self.__runner = Runner()


class Runner(QRunnable):

    def __init__(self):
        QRunnable.__init__(self)
        self.__queue = Queue()
        self.__net   = Mnist(self.__queue)
        self.signals = RunnerSignal()

    def run(self):
        self.__net.start()
        while True:
            data = self.__queue.get()
            if data == NetSignal.finished:
                self.signals.finished.emit()
                break

    def close(self):
        self.__net.end_process()


class RunnerSignal(QObject):
    finished = pyqtSignal()


class Mnist(Process):
    def __init__(self, queue: Queue):
        Process.__init__(self)
        self.__queue = queue

    def run(self):
        mnist = tf.keras.datasets.mnist  # 28x28 Bilder hangeschriebener Ziffern von 0-9

        (x_train, y_train), (x_test, y_test) = mnist.load_data()

        x_train = tf.keras.utils.normalize(x_train, axis=1)

        model   = tf.keras.models.Sequential()
        model.add(tf.keras.layers.Flatten())
        model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
        model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
        model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax))

        model.compile(optimizer="adam",
                      loss="sparse_categorical_crossentropy",
                      metrics=['accuracy'])

        model.fit(x_train, y_train, epochs=8)
        self.__queue.put(NetSignal.finished)
        self.__queue.close()

    def end_process(self):
        self.terminate()
        self.__queue.put(NetSignal.finished)
        self.__queue.close()
        

class NetSignal:
    finished = "finished"


if __name__ == "__main__":
    main_thread = QApplication(argv)
    main_window = Window()
    main_window.show()
    exit(main_thread.exec())