Python threading.join（）挂起

Question

My problem is as follows: I have a class that inherits from threading.Thread that I want to be able to stop gracefully. This class also has a Queue it get's its work from.

Since there are quite some classes in my project that should have this behaviour, I've created some superclasses to reduce duplicate code like this:

Thread related behaviour:

class StoppableThread(Thread):
def __init__(self):
    Thread.__init__(self)
    self._stop = Event()

def stop(self):
    self._stop.set()

def stopped(self):
    return self._stop.isSet()

Queue related behaviour:

class Queueable():
    def __init__(self):
        self._queue = Queue()

    def append_to_job_queue(self, job):
        self._queue.put(job)

Combining the two above and adding queue.join() to the stop() call

class StoppableQueueThread(StoppableThread, Queueable):
    def __init__(self):
        StoppableThread.__init__(self)
        Queueable.__init__(self)

    def stop(self):
        super(StoppableQueueThread, self).stop()
        self._queue.join()

A base class for a datasource:

class DataSource(StoppableThread, ABC):

    def __init__(self, data_parser):
        StoppableThread.__init__(self)
        self.setName("DataSource")
        ABC.__init__(self)
        self._data_parser = data_parser

    def run(self):
        while not self.stopped():
            record = self._fetch_data()
            self._data_parser.append_to_job_queue(record)

    @abstractmethod
    def _fetch_data(self):
        """implement logic here for obtaining a data piece
            should return the fetched data"""

An implementation for a datasource:

class CSVDataSource(DataSource):
    def __init__(self, data_parser, file_path):
        DataSource.__init__(self, data_parser)
        self.file_path = file_path
        self.csv_data = Queue()
        print('loading csv')
        self.load_csv()
        print('done loading csv')

    def load_csv(self):
        """Loops through csv and adds data to a queue"""
        with open(self.file_path, 'r') as f:

            self.reader = reader(f)
            next(self.reader, None)  # skip header
            for row in self.reader:
                self.csv_data.put(row)

    def _fetch_data(self):
        """Returns next item of the queue"""
        item = self.csv_data.get()
        self.csv_data.task_done()
        print(self.csv_data.qsize())
        return item

Suppose there is a CSVDataSource instance called ds , if I want to stop the thread I call:

ds.stop()
ds.join()

The ds.join() call however, never returns. I'm not sure why this is, because the run() method does check if the stop event is set.

Any Ideas?

Update

A little more clarity as requested: the applications is build up out of several threads. The RealStrategy thread (below) is the owner of all the other threads and is responsible for starting and terminating them. I haven't set the daemon flag for any of the threads, so they should be non-daemonic by default.

The main thread looks like this:

if __name__ == '__main__':
    def exit_handler(signal, frame):
        rs.stop_engine()
        rs.join()
        sys.exit(0)

    signal.signal(signal.SIGINT, exit_handler)



    rs = RealStrategy()
    rs.run_engine()

And here are the rs.run_engine() and rs.stop_engine() methods that are called in main :

class RealStrategy(Thread):
.....
.....
    def run_engine(self):
        self.on_start()
        self._order_handler.start()
        self._data_parser.start()
        self._data_source.start()
        self.start()

    def stop_engine(self):
        self._data_source.stop()
        self._data_parser.stop()
        self._order_handler.stop()

        self._data_source.join()
        self._data_parser.join()
        self._order_handler.join()

        self.stop()

Answer 1

If you want to use queue.Queue.join , then you must also use queue.Queue.task_done . You can read the linked documentation or see the following copied from information available online:

Queue. task_done()

Indicate that a formerly enqueued task is complete. Used by queue consumer threads. For each get() used to fetch a task, a subsequent call to task_done() tells the queue that the processing on the task is complete.

If a join() is currently blocking, it will resume when all items have been processed (meaning that a task_done() call was received for every item that had been put() into the queue).

Raises a ValueError if called more times than there were items placed in the queue.

---

Queue. join()

Blocks until all items in the queue have been gotten and processed.

The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer thread calls task_done() to indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks.

---

To test your problem, an example implementation was created to find out what was going on. It is slightly different from how your program works but demonstrates a method to solving your problem:

#! /usr/bin/env python3
import abc
import csv
import pathlib
import queue
import sys
import threading
import time


def main():
    source_path = pathlib.Path(r'C:\path\to\file.csv')
    data_source = CSVDataSource(source_path)
    data_source.start()
    processor = StoppableThread(target=consumer, args=[data_source])
    processor.start()
    time.sleep(0.1)
    data_source.stop()


def consumer(data_source):
    while data_source.empty:
        time.sleep(0.001)
    while not data_source.empty:
        task = data_source.get_from_queue(True, 0.1)
        print(*task.data, sep=', ', flush=True)
        task.done()


class StopThread(StopIteration):
    pass


threading.SystemExit = SystemExit, StopThread


class StoppableThread(threading.Thread):
    def _bootstrap(self, stop=False):
        # noinspection PyProtectedMember
        if threading._trace_hook:
            raise RuntimeError('cannot run thread with tracing')

        def terminate():
            nonlocal stop
            stop = True

        self.__terminate = terminate

        # noinspection PyUnusedLocal
        def trace(frame, event, arg):
            if stop:
                raise StopThread

        sys.settrace(trace)
        super()._bootstrap()

    def terminate(self):
        try:
            self.__terminate()
        except AttributeError:
            raise RuntimeError('cannot terminate thread '
                               'before it is started') from None


class Queryable:
    def __init__(self, maxsize=1 << 10):
        self.__queue = queue.Queue(maxsize)

    def add_to_queue(self, item):
        self.__queue.put(item)

    def get_from_queue(self, block=True, timeout=None):
        return self.__queue.get(block, timeout)

    @property
    def empty(self):
        return self.__queue.empty()

    @property
    def full(self):
        return self.__queue.full()

    def task_done(self):
        self.__queue.task_done()

    def join_queue(self):
        self.__queue.join()


class StoppableQueryThread(StoppableThread, Queryable):
    def __init__(self, target=None, name=None, args=(), kwargs=None,
                 *, daemon=None, maxsize=1 << 10):
        super().__init__(None, target, name, args, kwargs, daemon=daemon)
        Queryable.__init__(self, maxsize)

    def stop(self):
        self.terminate()
        self.join_queue()


class DataSource(StoppableQueryThread, abc.ABC):
    @abc.abstractmethod
    def __init__(self, maxsize=1 << 10):
        super().__init__(None, 'DataSource', maxsize=maxsize)

    def run(self):
        while True:
            record = self._fetch_data()
            self.add_to_queue(record)

    @abc.abstractmethod
    def _fetch_data(self):
        pass


class CSVDataSource(DataSource):
    def __init__(self, source_path):
        super().__init__()
        self.__data_parser = self.__build_data_parser(source_path)

    @staticmethod
    def __build_data_parser(source_path):
        with source_path.open(newline='') as source:
            parser = csv.reader(source)
            next(parser, None)
            yield from parser

    def _fetch_data(self):
        try:
            return Task(next(self.__data_parser), self.task_done)
        except StopIteration:
            raise StopThread from None


class Task:
    def __init__(self, data, callback):
        self.__data = data
        self.__callback = callback

    @property
    def data(self):
        return self.__data

    def done(self):
        self.__callback()


if __name__ == '__main__':
    main()