Synchronize worker threads with a main thread

Question

how to correctly synchronize worker threads with a main thread if a worker thread can generate another tasks? I've used std::queue to maintain tasks guarded by mutex and atomic variable to track busy threads. Unfortunately I'm facing deadlocks at the end of the execution.

I've extracted code from my project and created a following example (you can easily compile it with g++ or MSVC):

#include <iostream>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <stdexcept>
#include <functional>
#include <stack>
#include <atomic>
#include <queue>

template <class T, class Compare>
class USort {
    using Task = std::pair<T*, T*>;
private:
    size_t m_ThreadsNum;
    std::atomic<bool> m_Finished;
    std::atomic<size_t> m_Busy;
    std::thread* m_Threads;
    std::queue<Task> m_Tasks;
    size_t m_Size;
    T* m_Data;
    Compare m_Comparator;
    std::condition_variable m_WaitFinished;
    std::condition_variable m_WaitSorter;
    std::mutex m_TaskQueueMutex;

private:
    const size_t THREAD_THRESHOLD = 1024;
    const size_t THREAD_POOL_THRESHOLD = 8192;


    bool HasTask() {
        std::unique_lock<std::mutex> lock(m_TaskQueueMutex);
        return m_Tasks.size() > 0;
    }

    bool PopTask(T** L, T** R) {
        std::unique_lock<std::mutex> lock(m_TaskQueueMutex);

        if (m_Tasks.size() == 0) {
            *L = *R = nullptr;
            return false;
        }

        *L = m_Tasks.front().first;
        *R = m_Tasks.front().second;
        m_Tasks.pop();

        return true;
    }

    void PushTask(T* L, T* R) {
        std::unique_lock<std::mutex> lock(m_TaskQueueMutex);
        m_Tasks.emplace(std::pair<T*, T*>(L, R));
        m_WaitSorter.notify_one();
    }

    void SortThread(size_t Id) {
        std::mutex sorter_mutex;
        for (;;) {
            std::unique_lock<std::mutex> lock(sorter_mutex);
            ///
            ///  ----------------------------------> some threads wait here
            /// 
            m_WaitSorter.wait(lock, [this]() { return m_Finished || HasTask(); });

            if (m_Finished) break;

            m_Busy++;

            T *left, *right;
            while (PopTask(&left, &right)) {
                Sort(left, right);
            }

            if (--m_Busy == 0) {
                m_WaitFinished.notify_one();
            }
        }
    }

    // just simulate work
    void Sort(T* Left, T* Right) {
        if (Right - Left > 10) {
            PushTask(Left, Right-10);
        }
    }

    void WaitForSortingIsFinished() {
        std::mutex finished;
        std::unique_lock<std::mutex> lock(finished);
        m_WaitFinished.wait(lock, [this]() { return m_Busy == 0 && !HasTask(); });
    }

    void FinishThreads() {
        m_Finished = true;
        m_WaitSorter.notify_all();
    }

    void ReleaseThreads() {
        if (m_Threads) {
            for (size_t i = 0; i < m_ThreadsNum; i++) {
                ///
                ///  ----------------------------------> main thread stuck here
                /// 
                m_Threads[i].join();
            }
            delete[] m_Threads;
            m_Threads = nullptr;
        }
    }

public:
    USort(size_t NumberOfThreads = 0) : m_Comparator(Compare()) {
        if (NumberOfThreads == 0) {
            static const unsigned int max_concurrency = std::thread::hardware_concurrency();
            NumberOfThreads = max_concurrency;
            if (NumberOfThreads == 0) NumberOfThreads = 4;
        }

        m_Finished = false;
        m_ThreadsNum = NumberOfThreads;
        m_Threads = nullptr;
    }

    ~USort() {
        ReleaseThreads();
    }

    void Sort(T* Data, size_t Size) {
        // build thread pool
        m_Threads = new std::thread[m_ThreadsNum];
        for (size_t i = 0; i < m_ThreadsNum; i++) {
            m_Threads[i] = std::thread(&USort::SortThread, this, i);
        }

        // process data
        PushTask(Data, Data + Size - 1);
        WaitForSortingIsFinished();
        FinishThreads();
    }

};

template <class T, class Compare>
void usort(T* Data, size_t Size, size_t NumberOfThreads = 0) {
    USort<T, Compare> mt_sorter(NumberOfThreads);
    mt_sorter.Sort(Data, Size);
}


const size_t ARR_SIZE = 0x00010000;


struct comp {
    bool operator()(const int& L, const int& R) const {
        return L < R;
    }
};

int main()
{
    int* arr = new int[ARR_SIZE];
    for (int i = 0; i < ARR_SIZE; i++) {
        arr[i] = rand() % 3200000;
    }

    usort<int, comp>(arr, ARR_SIZE, 16);

    delete[] arr;

    return 0;
}

The thing is, that in my example threads aren't always finished. From time to time some thread pending in m_WaitSorter.wait() and therefore main thread pending in m_Threads[i].join(); . Where is the flaw in the logic. Why the calling to FinishThreads() doesn't finish all threads?

EDIT: Basically I'd like to implement multithread sorting algorithm.

1. The main thread creates thread pool, push first task(sort whole array) to a task queue and waits for sorting to be finished
2. The pool thread takes task, divide it to smaller tasks(1-3). One of this task is immediatelly processed by the current pool thread, others are push to the queue
3. The pool thread musn't finish until the whole data set is sorted(there are no task in the queue and all pool threads are pending)
4. When the sorting is finished the main thread should be woken
5. Main thread should finish pending threads

So for this, from my perspective, I need two conditional_variabes with predicate "all threads are pending && has no task in queue" in main thread and "has task in queue || finish thread" in pool thread.

Answer 1

OK, I've read the documentation through carefully and found a bug in my code. Calls to notify_one() , notify_all() and wait() have to be controlled via the same mutext. With that in mind I've update and little bit simplified my code:

    bool WaitAndPopTask(T** L, T** R) {
        std::unique_lock<std::mutex> lock(m_TaskQueueMutex);
        m_WaitSorter.wait(lock, [this]() { return m_Finished || !m_Tasks.empty(); });

        if (m_Finished) return false;

        m_Busy++;

        *L = m_Tasks.front().first;
        *R = m_Tasks.front().second;
        m_Tasks.pop();

        return true;
    }

    void SortThread(size_t Id) {
        for (;;) {
            T *left, *right;
            if (!WaitAndPopTask(&left, &right)) break;

            Sort(left, right);

            std::lock_guard<std::mutex> lk(m_TaskQueueMutex);
            if (--m_Busy == 0 && m_Tasks.empty()) {
                FinishThreads();
            }
        }
    }

    void Sort(T* Data, size_t Size) {
        // build thread pool
        m_Threads = new std::thread[m_ThreadsNum];
        for (size_t i = 0; i < m_ThreadsNum; i++) {
            m_Threads[i] = std::thread(&USort::SortThread, this, i);
        }

        // process data
        PushTask(Data, Data + Size - 1);
        ReleaseThreads();
    }