如何在主线程的 WaitForMultipleObjects 调用中间安全地终止工作线程？

Question

Hi I am using C++ with the Qt framework for a Windows program. I use Qt threads but this situation could relevant for other thread APIs as well. I am dedicating a worker thread to monitor for directory changes using ReadDirectoryChangesW and WaitForMultipleObjects from the Win API. I want to be able to cancel the worker thread gracefully from the main thread. I have read about CancellIOEx that takes a handle and OVERLAPPED parameter but these data types are both pointers. Is there some safe way to pass these pointers from the worker thread to the main thread safely? Is there a better way of doing things?

Here's some code from here but using WaitForSingleObject instead of WaitForMultipleObjects, the function will be called from the worker thread. Am I allowed to post this code from the link? Also see here for valuable information about ReadDirectoryChangesW outside of the Windows Dev Center.

Thanks!

void WatchDirectory(LPCWSTR path)
{
   char buf[2048];
   DWORD nRet;
   BOOL result=TRUE;
   char filename[MAX_PATH];
   DirInfo[0].hDir = CreateFile (path, GENERIC_READ|FILE_LIST_DIRECTORY, 
                                 FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE,
                                 NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_OVERLAPPED,
                                 NULL);

   if(DirInfo[0].hDir == INVALID_HANDLE_VALUE)
   {
       return; //cannot open folder
   }

   lstrcpy( DirInfo[0].lpszDirName, path);
   OVERLAPPED PollingOverlap;

   FILE_NOTIFY_INFORMATION* pNotify;
   int offset;
   PollingOverlap.OffsetHigh = 0;
   PollingOverlap.hEvent = CreateEvent(NULL,TRUE,FALSE,NULL);
   while(result)
   {
       result = ReadDirectoryChangesW(
                  DirInfo[0].hDir,// handle to the directory to be watched
                  &buf,// pointer to the buffer to receive the read results
                  sizeof(buf),// length of lpBuffer
                  TRUE,// flag for monitoring directory or directory tree
                  FILE_NOTIFY_CHANGE_FILE_NAME |
                  FILE_NOTIFY_CHANGE_DIR_NAME |
                  FILE_NOTIFY_CHANGE_SIZE,
                //FILE_NOTIFY_CHANGE_LAST_WRITE |
                //FILE_NOTIFY_CHANGE_LAST_ACCESS |
                //FILE_NOTIFY_CHANGE_CREATION,
                &nRet,// number of bytes returned
                &PollingOverlap,// pointer to structure needed for overlapped I/O
                NULL);

       WaitForSingleObject(PollingOverlap.hEvent,INFINITE);
       offset = 0;
       int rename = 0;
       char oldName[260];
       char newName[260];
       do
       {
           pNotify = (FILE_NOTIFY_INFORMATION*)((char*)buf + offset);
           strcpy(filename, "");
           int filenamelen = WideCharToMultiByte(CP_ACP, 0, pNotify->FileName, pNotify->FileNameLength/2, filename, sizeof(filename), NULL, NULL);
           filename[pNotify->FileNameLength/2] = '';
           switch(pNotify->Action)
           {
               case FILE_ACTION_ADDED:
                   printf("\nThe file is added to the directory: [%s] \n", filename);
                   break;
               case FILE_ACTION_REMOVED:
                   printf("\nThe file is removed from the directory: [%s] \n", filename);
                   break;
               case FILE_ACTION_MODIFIED:
                   printf("\nThe file is modified. This can be a change in the time stamp or attributes: [%s]\n", filename);
                   break;
               case FILE_ACTION_RENAMED_OLD_NAME:
                   printf("\nThe file was renamed and this is the old name: [%s]\n", filename);
                   break;
               case FILE_ACTION_RENAMED_NEW_NAME:
                   printf("\nThe file was renamed and this is the new name: [%s]\n", filename);
                   break;
               default:
                   printf("\nDefault error.\n");
                   break;
            }

           offset += pNotify->NextEntryOffset;

        }while(pNotify->NextEntryOffset); //(offset != 0);
      }

    CloseHandle( DirInfo[0].hDir );

}

Answer 1

The main thread could create the OVERLAPPED struct and give it to the thread to use, rather than the other way around. However, trying to cancel the I/O from the main thread would be a race condition either way. Since your worker thread has to make a new call to ReadDirectoryChangesEx() after every directory event, it could be between calls to ReadDirectoryChangesEx() when the main thread wants the worker thread to terminate, thus calling CancelIoEx() when there is no I/O in progress would be a no-op.

Instead, create another event object for the main thread and worker thread to share, in addition to the event object that you are creating for the I/O. Have the worker thread wait on both events at the same time with WaitForMultipleObjects() , and then the main thread can signal the shared event when it wants the worker thread to terminate.

WaitForMultipleObjects() will tell the worker thread which event was signaled. If the shared event is signaled, the worker thread can cancel its I/O in progress via CancelIo/Ex() before exiting.

// shared with both threads...
HANDLE hTermEvent = CreateEvent(NULL, TRUE, FALSE, NULL);

// in main thread...
HANDLE hWorkerThread = CreateThread(...);
...
SetEvent(hTermEvent);
WaitForSingleObject(hWorkerThread, INFINITE);

// called by worker thread...
void WatchDirectory(LPCWSTR path)
{
   DWORD buf[512];
   DWORD nRet, dwRet;
   char filename[MAX_PATH];
   DirInfo[0].hDir = CreateFile(path, GENERIC_READ | FILE_LIST_DIRECTORY, 
                                 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
                                 NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED,
                                 NULL);

   if (DirInfo[0].hDir == INVALID_HANDLE_VALUE)
   {
       return; //cannot open folder
   }

   lstrcpy(DirInfo[0].lpszDirName, path);

   OVERLAPPED PollingOverlap = {};
   PollingOverlap.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
   if (!PollingOverlap.hEvent)
   {
       return; //cannot create I/O event to wait on
   }

   FILE_NOTIFY_INFORMATION* pNotify;
   int offset;

   do
   {
       if (!ReadDirectoryChangesW(
              DirInfo[0].hDir,// handle to the directory to be watched
              &buf,// pointer to the buffer to receive the read results
              sizeof(buf),// length of lpBuffer
              TRUE,// flag for monitoring directory or directory tree
              FILE_NOTIFY_CHANGE_FILE_NAME |
              FILE_NOTIFY_CHANGE_DIR_NAME |
              FILE_NOTIFY_CHANGE_SIZE,
              //FILE_NOTIFY_CHANGE_LAST_WRITE |
              //FILE_NOTIFY_CHANGE_LAST_ACCESS |
              //FILE_NOTIFY_CHANGE_CREATION,
              &nRet,// number of bytes returned
              &PollingOverlap,// pointer to structure needed for overlapped I/O
              NULL))
       {
           break; // can't wait for an event
       }

       HANDLE events[] = {hTermEvent, PollingOverlap.hEvent};

       dwRet = WaitForMultipleObjects(2, events, FALSE, INFINITE);
       if (dwRet != (WAIT_OBJECT_0 + 1))
       {
           CancelIo(DirInfo[0].hDir);
           GetOverlappedResult(DirInfo[0].hDir, &PollingOverlap, &nRet, TRUE);
           break; // terminate requested, or wait failed
       }

       if (!GetOverlappedResult(DirInfo[0].hDir, &PollingOverlap, &nRet, TRUE))
       {
           break; // read failed
       }

       if (nRet == 0)
       {
           continue; // overflow, current event data discarded
       }

       offset = 0;
       int rename = 0;
       char oldName[MAX_PATH];
       char newName[MAX_PATH];
       do
       {
           pNotify = (FILE_NOTIFY_INFORMATION*) (buf + offset);
           int filenamelen = WideCharToMultiByte(CP_ACP, 0, pNotify->FileName, pNotify->FileNameLength/2, filename, sizeof(filename), NULL, NULL);
           switch (pNotify->Action)
           {
               case FILE_ACTION_ADDED:
                   printf("\nThe file is added to the directory: [%.*s] \n", filenamelen, filename);
                   break;
               case FILE_ACTION_REMOVED:
                   printf("\nThe file is removed from the directory: [%.*s] \n", filenamelen, filename);
                   break;
               case FILE_ACTION_MODIFIED:
                   printf("\nThe file is modified. This can be a change in the time stamp or attributes: [%.*s]\n", filenamelen, filename);
                   break;
               case FILE_ACTION_RENAMED_OLD_NAME:
                   printf("\nThe file was renamed and this is the old name: [%.*s]\n", filenamelen, filename);
                   break;
               case FILE_ACTION_RENAMED_NEW_NAME:
                   printf("\nThe file was renamed and this is the new name: [%.*s]\n", filenamelen, filename);
                   break;
               default:
                   printf("\nDefault error.\n");
                   break;
           }

           offset += pNotify->NextEntryOffset;
       }
       while (pNotify->NextEntryOffset);
   }
   while (true);

   CloseHandle(PollingOverlap.hEvent);
   CloseHandle(DirInfo[0].hDir);
}

Answer 2

terminate thread never safe. how minimum this lead to resource leaks. and main - you never can know at which point thread was when you terminate it. it can for example be inside heap alloc/free critical section. and terminate it at this point lead to deadlock on next heap operation (because it critical section will never released).

however exist many correct solutions, how stop I/O. possible of course use 2 special event how already described in comments, but in my opinion this is not the best solution

1) we can use CancelIoEx on file handle. of course simply call CancelIoEx not enough - because at this time can be no I/O active in dedicated thread. need yet use special flag ( _bQuit ) for task canceled, but even this not enough. need check/set this flag inside critical section or rundown-protection with ReadDirectoryChangesW/CancelIoEx

in dedicated thread

AcquireSRWLockExclusive(this);

if (!_bQuit) // (1)
{
    ReadDirectoryChangesW(*); // (4)
}

ReleaseSRWLockExclusive(this);

and for stop

AcquireSRWLockExclusive(this);

_bQuit = true; // (2)
CancelIoEx(*); (3)

ReleaseSRWLockExclusive(this);

without critical section or rundown-protection will be possible execution in next order:

if (!_bQuit) // (1)
_bQuit = true; // (2)
CancelIoEx(*); (3)
ReadDirectoryChangesW(*); // (4)

can be situation when worked thread first check flag _bQuit and it still false. then main thread set flag and call CancelIoEx which will be have no effect, because no I/O on file. and only then worked thread call ReadDirectoryChangesW whicj will be not canceled. but by use critical section (in wide sense) we make this impossible. so possible only 2 orders: or

if (!_bQuit) ReadDirectoryChangesW(*); // (1)
_bQuit = true; CancelIoEx(*); // (2)

in this case ReadDirectoryChangesW will be canceled by CancelIoEx

or

_bQuit = true; CancelIoEx(*); // (1)
if (!_bQuit) ReadDirectoryChangesW(*); // (2)

in this case worked thread view _bQuit flag set and not call ReadDirectoryChangesW more.

in complete code this can look like:

inline ULONG BOOL_TO_ERROR(BOOL f)
{
    return f ? NOERROR : GetLastError();
}

struct WatchFolder : SRWLOCK
{
    HANDLE _hThread, _hFile;
    BOOLEAN _bQuit;

    WatchFolder() : _hThread(0), _hFile(0), _bQuit(false)
    {
        InitializeSRWLock(this);
    }

    ~WatchFolder()
    {
        if (_hThread) {
            WaitForSingleObject(_hThread, INFINITE);
            CloseHandle(_hThread);
        }
        if (_hFile) CloseHandle(_hFile);
    }

    static ULONG CALLBACK _WatchDirectory(PVOID This)
    {
        reinterpret_cast<WatchFolder*>(This)->WatchDirectory();
        return 0;
    }

    void WatchDirectory()
    {
        OVERLAPPED ov {};

        if (ov.hEvent = CreateEvent(0, 0, 0, 0))
        {
            union {
                FILE_NOTIFY_INFORMATION fni;
                char buf[0x800];// must be aligned as FILE_NOTIFY_INFORMATION
            };

            for(;;) 
            {
                AcquireSRWLockExclusive(this);

                ULONG dwError = _bQuit ? ERROR_OPERATION_ABORTED : BOOL_TO_ERROR(
                    ReadDirectoryChangesW(_hFile, buf, sizeof(buf), TRUE, FILE_NOTIFY_VALID_MASK, 0, &ov, 0));

                ReleaseSRWLockExclusive(this);

                ULONG NumberOfBytesTransferred = 0;

                if (dwError == NOERROR)
                {
                    dwError = BOOL_TO_ERROR(GetOverlappedResult(_hFile, &ov, &NumberOfBytesTransferred, TRUE));
                }

                if (dwError || !NumberOfBytesTransferred)
                {
                    if (dwError != ERROR_OPERATION_ABORTED)
                    {
                        __nop();
                    }
                    break;
                }

                FILE_NOTIFY_INFORMATION* pNotify = &fni;

                ULONG NextEntryOffset = 0;
                do 
                {
                    (PBYTE&)pNotify += NextEntryOffset;

                    DbgPrint("%x %.*S\n", pNotify->Action, pNotify->FileNameLength / sizeof(WCHAR), pNotify->FileName);

                } while (NextEntryOffset = pNotify->NextEntryOffset);
            }

            CloseHandle(ov.hEvent);
        }
    }

    ULONG Start(PCWSTR szFile)
    {
        HANDLE hFile = CreateFileW(szFile, FILE_GENERIC_READ, FILE_SHARE_VALID_FLAGS,
            NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_OVERLAPPED, NULL);

        ULONG dwError;

        if (hFile != INVALID_HANDLE_VALUE)
        {
            if (_hThread = CreateThread(0, 0, _WatchDirectory, this, 0, 0))
            {
                _hFile = hFile;

                return NOERROR;
            }
            dwError = GetLastError();
            CloseHandle(hFile);
        }
        else
        {
            dwError = GetLastError();
        }

        return dwError;
    }

    void Stop()
    {
        AcquireSRWLockExclusive(this);

        _bQuit = true, CancelIoEx(_hFile, 0);

        ReleaseSRWLockExclusive(this);
    }
};

void test()
{
    WatchFolder wf;
    if (wf.Start(L"somepath") == NOERROR)
    {
        MessageBoxW(0,0,0,0);
        wf.Stop();
    }
}

2) another way do this simply call CloseHandle(_hFile) instead CancelIoEx(_hFile, 0); . when handle(last, but assume you have only single handle) is closed - system end complete ReadDirectoryChangesW with status STATUS_NOTIFY_CLEANUP . code will be very similar to case CancelIoEx except now error on termination will be ERROR_NOTIFY_CLEANUP instead ERROR_OPERATION_ABORTED . but if use GetOverlappedResult[Ex] exist problem - this api have error in implementation - it lost all positive status values. it simply lost STATUS_NOTIFY_CLEANUP (but we of course can view it in Internal field of OVERLAPPED . code can be next:

            AcquireSRWLockExclusive(this);

            ULONG dwError = _bQuit ? ERROR_NOTIFY_CLEANUP : BOOL_TO_ERROR(
                ReadDirectoryChangesW(_hFile, buf, sizeof(buf), TRUE, FILE_NOTIFY_VALID_MASK, 0, &ov, 0));

            ReleaseSRWLockExclusive(this);

            ULONG NumberOfBytesTransferred = 0;

            if (dwError == NOERROR)
            {
                dwError = BOOL_TO_ERROR(GetOverlappedResult(_hFile, &ov, &NumberOfBytesTransferred, TRUE));
                // fix for error in GetOverlappedResult
                if (dwError == NOERROR && ov.Internal) dwError = RtlNtStatusToDosError((NTSTATUS)ov.Internal);
            }

            if (dwError || !NumberOfBytesTransferred)
            {
                if (dwError != ERROR_NOTIFY_CLEANUP)
                {
                    __nop();
                }
                break;
            }

and for stop

    AcquireSRWLockExclusive(this);

    _bQuit = true, CloseHandle(_hFile), _hFile = 0;

    ReleaseSRWLockExclusive(this);

3) else one option use alertable wait inside GetOverlappedResultEx and insert apc (or alert to worked thread). in this case we not need use critical section/or rundown-protection - because no matter will be apc (or alert) inserted before or after call ReadDirectoryChangesW - it anyway break wait.

            ULONG dwError = _bQuit ? STATUS_USER_APC : BOOL_TO_ERROR(
                ReadDirectoryChangesW(_hFile, buf, sizeof(buf), TRUE, FILE_NOTIFY_VALID_MASK, 0, &ov, 0));

            ULONG NumberOfBytesTransferred = 0;

            if (dwError == NOERROR)
            {
                dwError = BOOL_TO_ERROR(GetOverlappedResultEx(_hFile, &ov, &NumberOfBytesTransferred, INFINITE, TRUE));
            }

            if (dwError || !NumberOfBytesTransferred)
            {
                if (dwError == STATUS_USER_APC)
                {
                    CancelIo(_hFile);
                    GetOverlappedResult(_hFile, &ov, &NumberOfBytesTransferred, TRUE);
                }
                break;
            }

and for stop we need

static VOID NTAPI dummyAPC(_In_ ULONG_PTR )
{

}

_bQuit = true;
QueueUserAPC(dummyAPC, _hThread, 0);

of course instead call dummyAPC (which not need) better use alert, but GetOverlappedResultEx (more exactly WaitForSingleObjectEx ) ignore STATUS_ALERT and again begin wait after it interrupted with STATUS_ALERT . so need use custom code here

ULONG
WINAPI
GetOverlappedResult2( _In_ LPOVERLAPPED lpOverlapped,
                      _Out_ PULONG_PTR lpNumberOfBytesTransferred)
{
    while (lpOverlapped->Internal == STATUS_PENDING)
    {
        if (NTSTATUS status = ZwWaitForSingleObject(lpOverlapped->hEvent, TRUE, 0))
        {
            return RtlNtStatusToDosError(status);
        }
    }

    KeMemoryBarrier();

    *lpNumberOfBytesTransferred = lpOverlapped->InternalHigh;

    return RtlNtStatusToDosError((NTSTATUS)lpOverlapped->Internal);
}

and with it can use next code:

            ULONG dwError = _bQuit ? ERROR_ALERTED : BOOL_TO_ERROR(
                ReadDirectoryChangesW(_hFile, buf, sizeof(buf), TRUE, FILE_NOTIFY_VALID_MASK, 0, &ov, 0));

            ULONG_PTR NumberOfBytesTransferred = 0;

            if (dwError == NOERROR)
            {
                dwError = GetOverlappedResult2(&ov, &NumberOfBytesTransferred);
            }

            if (dwError || !NumberOfBytesTransferred)
            {
                if (dwError == ERROR_ALERTED)
                {
                    CancelIo(_hFile);
                    GetOverlappedResult(_hFile, &ov, (ULONG*)&NumberOfBytesTransferred, TRUE);
                }
                break;
            }

and for stop

_bQuit = true;
NtAlertThread(_hThread);

4) however the best way for my option - not use dedicated thread all all, but use complete asynchronous I/O. example of code

struct WatchFolderCB : SRWLOCK, OVERLAPPED
{
    HANDLE _hFile;
    LONG _dwRefCount;
    union {
        FILE_NOTIFY_INFORMATION fni;
        char buf[0x800];// must be aligned as FILE_NOTIFY_INFORMATION
    };
    BOOLEAN _bQuit;

    void AddRef()
    {
        InterlockedIncrementNoFence(&_dwRefCount);
    }

    void Release()
    {
        if (!InterlockedDecrement(&_dwRefCount))
        {
            delete this;
        }
    }

    WatchFolderCB() : _hFile(0), _bQuit(false), _dwRefCount(1)
    {
        InitializeSRWLock(this);
        RtlZeroMemory(static_cast<OVERLAPPED*>(this), sizeof(OVERLAPPED));
    }

    ~WatchFolderCB()
    {
        if (_hFile) CloseHandle(_hFile);
    }

    static VOID WINAPI _IoCompletionCallback(
        _In_    DWORD dwErrorCode,
        _In_    DWORD dwNumberOfBytesTransfered,
        _Inout_ LPOVERLAPPED lpOverlapped
        )
    {
        static_cast<WatchFolderCB*>(lpOverlapped)->IoCompletionCallback(
            RtlNtStatusToDosError(dwErrorCode), dwNumberOfBytesTransfered);
    }

    VOID IoCompletionCallback(DWORD dwErrorCode, DWORD NumberOfBytesTransferred)
    {
        if (dwErrorCode || !NumberOfBytesTransferred)
        {
            if (dwErrorCode != ERROR_NOTIFY_CLEANUP)
            {
                __nop();
            }
        }
        else
        {
            FILE_NOTIFY_INFORMATION* pNotify = &fni;

            ULONG NextEntryOffset = 0;
            do 
            {
                (PBYTE&)pNotify += NextEntryOffset;

                DbgPrint("%x %.*S\n", pNotify->Action, pNotify->FileNameLength / sizeof(WCHAR), pNotify->FileName);

            } while (NextEntryOffset = pNotify->NextEntryOffset);

            ReadChanges();
        }

        Release();
    }

    void ReadChanges()
    {
        AddRef();

        AcquireSRWLockExclusive(this);

        ULONG dwError = _bQuit ? ERROR_NOTIFY_CLEANUP : BOOL_TO_ERROR(
            ReadDirectoryChangesW(_hFile, buf, sizeof(buf), TRUE, FILE_NOTIFY_VALID_MASK, 0, this, 0));

        ReleaseSRWLockExclusive(this);

        if (dwError)
        {
            IoCompletionCallback(dwError, 0);
        }
    }

    ULONG Start(PCWSTR szFile)
    {
        HANDLE hFile = CreateFileW(szFile, FILE_GENERIC_READ, FILE_SHARE_VALID_FLAGS,
            NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_OVERLAPPED, NULL);

        ULONG dwError;

        if (hFile != INVALID_HANDLE_VALUE)
        {
            if (BindIoCompletionCallback(hFile, _IoCompletionCallback, 0))
            {
                _hFile = hFile;
                ReadChanges();
                return NOERROR;
            }
            dwError = GetLastError();
            CloseHandle(hFile);
        }
        else
        {
            dwError = GetLastError();
        }

        return dwError;
    }

    void Stop()
    {
        AcquireSRWLockExclusive(this);

        _bQuit = true, CloseHandle(_hFile), _hFile = 0;

        ReleaseSRWLockExclusive(this);
    }
};


void test1()
{
    if (WatchFolderCB* p = new WatchFolderCB)
    {
        if (p->Start(L"*") == NOERROR)
        {
            MessageBoxW(0,0,0,0);
            p->Stop();
        }
        p->Release();
    }
}