SetCapture does not work and returns nullptr

Question

I'm trying to capture the mouse for a windowed game to handle mouse input and everytime I try, it returns nullptr. What I am seeing on the window is a waiting cursor (blue circle).

Here is the code for Win32 window wrapper:

XWindow::XWindow(WCHAR* title, int width, int height)
{
     memcpy(m_szTitle, title, sizeof(title));
     m_width = width;
     m_height = height;
}

ATOM XWindow::MyRegisterClass(HINSTANCE hInstance)
{
    WNDCLASSEXW wcex;

    wcex.cbSize = sizeof(WNDCLASSEX);

    wcex.style          = CS_HREDRAW | CS_VREDRAW;
    wcex.lpfnWndProc    = WndProc;
    wcex.cbClsExtra     = 0;
    wcex.cbWndExtra     = 0;
    wcex.hInstance      = hInstance;
    wcex.hIcon          = LoadIcon(hInstance, 
    MAKEINTRESOURCE(IDI_WIN32PROJECT1));
    wcex.hCursor        = LoadCursor(nullptr, IDC_ARROW);
    wcex.hbrBackground  = (HBRUSH)(COLOR_WINDOW+1);
    wcex.lpszMenuName = nullptr;
    wcex.lpszClassName  = m_szWindowClass;
    wcex.hIconSm        = LoadIcon(wcex.hInstance, 
    MAKEINTRESOURCE(IDI_SMALL));

    return RegisterClassExW(&wcex);
}

BOOL XWindow::InitInstance(HINSTANCE hInstance, int nCmdShow)
{
    m_hInst = hInstance; 
    m_hWnd = CreateWindowW(m_szWindowClass, m_szTitle, WS_OVERLAPPEDWINDOW,
    CW_USEDEFAULT, 0, m_width, m_height, nullptr, nullptr, hInstance, nullptr);

    if (!m_hWnd)
        return FALSE;

    ShowWindow(m_hWnd, nCmdShow);
    UpdateWindow(m_hWnd);

    return TRUE;
}


LRESULT CALLBACK XWindow::WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    switch (message)
    {
        case WM_COMMAND:
        {
            int wmId = LOWORD(wParam);
            switch (wmId)
            {
                case IDM_EXIT:
                DestroyWindow(hWnd);
                break;
                default:
                return DefWindowProc(hWnd, message, wParam, lParam);
            }
        }
        break;
        case WM_PAINT:
        {
            PAINTSTRUCT ps;
            HDC hdc = BeginPaint(hWnd, &ps);

            EndPaint(hWnd, &ps);
        }
        break;
        case WM_DESTROY:
            PostQuitMessage(0);
            break;
        default:
            return DefWindowProc(hWnd, message, wParam, lParam);
        }
        return 0;
}

And this is the WinMain function:

int APIENTRY wWinMain(_In_ HINSTANCE hInstance, _In_opt_ HINSTANCE 
hPrevInstance, _In_ LPWSTR lpCmdLine, _In_ int nCmdShow)
{
    XIllumin* pMainGameWindow = new XIllumin(L"X3DEngine", 800, 600);

    UNREFERENCED_PARAMETER(hPrevInstance);
    UNREFERENCED_PARAMETER(lpCmdLine);

    if (LoadStringW(hInstance, IDS_APP_TITLE, pMainGameWindow-
    >GetWindowTitle(), MAX_LOADSTRING) == 0)
    return FALSE;

    if (LoadStringW(hInstance, IDC_WIN32PROJECT1, pMainGameWindow-
    >GetWindowClass(), MAX_LOADSTRING) == 0)
    return FALSE;

    pMainGameWindow->MyRegisterClass(hInstance);

    if (!pMainGameWindow->InitInstance(hInstance, nCmdShow))
        return FALSE;

    if (SetCapture(pMainGameWindow->GetWindowHandle()) == nullptr)
    //  return FALSE;

    if (!pMainGameWindow->ParseInitFile("Config/GameInit.txt"))
    {
        pMainGameWindow->FAIL_MSG_BOX(L"Error loading init file.");
        return FALSE;
    }

    pMainGameWindow->InitGameObjects(" ");

    HACCEL hAccelTable = LoadAccelerators(hInstance, 
    MAKEINTRESOURCE(IDC_WIN32PROJECT1));

    MSG msg;

    while (1)
    {
        if (PeekMessage(&msg, nullptr, 0, 0, PM_REMOVE))
        {
            if (msg.message == WM_QUIT)
            {
                 pMainGameWindow->CleanUp();
                 break;
            }

            TranslateMessage(&msg);
            DispatchMessage(&msg);
        }

        if (!pMainGameWindow->GameMain(" "))
            break;
    }

    return (int)msg.wParam;
}

I am not sure what exactly is wrong. I think it might be the window creation parameters for CreateWindowW(), but don't really know which to use.

EDIT: The input is supposed to be handled by the game window when I hover the mouse cursor on it, but I see a waiting cursor as soon as the window pops up and the cursor is over it. Not really sure what else to say on top of that.

Answer 1

Note that SetCapture returns a handle to the window previously captured the mouse. So it is OK for SetCapture to return NULL when mouse was not captured by some other window previously. "Wait" cursor has nothing to do with it.

Note that by commenting // return FALSE; you change if scope to the following, potentially skipping config loading:

if (SetCapture(pMainGameWindow->GetWindowHandle()) == nullptr)
{
    //  return FALSE;

    if (!pMainGameWindow->ParseInitFile("Config/GameInit.txt"))
    {
        pMainGameWindow->FAIL_MSG_BOX(L"Error loading init file.");
        return FALSE;
    }
}

This is a good example of why {} should never be omitted.

You should pump messages prior to calling SetCapture so window would be initialized and get to foreground propertly:

// at InitInstance
UpdateWindow(m_hWnd);
MSG msg;
while(PeekMessageW(&msg, nullptr, 0, 0, PM_REMOVE))
{
       TranslateMessage(&msg);
       DispatchMessageW(&msg);
}