Hook WH_GETMESSAGE message

Question

I'm trying to hook WH_GETMESSAGE from my class to determine the moment when specific window is resizing. However, looks like the hook isn't set.

Class from where I try to hook:

class WindowDisplayHelper : // public ...
{    
public:
    // some other public methods here
    void SetMsgHook();
protected:
    LRESULT CALLBACK GetMsgProcHook(int code, WPARAM wParam, LPARAM lParam);
    static LRESULT CALLBACK MsgPoc(int code, WPARAM wParam, LPARAM lParam);
private:
    // some other private members there
    HWND m_windowHandle;
    bool m_isWindowResizing = false;
    static HHOOK m_msgHook;
    static WindowsDisplayHelperMasterWindow* m_pThis;
};

.cpp file:

WindowDisplayHelper* WindowDisplayHelper ::m_pThis = nullptr;
HHOOK WindowDisplayHelper ::m_msgHook = NULL;

void WindowDisplayHelper ::SetMsgHook()
{
    m_pThis = this;
    m_msgHook = SetWindowsHookEx(WH_GETMESSAGE, MsgPoc, NULL, 0);
}

LRESULT CALLBACK WindowDisplayHelper::MsgPoc(int code, WPARAM wParam, LPARAM lParam)
{
    if (m_pThis != nullptr)
    {
        return m_pThis->GetMsgProcHook(code, wParam, lParam);
    }
    return CallNextHookEx(0, code, wParam, lParam);
}

LRESULT CALLBACK WindowDisplayHelper::GetMsgProcHook(int code, WPARAM wParam, LPARAM lParam)
{
    DUMPER_INFO("Hooked");
    if (code < 0)
    {
        return CallNextHookEx(0, code, wParam, lParam);
    }
    MSG* lpmsg = (MSG*)lParam;
    DUMPER_INFO("Hooked for HWND: %p. Current window %p", lpmsg->hwnd, m_windowHandle);
    if (lpmsg->hwnd != m_windowHandle)
    {
        return CallNextHookEx(0, code, wParam, lParam);
    }
    if (lpmsg->message == WM_ENTERSIZEMOVE && !m_isWindowResizing)
    {
        DUMPER_INFO("Start window resizing");
        m_isWindowResizing = true;
    }
    else if (lpmsg->message == WM_EXITSIZEMOVE && m_isWindowResizing)
    {
        DUMPER_INFO("Stop window resizing");
        m_isWindowResizing = false;
    }

    return CallNextHookEx(0, code, wParam, lParam);
}

Here is how I create WindowDisplayHelper object:

bool DisplayManager::CreateWindowDisplay(TDisplayId displayId, void * windowHandle)
{
    auto helper = boost::make_shared<WindowDisplayHelper>(windowHandle);
    helper->SetMsgHook();
    AddDisplayHelper(displayId, helper);

    return true;
}

Though I call SetMsgHook() after the object is created, looks like hook isn't set, because I don't see any debug outputs in my log file and m_isWindowResizing variable always == false. So the question is why my hook doesn't work? Thanks.

Answer 1

m_msgHook = SetWindowsHookEx(WH_GETMESSAGE, MsgPoc, NULL, 0);

This line produce a ERROR_HOOK_NEEDS_HMOD (1428) system error . It means that Cannot set nonlocal hook without a module handle. If you set dwThreadId parameter to zero, the hook procedure is associated with all existing threads running in the same desktop as the calling thread. It is a nonload hook you need specified a valid hmod parameter. You need put the hook code in a DLL as @Remy Lebeau pointed out.

Or set a valid dwThreadId parameter using GetCurrentThreadId() as @rudolfninja pointed out.

I test based on a Windows Desktop Application Template with the following code. It works. You can have a try.

int APIENTRY wWinMain(_In_ HINSTANCE hInstance,
                     _In_opt_ HINSTANCE hPrevInstance,
                     _In_ LPWSTR    lpCmdLine,
                     _In_ int       nCmdShow)
{
    // ...

    HHOOK m_msgHook = SetWindowsHookEx(WH_GETMESSAGE, MsgPoc, NULL, GetCurrentThreadId());
    DWORD errCode = GetLastError();

    MSG msg;

    // Main message loop:
    while (GetMessage(&msg, nullptr, 0, 0))
    {
        if (!TranslateAccelerator(msg.hwnd, hAccelTable, &msg))
        {
            TranslateMessage(&msg);
            DispatchMessage(&msg);
        }
    }

    return (int) msg.wParam;
}

LRESULT CALLBACK MsgPoc(int code, WPARAM wParam, LPARAM lParam)
{
    OutputDebugString(L"Hooked");
    if (code < 0)
    {
        return CallNextHookEx(0, code, wParam, lParam);
    }
    MSG* lpmsg = (MSG*)lParam;
    //OutputDebugString("Hooked for HWND: %p. Current window %p", lpmsg->hwnd, m_windowHandle);
    if (lpmsg->hwnd != m_windowHandle)
    {
        return CallNextHookEx(0, code, wParam, lParam);
    }
    if (lpmsg->message == WM_ENTERSIZEMOVE && !m_isWindowResizing)
    {
        OutputDebugString(L"Start window resizing");
        m_isWindowResizing = true;
    }
    else if (lpmsg->message == WM_EXITSIZEMOVE && m_isWindowResizing)
    {
        OutputDebugString(L"Stop window resizing");
        m_isWindowResizing = false;
    }

    return CallNextHookEx(0, code, wParam, lParam);
}