Escaping trapflag/single step

Question

I'm writing a program that traces the execution of other programs. I'm using dynamic instruction instrumentation to track the behavior of x86's CMP instruction.

I'm using the windows debugging api to control the behavior of the debugged program. I start the program with the 'debug only this process' flag, and then set the trap flag on the main thread.

I then enter the main debugging loop:

bool cDebugger::ProcessNextDebugEvent(bool Verbose)
{
    bool Result = true;
    DEBUG_EVENT Event = { 0 };

    DWORD Status = DBG_CONTINUE;

    if (!WaitForDebugEvent(&Event, INFINITE))
    {
        _Reporter("Error: WaitForDebugEvent: " + to_string(GetLastError()));
        return Result;
    }
    else
    {
        if (Event.dwDebugEventCode == CREATE_PROCESS_DEBUG_EVENT)
        {
            if (Verbose)
                _Reporter("Created process: " + GetFilenameFromHandle(Event.u.CreateProcessInfo.hFile));
        }
        else if (Event.dwDebugEventCode == LOAD_DLL_DEBUG_EVENT)
        {
            if (Verbose)
                _Reporter("Dll: " + GetFilenameFromHandle(Event.u.LoadDll.hFile) + " loaded at: " + to_string((unsigned int)Event.u.LoadDll.lpBaseOfDll));

            _Dlls.insert(make_pair((unsigned int)Event.u.LoadDll.lpBaseOfDll, GetFilenameFromHandle(Event.u.LoadDll.hFile)));
        }
        else if (Event.dwDebugEventCode == CREATE_THREAD_DEBUG_EVENT)
        {
            if (Verbose)
                _Reporter("Thread[" + to_string(Event.dwThreadId) + "] created at: " + to_string((unsigned int)Event.u.CreateThread.lpStartAddress));

            _Threads.push_back(Event.dwThreadId);
        }
        else if (Event.dwDebugEventCode == EXIT_THREAD_DEBUG_EVENT)
        {
            if (Verbose)
                _Reporter("Thread[" + to_string(Event.dwThreadId) + "] exited with: " + to_string(Event.u.ExitThread.dwExitCode));

            auto It = std::find(_Threads.begin(), _Threads.end(), Event.dwThreadId);

            if (It != _Threads.end())
                _Threads.erase(It);
        }
        else if (Event.dwDebugEventCode == UNLOAD_DLL_DEBUG_EVENT)
        {
            if (Verbose)
                _Reporter("Dll " + _Dlls[(unsigned int)Event.u.UnloadDll.lpBaseOfDll] + " unloaded at : " + to_string((unsigned int)Event.u.UnloadDll.lpBaseOfDll));
        }
        else if (Event.dwDebugEventCode == EXIT_PROCESS_DEBUG_EVENT)
        {
            if (Verbose)
                _Reporter("Process exited with: " + to_string(Event.u.ExitProcess.dwExitCode));

            Result = false;

            _Threads.clear();
        }
        else if (Event.dwDebugEventCode == EXCEPTION_DEBUG_EVENT)
        {
            if (Event.u.Exception.ExceptionRecord.ExceptionCode == EXCEPTION_SINGLE_STEP)
            {
                Status = DBG_EXCEPTION_HANDLED;
            }
            else
            {
                Status = DBG_EXCEPTION_NOT_HANDLED;
            }
        }

        for (size_t i = 0; i < _Threads.size(); i++)
        {
            HANDLE hThread = OpenThread(THREAD_ALL_ACCESS, FALSE, _Threads[i]);

            if (hThread == NULL)
            {
                _Reporter("Error: Failed to open thread: " + to_string(GetLastError()));
            }
            else
            {
                CONTEXT ThreadContext = GetThreadContext(hThread);

                ProcessStep(ThreadContext, hThread);

                ThreadContext.EFlags |= 0x100; // Set trap flag.
                SetThreadContext(hThread, ThreadContext);

                CloseHandle(hThread);
            }
        }

        if (!ContinueDebugEvent(Event.dwProcessId, Event.dwThreadId, Status))
        {
            _Reporter("Error: ContinueDebugEvent: " + to_string(GetLastError()));
        }
    }

    return Result;
}

As you can see I loop through all threads at the end of the function to make sure that the single-step exception will trigger on every next instruction in every thread. However sometimes execution seems to 'escape' this trap, often executing millions of instructions before being caught again by the next debugging event.

I wrote another small application to test the behavior of my program:

int main(int argc, char* argv[])
{

    //__asm int 3h
    if (argc == 41234123)
    {
        printf("Got one\n");
    }

    return 0;
}

The expected output of the tracer should be:

0xDEADBEEF CMP 1 41234123

However somehow the tracer does not record this instruction (indicating that no debug event was raised, and that the trap flag was not set).

Can anybody see if I'm doing something wrong in my debug loop? Or what kind of behavior of the test program (loading of a dll) could be responsible for this?

Answer 1

The problem had something to do with code entering kernel space when calling windows apis. My solution was to set the page protection of the executable section of the test program to PAGE_GUARD:

    SYSTEM_INFO Info;
    GetSystemInfo(&Info);

    DWORD StartAddress = (DWORD)Info.lpMinimumApplicationAddress;
    DWORD StopAddress = (DWORD)Info.lpMaximumApplicationAddress;
    DWORD PageSize = 0;

    PageSize = Info.dwPageSize;

    _Sections.clear();

    for (DWORD AddressPointer = StartAddress; AddressPointer < StopAddress; AddressPointer += PageSize)
    {
        MEMORY_BASIC_INFORMATION Buffer;
        VirtualQueryEx(_Process.GetHandle(), (LPCVOID)AddressPointer, &Buffer, sizeof(Buffer));

        if (CheckBit(Buffer.Protect, 4) || CheckBit(Buffer.Protect, 5) || CheckBit(Buffer.Protect, 6) || CheckBit(Buffer.Protect, 7))
        {
            if (Buffer.State == MEM_COMMIT)
            {
                _Sections.push_back(make_pair((unsigned int)Buffer.BaseAddress, (unsigned int)Buffer.RegionSize));
                AddressPointer = (unsigned int)Buffer.BaseAddress + (unsigned int)Buffer.RegionSize;
            }
        }
    }


void cDebugger::SetPageGuard()
{
    for (size_t i = 0; i < _Sections.size(); i++)
    {
        DWORD Dummy;
        VirtualProtectEx(_Process.GetHandle(), (LPVOID)_Sections[i].first, _Sections[i].second, PAGE_GUARD | PAGE_EXECUTE_READWRITE, &Dummy);
    }
}

This way I regain control because the system will fire a EXCEPTION_GUARD_PAGE when execution returns to a guarded page.

if (Event.u.Exception.ExceptionRecord.ExceptionCode == EXCEPTION_SINGLE_STEP)
            {
                Status = DBG_CONTINUE;
                if (!_Tracing)
                {
                    HANDLE hThread = OpenThread(THREAD_ALL_ACCESS, FALSE, Event.dwThreadId);
                    CONTEXT ThreadContext = GetThreadContext(hThread);

                    if (ThreadContext.Eip == _EntryAddress)
                    {
                        ClearHardwareBreakpoint(0, hThread);
                        _Tracing = true;
                    }

                    CloseHandle(hThread);
                }

                SetPageGuard();

                _Guarded = true;
            }
            else if (Event.u.Exception.ExceptionRecord.ExceptionCode == EXCEPTION_BREAKPOINT)
            {
                Status = DBG_CONTINUE;
            }
            else if (Event.u.Exception.ExceptionRecord.ExceptionCode == EXCEPTION_GUARD_PAGE)
            {
                Status = DBG_CONTINUE;   // fires when processor lands on guarded pages
            }
            else
            {
                Status = DBG_EXCEPTION_NOT_HANDLED;
            }

This solution is not perfect. There are possibly still some situations under which execution can still escape the 'trap'. But it solved my most immediate problem (being able to see the comparisons in my test program).