Setting a hardwarebreakpoint in multithreaded application doesn't fire

Question

I wrote a little debugger for analysing and looging certain problems. Now I implemented a hardwarebreakpoint for detecting the access of a memory address being overwritten. When I run my debugger with a test process, then everything works fine. When I access the address, the breakpoint fires and the callstack is logged. The problem is, when I run the same against an application running multiple threads. I'm replicating the breakpoint into every thread that gets created and also the main thread. None of the functions report an error and everything looks fine, but when the address is accessed, the breakpoint never fires.

So I wonder if there is some documentation where this is described or if there are additionaly things that I have to do in case of a multithreaded application.

The function to set the breakpoint is this:

#ifndef _HARDWARE_BREAKPOINT_H
#define _HARDWARE_BREAKPOINT_H

#include "breakpoint.h"

#define MAX_HARDWARE_BREAKPOINT     4

#define REG_DR0_BIT                 1
#define REG_DR1_BIT                 4
#define REG_DR2_BIT                 16
#define REG_DR3_BIT                 64


class HardwareBreakpoint : public Breakpoint
{
public:
    typedef enum 
    {
        REG_INVALID     = -1,
        REG_DR0         =  0,
        REG_DR1         =  1,
        REG_DR2         =  2,
        REG_DR3         =  3
    } Register;

    typedef enum
    {
        CODE,
        READWRITE,
        WRITE,
    } Type;

    typedef enum
    {
        SIZE_1,
        SIZE_2,
        SIZE_4,
        SIZE_8,
    } Size;

    typedef struct 
    {
        void *pAddress;
        bool bBusy;
        Type nType;
        Size nSize;
        Register nRegister;
    } Info;

public:
    HardwareBreakpoint(HANDLE hThread);
    virtual ~HardwareBreakpoint(void);

    /**
     * Sets a hardware breakpoint. If no register is free or an error occured
     * REG_INVALID is returned, otherwise the hardware register for the given breakpoint.
     */
    HardwareBreakpoint::Register set(void *pAddress, Type nType, Size nSize);
    void remove(void *pAddress);
    void remove(Register nRegister);

    inline Info const *getInfo(Register nRegister) const { return &mBreakpoint[nRegister]; }

private:
    typedef Breakpoint super;

private:
    Info mBreakpoint[MAX_HARDWARE_BREAKPOINT];
    size_t mRegBit[MAX_HARDWARE_BREAKPOINT];
    size_t mRegOffset[MAX_HARDWARE_BREAKPOINT];
};
#endif // _HARDWARE_BREAKPOINT_H

void SetBits(DWORD_PTR &dw, size_t lowBit, size_t bits, size_t newValue)
{
    DWORD_PTR mask = (1 << bits) - 1; 
    dw = (dw & ~(mask << lowBit)) | (newValue << lowBit);
}

HardwareBreakpoint::HardwareBreakpoint(HANDLE hThread)
    : super(hThread)
{
    mRegBit[REG_DR0] = REG_DR0_BIT;
    mRegBit[REG_DR1] = REG_DR1_BIT;
    mRegBit[REG_DR2] = REG_DR2_BIT;
    mRegBit[REG_DR3] = REG_DR3_BIT;

    CONTEXT ct;
    mRegOffset[REG_DR0] = reinterpret_cast<size_t>(&ct.Dr0) - reinterpret_cast<size_t>(&ct);
    mRegOffset[REG_DR1] = reinterpret_cast<size_t>(&ct.Dr1) - reinterpret_cast<size_t>(&ct);
    mRegOffset[REG_DR2] = reinterpret_cast<size_t>(&ct.Dr2) - reinterpret_cast<size_t>(&ct);
    mRegOffset[REG_DR3] = reinterpret_cast<size_t>(&ct.Dr3) - reinterpret_cast<size_t>(&ct);

    memset(&mBreakpoint[0], 0, sizeof(mBreakpoint));
    for(int i = 0; i < MAX_HARDWARE_BREAKPOINT; i++)
        mBreakpoint[i].nRegister = (Register)i;
}

HardwareBreakpoint::Register HardwareBreakpoint::set(void *pAddress, Type nType, Size nSize)
{
    CONTEXT ct = {0};
    super::setAddress(pAddress);

    ct.ContextFlags = CONTEXT_DEBUG_REGISTERS;
    if(!GetThreadContext(getThread(), &ct))
        return HardwareBreakpoint::REG_INVALID;

    size_t iReg = 0;
    for(int i = 0; i < MAX_HARDWARE_BREAKPOINT; i++)
    {
        if (ct.Dr7 & mRegBit[i])
            mBreakpoint[i].bBusy = true;
        else
            mBreakpoint[i].bBusy = false;
    }

    Info *reg = NULL;

    // Address already used?
    for(int i = 0; i < MAX_HARDWARE_BREAKPOINT; i++)
    {
        if(mBreakpoint[i].pAddress == pAddress)
        {
            iReg = i;
            reg = &mBreakpoint[i];
            break;
        }
    }

    if(reg == NULL)
    {
        for(int i = 0; i < MAX_HARDWARE_BREAKPOINT; i++)
        {
            if(!mBreakpoint[i].bBusy)
            {
                iReg = i;
                reg = &mBreakpoint[i];
                break;
            }
        }
    }

    // No free register available
    if(!reg)
        return HardwareBreakpoint::REG_INVALID;

    *(void **)(((char *)&ct)+mRegOffset[iReg]) = pAddress;
    reg->bBusy = true;

    ct.Dr6 = 0;
    int st = 0;
    if (nType == CODE)
        st = 0;
    if (nType == READWRITE)
        st = 3;
    if (nType == WRITE)
        st = 1;

    int le = 0;
    if (nSize == SIZE_1)
        le = 0;
    else if (nSize == SIZE_2)
        le = 1;
    else if (nSize == SIZE_4)
        le = 3;
    else if (nSize == SIZE_8)
        le = 2;

    SetBits(ct.Dr7, 16 + iReg*4, 2, st);
    SetBits(ct.Dr7, 18 + iReg*4, 2, le);
    SetBits(ct.Dr7, iReg*2, 1, 1);

    ct.ContextFlags = CONTEXT_DEBUG_REGISTERS;
    if(!SetThreadContext(getThread(), &ct))
        return REG_INVALID;

    return reg->nRegister;
}

I'm setting the breakpoint in the main debugger loop whenever a new thread is created CREATE_THREAD_DEBUG_EVENT but looking at the sourcecode of GDB it seems not to be done there, so maybe that is to early?

Answer 1

So I finally found the answer to this problem.

In the debug event loop, I'm monitoring the events that windows sends me. One of those events is CREATE_THREAD_DEBUG_EVENT which I used to set the hardware breakpoint whenever a new thread was created.

The problem is, that the notification of this event comes before the thread got actually started. So Windows is setting the context for the first time AFTER this event is sent, which of course overwrites any context data that I have set before.

The solution I implemented now is, when a CREATE_THREAD_DEBUG_EVENT comes I put a software breakpoint at the start adress of the thread, so that the first instruction is my breakpoint. When I receive the breakpoint event, I restore the original code and install the hardware breakpoint, which now fires fine.

If there is a better solution, I'm all ears. :)