Detouring a member-function via an injected DLL

Question

Original Post:

I'm trying to detour a member-function from within my injected DLL. I've got the address of the function I'm trying to hook, but cannot figure out the proper syntax or way about hooking it via detours library. I've commented the line that's giving me the error with the error message.

I've read the source for the detours example of member-function hooking and that's what this code is based upon, but for some reason it's not working.

Any help would be much appreciated, thanks!

#include <windows.h>
#include <detours.h>

class CDetour
{
public:
    bool My_MemFn(unsigned int unk1);
    static bool (CDetour::* Real_MemFn)(unsigned int);
};

bool CDetour::My_MemFn(unsigned int unk1)
{
        /* do stuff here */
    return (this->*Real_MemFn)(unk1);
}

typedef bool (CDetour::* MemFn_t)(unsigned int unk1);

MemFn_t CDetour::Real_MemFn= *(MemFn_t *)((void*)0x23234545);

BOOL APIENTRY DllMain(HANDLE hModule, DWORD dwReason, LPVOID lpReserved)
{
    switch (dwReason)
    {
        case DLL_PROCESS_ATTACH:
        {
            DetourTransactionBegin();
            DetourUpdateThread(GetCurrentThread());
            DetourAttach(&(PVOID&)CDetour::Real_MemFn, *(PBYTE*)&CDetour::My_MemFn); // ERROR: C2440: 'type cast' : cannot convert from 'bool __thiscall CDetour::* )(unsigned int)' to 'PBYTE *'
            DetourTransactionCommit();
            break;
        }
    }

    return TRUE;
}

Solution:

#include <windows.h>
#include <detours.h>

typedef void (__thiscall * CClassFunction_t)(void *__this, unsigned int unk1);
CClassFunction_t Real_CClassFunction;

void __fastcall Mine_CClassFunction(void *__this, int edx, unsigned int unk1)
{
    Real_CClassFunction(__this, unk1);
}

template<typename T>
void HookFunction(const char *module, char *signature, T &fn_real, PVOID fn_mine)
{
    HookFunction<T>(DetourFindFunction(module, signature), fn_real, fn_mine);
}

template<typename T>
void HookFunction(DWORD address, T &fn_real, PVOID fn_mine)
{
    HookFunction<T>(reinterpret_cast<PVOID>(address), fn_real, fn_mine);
}

template<typename T>
void HookFunction(PVOID target, T &fn_real, PVOID fn_mine)
{
    fn_real = reinterpret_cast<T>(target);

    HookFunction<T>(fn_real, fn_mine);
}

template<typename T>
void HookFunction(T &fn_real, PVOID fn_mine)
{
    DetourAttach(&(PVOID&)fn_real, fn_mine);
}


void ApplyHooks()
{
    DetourTransactionBegin();
    DetourUpdateThread(GetCurrentThread());

    DWORD function_address = 0x12345678;

    HookFunction<CClassFunction_t>(address, Real_CClassFunction, Mine_CClassFunction);

    DetourTransactionCommit();
}

BOOL APIENTRY DllMain(HINSTANCE hInstance, DWORD dwReason, LPVOID lpReserved)
{
    switch (dwReason)
    {
        case DLL_PROCESS_ATTACH:
        case DLL_THREAD_ATTACH:
        {
            DisableThreadLibraryCalls(hInstance);

            CreateThread(0, 0, (LPTHREAD_START_ROUTINE)ApplyHooks, 0, 0, 0);

            break;
        }
    }

    return TRUE;
}

Answer 1

Try using:

union { bool (CDetour::*lpMyFunction)(unsigned int); PBYTE lpAddr; } u;

save the pointer on u.lpMyFunction and get it from u.lpAddr

Does this solve your compilation problem?

Answer 2

After hours of searching and trying to find a solution, I've come up with this nice little solution:

#include <windows.h>
#include <detours.h>

typedef void (__thiscall * CClassFunction_t)(void *__this, unsigned int unk1);
CClassFunction_t Real_CClassFunction;

void __fastcall Mine_CClassFunction(void *__this, int edx, unsigned int unk1)
{
    Real_CClassFunction(__this, unk1);
}

template<typename T>
void HookFunction(const char *module, char *signature, T &fn_real, PVOID fn_mine)
{
    HookFunction<T>(DetourFindFunction(module, signature), fn_real, fn_mine);
}

template<typename T>
void HookFunction(DWORD address, T &fn_real, PVOID fn_mine)
{
    HookFunction<T>(reinterpret_cast<PVOID>(address), fn_real, fn_mine);
}

template<typename T>
void HookFunction(PVOID target, T &fn_real, PVOID fn_mine)
{
    fn_real = reinterpret_cast<T>(target);

    HookFunction<T>(fn_real, fn_mine);
}

template<typename T>
void HookFunction(T &fn_real, PVOID fn_mine)
{
    DetourAttach(&(PVOID&)fn_real, fn_mine);
}


void ApplyHooks()
{
    DetourTransactionBegin();
    DetourUpdateThread(GetCurrentThread());

    DWORD function_address = 0x12345678;

    HookFunction<CClassFunction_t>(address, Real_CClassFunction, Mine_CClassFunction);

    DetourTransactionCommit();
}

BOOL APIENTRY DllMain(HINSTANCE hInstance, DWORD dwReason, LPVOID lpReserved)
{
    switch (dwReason)
    {
        case DLL_PROCESS_ATTACH:
        case DLL_THREAD_ATTACH:
        {
            DisableThreadLibraryCalls(hInstance);

            CreateThread(0, 0, (LPTHREAD_START_ROUTINE)ApplyHooks, 0, 0, 0);

            break;
        }
    }

    return TRUE;
}

Answer 3

Yeah, I actually hate those casts, so I made up my own:

// Cast a member function pointer that cannot have a reference taken to a void *
template <typename RET_TYPE, typename CLASS, typename...ARGs>
void* castToVoidPtr(RET_TYPE(CLASS::*&&pOriginalFunction)(ARGs...))
{
    union
    {
        RET_TYPE(CLASS::*pMemFn)(ARGs...);
        void* voidPtr;
    } cast = { pOriginalFunction };
    static_assert(sizeof(cast.pMemFn) == sizeof(cast.voidPtr), "Cannot cast this member function pointer to a void*.  Not the same size.");
    return cast.voidPtr;
}

// Cast a member function pointer to a void*&
template <typename RET_TYPE, typename CLASS, typename...ARGs>
void*& castToVoidPtr(RET_TYPE(CLASS::*&pOriginalFunction)(ARGs...))
{
    union
    {
        RET_TYPE(CLASS::*&pMemFn)(ARGs...);
        void*& voidPtr;
    } cast = { pOriginalFunction };
    static_assert(sizeof(cast.pMemFn) == sizeof(cast.voidPtr), "Cannot cast this member function pointer to a void*.  Not the same size.");
    return cast.voidPtr;
}

The only problem with your solution is that you're pushing an extra DWORD onto the stack (the EDX register), which is unnecessary. These casts should work in all cases that you will be using it in. From what I gather, it will not work for a multiple inherited class function, which is not something you are likely to use for detouring and will assert if you were to try.

This will allow you to do this:

DetourAttach(&castToVoidPtr(CDetour::Real_MemFn), castToVoidPtr(&CDetour::My_MemFn));

Answer 4

"void* castToVoidPtr(RET_TYPE(CLASS::*&&pOriginalFunction)(ARGs...))"

is this && exactly what you want to write? VS2008 gives an error when compiling.