Using detours for hooking writing text in notepad

Question

I'm trying to use detours to hook text output for example in notepad.

I wrote the following code. I will not put here all code, but the most significant parts.

DLL part:

DLLEXPORT LRESULT CALLBACK CBTProc(int nCode, WPARAM wParam, LPARAM lParam) {

if (nCode < 0) {
    return CallNextHookEx(0, nCode, wParam, lParam);
}

if (nCode == HCBT_ACTIVATE)
{
    HWND hWnd = (HWND)wParam;

    TCHAR szTemp[255];
    GetWindowText(hWnd, szTemp, 255);

    DetourTransactionBegin();
    DetourUpdateThread(hWnd);
    DetourAttach(&(PVOID&)Real_DrawText, Mine_DrawText);
    DetourTransactionCommit();

    DetourTransactionBegin();
    DetourUpdateThread(hWnd);
    DetourAttach(&(PVOID&)Real_DrawTextEx, Mine_DrawTextEx);
    DetourTransactionCommit();

    DetourTransactionBegin();
    DetourUpdateThread(hWnd);
    DetourAttach(&(PVOID&)Real_TextOut, Mine_TextOut);
    DetourTransactionCommit();

    DetourTransactionBegin();
    DetourUpdateThread(hWnd);
    DetourAttach(&(PVOID&)Real_ExtTextOut, Mine_ExtTextOut);
    DetourTransactionCommit();
}

return 0;
}

Client part:

int main(int argc, char* argv[]) {
HOOKPROC hkprcSysMsg;
static HINSTANCE hinstDLL; 
static HHOOK hhookSysMsg; 

hinstDLL = LoadLibrary(TEXT("dllsample.dll"));

//cout << (hinstDLL == NULL);

hkprcSysMsg = (HOOKPROC)GetProcAddress(hinstDLL, "_CBTProc@12");
DWORD dw = GetLastError();

//cout << (hkprcSysMsg == NULL);
//cout << dw;

hhookSysMsg = SetWindowsHookEx( 
                    WH_CBT,
                    hkprcSysMsg,
                    hinstDLL,
                    0); 

//std::cout << (hhookSysMsg == NULL);

int i;

std::cin >> i; 
}

The problem is that all 4 functions which draw text are not hooked. What do I do wrong. I'v started studing detours and didn't find answer for my question in docs.

If other parts of code are required, I'll put them here later.

Answer 1

DrawText is a macro that goes to either DrawTextW or DrawTextA depending on the UNICODE preprocessor setting. So maybe notepad is calling one, and you are hooking the other?

I think DrawTextA forwards to DrawTextW, so try hooking that directly.

edit, sample code below, compile with commands at top. run main.exe. run sysinternals debug view to see the output.

The code compiles to a dll called t4.dll and an executable called main.exe, when you run main.exe, the dll is loaded into every running process by the SetWindowHookEx call, and then the CBTProc function is called on every thread at the appropriate time.

/* t4.cpp
cl.exe /Zi /EHa /c /DUNICODE /D_UNICODE /I "c:/program files/Microsoft Research/Detours Express 2.1/include/" t4.cpp
link /DLL /DEBUG /LIBPATH:"c:/Program Files/Microsoft Visual Studio 10.0/VC/lib" /LIBPATH:"C:\Program Files\Microsoft SDKs\Windows\v7.0A\Lib" /LIBPATH:"c:/program files/Microsoft Research/Detours Express 2.1/lib" t4.obj user32.lib gdi32.lib
link /OUT:main.exe /DEBUG /LIBPATH:"c:/Program Files/Microsoft Visual Studio 10.0/VC/lib" /LIBPATH:"C:\Program Files\Microsoft SDKs\Windows\v7.0A\Lib" /LIBPATH:"c:/program files/Microsoft Research/Detours Express 2.1/lib" t4.obj user32.lib gdi32.lib
*/

#include <iostream>

#define NOMINMAX

#include <string.h>
#include <tchar.h>
#include <windows.h>
#include "detours.h"

#pragma comment(lib, "detours")
#pragma comment(lib, "detoured")

int (WINAPI *Real_DrawText)(HDC hDC, LPCTSTR lpchText, int, LPRECT, UINT) = DrawText;

BOOL (WINAPI *Real_TextOut)(
  __in  HDC hdc,
  __in  int nXStart,
  __in  int nYStart,
  __in  LPCTSTR lpString,
  __in  int cbString
  ) = TextOut;

int (WINAPI *Real_DrawTextEx)(
  __in     HDC hdc,
  __inout  LPTSTR lpchText,
  __in     int cchText,
  __inout  LPRECT lprc,
  __in     UINT dwDTFormat,
  __in     LPDRAWTEXTPARAMS lpDTParams
  ) = DrawTextEx;

BOOL (WINAPI *Real_ExtTextOut)(
  __in  HDC hdc,
  __in  int X,
  __in  int Y,
  __in  UINT fuOptions,
  __in  const RECT *lprc,
  __in  LPCTSTR lpString,
  __in  UINT cbCount,
  __in  const INT *lpDx
  ) = ExtTextOut;

int WINAPI Mine_DrawText(
  __in     HDC hDC,
  __inout  LPCTSTR lpchText,
  __in     int nCount,
  __inout  LPRECT lpRect,
  __in     UINT uFormat
  )
{
  OutputDebugString(TEXT("DrawText"));
  OutputDebugString(lpchText);
  return Real_DrawText(hDC, lpchText, nCount, lpRect, uFormat);
}

BOOL WINAPI Mine_TextOut(
  __in  HDC hdc,
  __in  int nXStart,
  __in  int nYStart,
  __in  LPCTSTR lpString,
  __in  int cbString
  )
{
  OutputDebugString(TEXT("TextOut"));
  OutputDebugString(lpString);
  return Real_TextOut(hdc, nXStart, nYStart, lpString, cbString); 
}

int WINAPI Mine_DrawTextEx(
  __in     HDC hdc,
  __inout  LPTSTR lpchText,
  __in     int cchText,
  __inout  LPRECT lprc,
  __in     UINT dwDTFormat,
  __in     LPDRAWTEXTPARAMS lpDTParams
  )
{
  OutputDebugString(TEXT("DrawTextEx"));
  OutputDebugString(lpchText);
  return Real_DrawTextEx(hdc, lpchText, cchText, lprc, dwDTFormat, lpDTParams);
}

BOOL WINAPI Mine_ExtTextOut(
  __in  HDC hdc,
  __in  int X,
  __in  int Y,
  __in  UINT fuOptions,
  __in  const RECT *lprc,
  __in  LPCTSTR lpString,
  __in  UINT cbCount,
  __in  const INT *lpDx
  )
{
  OutputDebugString(TEXT("ExtTextOut"));
  OutputDebugString(lpString);
  return Real_ExtTextOut( hdc, X, Y, fuOptions, lprc, lpString, cbCount, lpDx );
}

#define DLLEXPORT extern "C" __declspec(dllexport)

static DWORD dwTlsIndex; // address of shared memory

// Stores a DWORD in thread local storage
BOOL WINAPI StoreData(DWORD dw)
{
   LPVOID lpvData; 
   DWORD * pData;  // The stored memory pointer 

   lpvData = TlsGetValue(dwTlsIndex); 
   if (lpvData == NULL)
   {
      lpvData = (LPVOID) LocalAlloc(LPTR, 256); 
      if (lpvData == NULL) 
         return FALSE;
      if (!TlsSetValue(dwTlsIndex, lpvData))
         return FALSE;
   }

   pData = (DWORD *) lpvData; // Cast to my data type.
   // In this example, it is only a pointer to a DWORD
   // but it can be a structure pointer to contain more complicated data.

   (*pData) = dw;
   return TRUE;
}

// Retrieve a DWORD from thread local storage
BOOL WINAPI GetData(DWORD *pdw)
{
   LPVOID lpvData; 
   DWORD * pData;  // The stored memory pointer 

   lpvData = TlsGetValue(dwTlsIndex); 
   if (lpvData == NULL)
      return FALSE;

   pData = (DWORD *) lpvData;
   (*pdw) = (*pData);
   return TRUE;
}

DLLEXPORT LRESULT CALLBACK CBTProc(int nCode, WPARAM wParam, LPARAM lParam) 
{
  if (nCode < 0) {
    return CallNextHookEx(0, nCode, wParam, lParam);
  }

  // Get a flag indicating if we're already detoured.
  DWORD detoured=false;
  GetData(&detoured);

  // If the window is activating and we're not detoured...
  if (nCode == HCBT_ACTIVATE && !detoured)
  {
    OutputDebugString(TEXT("CBTProc"));
    HWND hWnd = (HWND)wParam;

    // get window title
    TCHAR szTemp[256];
    int rc = GetWindowText(hWnd, szTemp, 255);

    if (rc != 0)
    {
      OutputDebugString(szTemp);

      // hook notepad functions.
      if (_tcsstr(szTemp, TEXT("Notepad"))!=0)
      {
        OutputDebugString(TEXT("Detouring"));
        HANDLE hThread = GetCurrentThread();

        DetourTransactionBegin();
        DetourUpdateThread(hThread);
        DetourAttach(&(PVOID&)Real_DrawText, Mine_DrawText);
        DetourAttach(&(PVOID&)Real_TextOut, Mine_TextOut);
        DetourAttach(&(PVOID&)Real_DrawTextEx, Mine_DrawTextEx);
        DetourAttach(&(PVOID&)Real_ExtTextOut, Mine_ExtTextOut);
        DetourTransactionCommit();
        StoreData(true);
      }
    }
  }

  return 0;
}


BOOL WINAPI DllMain(HINSTANCE hinstDLL, // DLL module handle
  DWORD fdwReason,                    // reason called
  LPVOID lpvReserved)                 // reserved
{ 
  LPVOID lpvData; 
  BOOL fIgnore; 

  switch (fdwReason) 
  { 
    // The DLL is loading due to process 
    // initialization or a call to LoadLibrary. 
  case DLL_PROCESS_ATTACH: 
    OutputDebugString(TEXT("DLL_PROCESS_ATTACH"));
    // Allocate a TLS index.
    if ((dwTlsIndex = TlsAlloc()) == TLS_OUT_OF_INDEXES) 
      return FALSE; 
    // No break: Initialize the index for first thread.
    // The attached process creates a new thread. 
  case DLL_THREAD_ATTACH: 
    OutputDebugString(TEXT("DLL_THREAD_ATTACH"));
    // Initialize the TLS index for this thread.
    lpvData = (LPVOID) LocalAlloc(LPTR, 256); 
    if (lpvData != NULL) 
      fIgnore = TlsSetValue(dwTlsIndex, lpvData); 
    break; 
    // The thread of the attached process terminates.
  case DLL_THREAD_DETACH: 
    {
      OutputDebugString(TEXT("DLL_THREAD_DETACH"));
      DWORD detoured=false;
      GetData(&detoured);
      if(detoured)
      {
        OutputDebugString(TEXT("Un-Detouring"));
        HANDLE hThread = GetCurrentThread();

        DetourTransactionBegin();
        DetourUpdateThread(hThread);
        DetourDetach(&(PVOID&)Real_DrawText, Mine_DrawText);
        DetourDetach(&(PVOID&)Real_TextOut, Mine_TextOut);
        DetourDetach(&(PVOID&)Real_DrawTextEx, Mine_DrawTextEx);
        DetourDetach(&(PVOID&)Real_ExtTextOut, Mine_ExtTextOut);
        DetourTransactionCommit();
      }

      // Release the allocated memory for this thread.
      lpvData = TlsGetValue(dwTlsIndex); 
      if (lpvData != NULL) 
        LocalFree((HLOCAL) lpvData); 
      break; 
    }
    // DLL unload due to process termination or FreeLibrary. 
  case DLL_PROCESS_DETACH: 
    OutputDebugString(TEXT("DLL_PROCESS_DETACH"));
    // Release the allocated memory for this thread.
    lpvData = TlsGetValue(dwTlsIndex); 
    if (lpvData != NULL) 
      LocalFree((HLOCAL) lpvData); 
    // Release the TLS index.
    TlsFree(dwTlsIndex); 
    break; 
  default: 
    break; 
  } 

  return TRUE;
}

int main(int argc, char* argv[]) 
{
  HOOKPROC hkprcSysMsg;
  HHOOK hhookSysMsg; 

  HINSTANCE dll = LoadLibrary(TEXT("t4.dll"));
  hkprcSysMsg = (HOOKPROC)GetProcAddress(dll, "_CBTProc@12");
  DWORD dw = GetLastError();

  hhookSysMsg = SetWindowsHookEx( 
    WH_CBT,
    hkprcSysMsg,
    dll,
    0); 

  int i;
  std::cin >> i; 

  UnhookWindowsHookEx(hhookSysMsg);
}