The code is a MIPS->ARM dynamic recompiler. After many times of running recompile_function(), it crashes at the condition clause of below code, though it can run this line of code without any issue during the earlier function running.
void recompile_function(){
//recompilation code
......
if (out > (u_char *)((u_char *)base_addr + (1 << TARGET_SIZE_2) - MAX_OUTPUT_BLOCK_SIZE - JUMP_TABLE_SIZE))
out = (u_char *)base_addr;
// other code
......
}
Variable out is the pointer used to write the recompiled code. base_addr always points to the original start of the allocated memory space. Variable out progresses 4 bytes each time an instruction is written, while base_addr keeps unchanged.
extern char extra_memory[33554432];
#define BASE_ADDR ((int)(&extra_memory))
void *base_addr;
u_char *out;
void new_dynarec_init()
{
protect_readwrite();
base_addr = ((int)(&extra_memory));
out = (u_char *)base_addr;
}
The error is "Unhandled exception at 0x7738EC9F (ntdll.dll) in frontend.exe: 0xC0000008: An invalid handle was specified."
This is the disassemble code around the faulting clause instruction.
#if NEW_DYNAREC == NEW_DYNAREC_ARM
__clear_cache((void *)beginning, out);
53830242 ldr r1,[r9]
53830246 add r3,r4,r5,lsl #2
5383024A mov r0,r7
5383024C str r3,[r2]
5383024E blx __clear_cache_bugfix (537D19DCh)
//cacheflush((void *)beginning,out,0);
#endif
// If we're within 256K of the end of the buffer,
// start over from the beginning. (Is 256K enough?)
if (out > (u_char *)((u_char *)base_addr + (1 << TARGET_SIZE_2) - MAX_OUTPUT_BLOCK_SIZE - JUMP_TABLE_SIZE))
53830252 mov r2,#0xAA98
53830256 movt r2,#0x5462
5383025A ldr r3,new_recompile_block+0A1E8h (53830550h)
5383025C ldr r4,[r2]
5383025E ldr r2,[r9]
53830262 add r3,r3,r4
53830264 cmp r2,r3
53830266 bls new_recompile_block+9F06h (5383026Eh)
out = (u_char *)base_addr;
53830268 mov r2,r4
5383026A str r4,[r9]
It's the line the debugger prompts me. I checked the disassemble window that also points to this line. What's more, if I choose continue, a new error will pop up and the program will crash at the code line "__fastfail" in function __report_gsfailure. The new error is "Unhandled exception at 0x53831547 (mupen64plus.dll) in frontend.exe: Stack cookie instrumentation code detected a stack-based buffer overrun". 0x53831546 is the address of code line "__fastfail".
#pragma warning(push)
#pragma warning(disable: 4100) // unreferenced formal parameter
__declspec(noreturn) void __cdecl __report_gsfailure(GSFAILURE_PARAMETER)
{
5383153C push {r0,r1}
5383153E push {r11,lr}
53831542 mov r11,sp
__fastfail(FAST_FAIL_STACK_COOKIE_CHECK_FAILURE);
53831544 movs r0,#2
53831546 __fastfail
}
// Declare stub for rangecheckfailure, since these occur often enough that the
// code bloat of setting up the parameters hurts performance
__declspec(noreturn) void __cdecl __report_rangecheckfailure()
{
53831548 push {r11,lr}
5383154C mov r11,sp
__report_securityfailure(FAST_FAIL_RANGE_CHECK_FAILURE);
5383154E movs r0,#8
53831550 bl __report_securityfailure (53831558h)
53831554 __debugbreak
The register PC = 53831546 so the execution point is __fastfail.
