[amd64] fix tailcall insn size (mono#5483)

commit
mono@9a634c1
introduces a regression for
https://github.com/mono/coreclr/blob/mono/tests/src/JIT/Methodical/Invoke/25params/25param1c.il

```
Testing method of 25 parameters, all of int data type, tail.call
mono-sgen(46435,0x7fff9d34b3c0) malloc: *** error for object 0x101045400: incorrect checksum for freed object - object was probably modified after being freed.
*** set a breakpoint in malloc_error_break to debug
Process 46435 stopped
* thread #1, name = 'tid_307', queue = 'com.apple.main-thread', stop reason = breakpoint 1.1
    frame #0: 0x00007fff945c915f libsystem_malloc.dylib`malloc_error_break
libsystem_malloc.dylib`malloc_error_break:
->  0x7fff945c915f <+0>: pushq  %rbp
    0x7fff945c9160 <+1>: movq   %rsp, %rbp
    0x7fff945c9163 <+4>: nop
    0x7fff945c9164 <+5>: nopl   (%rax)
(lldb) mbt
* thread #1
  * frame #0: 0x00007fff945c915f libsystem_malloc.dylib`malloc_error_break
    frame #1: 0x00007fff945c5e81 libsystem_malloc.dylib`szone_error + 406
    frame #2: 0x00007fff945c7925 libsystem_malloc.dylib`small_free_list_remove_ptr_no_clear + 766
    frame #3: 0x00007fff945c7cb2 libsystem_malloc.dylib`free_small + 881
    frame #4: 0x00000001004d2110 mono-sgen`monoeg_g_free(ptr=0x0000000101083c00) at gmem.c:66
    frame mono#5: 0x0000000100007b64 mono-sgen`mono_codegen(cfg=0x000000010108c800) at mini.c:2300
    frame mono#6: 0x000000010000aedf mono-sgen`mini_method_compile(method=0x0000000100910310, opts=370239999, domain=0x000000010090ebd0, flags=JIT_FLAG_RUN_CCTORS, parts=0, aot_method_index=-1) at mini.c:3829
    frame mono#7: 0x000000010000edd0 mono-sgen`mono_jit_compile_method_inner(method=0x0000000100910310, target_domain=0x000000010090ebd0, opt=370239999, error=0x00007fff5fbfe048) at mini.c:4156
    frame mono#8: 0x000000010001459e mono-sgen`mono_jit_compile_method_with_opt(method=0x0000000100910310, opt=370239999, jit_only=0, error=0x00007fff5fbfe048) at mini-runtime.c:2127
    frame mono#9: 0x0000000100013e6d mono-sgen`mono_jit_compile_method(method=0x0000000100910310, error=0x00007fff5fbfe048) at mini-runtime.c:2173
    frame mono#10: 0x0000000100131eea mono-sgen`common_call_trampoline(regs=0x00007fff5fbfe128, code="H\x8bй\x01", m=0x0000000100910310, vt=0x0000000000000000, vtable_slot=0x0000000000000000, error=0x00007fff5fbfe048) at mini-trampolines.c:704
    frame mono#11: 0x00000001001313b7 mono-sgen`mono_magic_trampoline(regs=0x00007fff5fbfe128, code="H\x8bй\x01", arg=0x0000000100910310, tramp="����\b\x10\x03\x91") at mini-trampolines.c:835
```

turns out we smashed our code buffer.

Also, transform this comparison:

```
if (G_UNLIKELY (offset > (cfg->code_size - max_len - EXTRA_CODE_SPACE))) {

if (G_UNLIKELY ((offset + max_len + EXTRA_CODE_SPACE) > cfg->code_size)) {
```

we deal with unsigned values here, and if `max_len` is bigger then
`cfg->code_size`, we won't resize the buffer.

mono/mini/mini-amd64.c

@@ -3650,7 +3650,7 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
 
 #define EXTRA_CODE_SPACE (16)
 
-		if (G_UNLIKELY (offset > (cfg->code_size - max_len - EXTRA_CODE_SPACE))) {
+		if (G_UNLIKELY ((offset + max_len + EXTRA_CODE_SPACE) > cfg->code_size)) {
 			cfg->code_size *= 2;
 			cfg->native_code = (unsigned char *)mono_realloc_native_code(cfg);
 			code = cfg->native_code + offset;
@@ -4487,6 +4487,17 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
 
 			g_assert (!cfg->method->save_lmf);
 
+			/* the size of the tailcall op depends on signature, let's check for enough
+			 * space in the code buffer here again */
+			max_len += AMD64_NREG * 4 + call->stack_usage * 15 + EXTRA_CODE_SPACE;
+
+			if (G_UNLIKELY (offset + max_len > cfg->code_size)) {
+				cfg->code_size *= 2;
+				cfg->native_code = (unsigned char *) mono_realloc_native_code(cfg);
+				code = cfg->native_code + offset;
+				cfg->stat_code_reallocs++;
+			}
+
 			/* Restore callee saved registers */
 			save_area_offset = cfg->arch.reg_save_area_offset;
 			for (i = 0; i < AMD64_NREG; ++i)
@@ -4517,7 +4528,6 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
 #endif
 			}
 
-			offset = code - cfg->native_code;
 			mono_add_patch_info (cfg, code - cfg->native_code, MONO_PATCH_INFO_METHOD_JUMP, call->method);
 			if (cfg->compile_aot)
 				amd64_mov_reg_membase (code, AMD64_R11, AMD64_RIP, 0, 8);