JIT inline decisions might lead to StackOverflow during execution time

[echesakov]

While working on the #45184 noticed interesting behaviour.

The following IL program would run with no issues.

.assembly extern System.Runtime
{
}

.assembly Runtime_45090
{
}

.module Runtime_45090.dll

.class private explicit ansi sealed beforefieldinit Runtime_45090._00001000
  extends [System.Runtime]System.ValueType
{
  .field [0] private int32 _0
  .field [4096] private int32 _00001000

  .method public hidebysig static int32 AllocLocal(int32 a) cil managed noinlining
  {
    .maxstack  8
    .locals (valuetype Runtime_45090._00001000 V_0)

    ldloca.s   V_0
    ldflda     int32 Runtime_45090._00001000::_0
    ldind.i4
    ldarg.0
    add

    ret
  }
}

.class private auto ansi beforefieldinit Runtime_45090.Program
       extends [System.Runtime]System.Object
{
  .method public hidebysig static void Keep(int32 i) cil managed noinlining
  {
    .maxstack  8
    ret
  }

  .method private hidebysig static int32 Main(string[] args) cil managed
  {
    .entrypoint
    .maxstack  8

    ldc.i4.0
    call       int32 Runtime_45090._00001000::AllocLocal(int32)
    call       void Runtime_45090.Program::Keep(int32)
//
//  ... Repeat the sequence 0x1000 times
//
    ldc.i4.0
    call       int32 Runtime_45090._00001000::AllocLocal(int32)
    call       void Runtime_45090.Program::Keep(int32)

    ldc.i4.s   100
    ret
  }
}

Method Main is compiled down to

G_M30010_IG01:              ;; offset=0000H
       55           push     ebp
       8BEC         mov      ebp, esp
						;; bbWeight=1    PerfScore 1.25
G_M30010_IG02:              ;; offset=0003H
       33C9         xor      ecx, ecx
       E8F67E13FF   call     Runtime_45090._00001000:AllocLocal(int):int
       8BC8         mov      ecx, eax
       E817E5DAF9   call     Runtime_45090.Program:Keep(int)
       33C9         xor      ecx, ecx
       E8E87E13FF   call     Runtime_45090._00001000:AllocLocal(int):int
       8BC8         mov      ecx, eax
//
// ... Same sequence of instruction
//

       8BC8         mov      ecx, eax
       E82505DAF9   call     Runtime_45090.Program:Keep(int)
       B864000000   mov      eax, 100
						;; bbWeight=1    PerfScore 40.00
G_M30010_IG243:              ;; offset=E008H
       5D           pop      ebp
       C3           ret      

During the program execution Runtime_45090._00001000::AllocLocal is called one at a time and the required stack size should not exceed two page sizes.

However, if AllocLocal didn't noinlining attribute, all the callees would be inlined into Main:

G_M30010_IG01:              ;; offset=0000H
       55           push     ebp
       8BEC         mov      ebp, esp
       8D84240408E0FF lea      eax, [esp-1FF7FCH]
       E87769646F   call     CORINFO_HELP_STACK_PROBE
       8BE0         mov      esp, eax
						;; bbWeight=1    PerfScore 3.00
G_M30010_IG02:              ;; offset=0011H
       8B8DFCEFFFFF mov      ecx, dword ptr [ebp-1004H]
       E80CE5CEFE   call     Runtime_45090.Program:Keep(int)
       8B8DF8DFFFFF mov      ecx, dword ptr [ebp-2008H]
       E801E5CEFE   call     Runtime_45090.Program:Keep(int)
       8B8DF4CFFFFF mov      ecx, dword ptr [ebp-300CH]
       E8F6E4CEFE   call     Runtime_45090.Program:Keep(int)
       8B8DF0BFFFFF mov      ecx, dword ptr [ebp-4010H]
       E8EBE4CEFE   call     Runtime_45090.Program:Keep(int)
       8B8DECAFFFFF mov      ecx, dword ptr [ebp-5014H]
       E8E0E4CEFE   call     Runtime_45090.Program:Keep(int)
       8B8DE89FFFFF mov      ecx, dword ptr [ebp-6018H]
       E8D5E4CEFE   call     Runtime_45090.Program:Keep(int)
       8B8DE48FFFFF mov      ecx, dword ptr [ebp-701CH]
       E8CAE4CEFE   call     Runtime_45090.Program:Keep(int)
       8B8DE07FFFFF mov      ecx, dword ptr [ebp-8020H]
       E8BFE4CEFE   call     Runtime_45090.Program:Keep(int)
       8B8DDC6FFFFF mov      ecx, dword ptr [ebp-9024H]
       E8B4E4CEFE   call     Runtime_45090.Program:Keep(int)
       8B8DD85FFFFF mov      ecx, dword ptr [ebp-A028H]

increasing the required stack size to 2 Megabytes and causing the program to fail with StackOverflow during its execution.

@dotnet/jit-contrib Should the JIT take into account the inlinee stack frame size (at least its rough estimate) when making inline decisions?

Inline.txt
NoInline.txt

category:cq
theme:inlining
skill-level:expert
cost:medium

[AndyAyersMS]

Interesting, but I'm not convinced we need to fix this. The inliner behavior has been this way for many many years and I don't recall anyone running into this problem.