Fix IJW OverflowException with 17+ by-ref parameters

src/coreclr/vm/stubgen.cpp

@@ -2003,9 +2003,13 @@ void StubSigBuilder::EnsureEnoughQuickBytes(size_t cbToAppend)
     STANDARD_VM_CONTRACT;
 
     SIZE_T cbBuffer = m_qbSigBuffer.Size();
-    if ((m_cbSig + cbToAppend) >= cbBuffer)
+    if ((cbBuffer - m_cbSig) < cbToAppend)
     {
-        m_qbSigBuffer.ReSizeThrows(2 * cbBuffer);
+        SIZE_T cbNew = max(m_cbSig + cbToAppend, 2 * cbBuffer);
+        // Detect integer overflow
+        if ((cbNew - m_cbSig) < cbToAppend)
+            COMPlusThrowHR(COR_E_OVERFLOW);
+        m_qbSigBuffer.ReSizeThrows(cbNew);
         m_pbSigCursor = ((BYTE*)m_qbSigBuffer.Ptr()) + m_cbSig;
     }
 }

src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp

@@ -16,6 +16,45 @@ extern "C" DLL_EXPORT int __cdecl NativeEntryPoint()
     return NativeFunction();
 }
 
+struct IntWrapper
+{
+    int Value;
+};
+
+#pragma managed
+int64_t ManagedSum18ByRef(const int& a0,
+    const IntWrapper& a1,
+    const IntWrapper& a2,
+    const IntWrapper& a3,
+    const IntWrapper& a4,
+    const IntWrapper& a5,
+    const IntWrapper& a6,
+    const IntWrapper& a7,
+    const IntWrapper& a8,
+    const IntWrapper& a9,
+    const IntWrapper& a10,
+    const IntWrapper& a11,
+    const IntWrapper& a12,
+    const IntWrapper& a13,
+    const IntWrapper& a14,
+    const IntWrapper& a15,
+    const IntWrapper& a16,
+    const int& a17);
+
+#pragma unmanaged
+int64_t NativeSum18ByRef()
+{
+    int a0 = 0;
+    IntWrapper w[16];
+    for (int i = 0; i < 16; ++i)
+    {
+        w[i].Value = i + 1;
+    }
+    int a17 = 17;
+    return ManagedSum18ByRef(a0, w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7],
+        w[8], w[9], w[10], w[11], w[12], w[13], w[14], w[15], a17);
+}
+
 #pragma managed
 
 // Needed to provide a regression case for https://github.com/dotnet/runtime/issues/110365
@@ -39,6 +78,11 @@ public ref class TestClass
         return NativeFunction();
     }
 
+    int64_t ManagedEntryPointSum18ByRef()
+    {
+        return NativeSum18ByRef();
+    }
+
     static void ChangeReturnedValue(int i)
     {
         s_valueToReturnStorage.valueToReturn = i;
@@ -58,3 +102,26 @@ int ManagedCallee()
 {
     return TestClass::GetReturnValue();
 }
+
+int64_t ManagedSum18ByRef(const int& a0,
+    const IntWrapper& a1,
+    const IntWrapper& a2,
+    const IntWrapper& a3,
+    const IntWrapper& a4,
+    const IntWrapper& a5,
+    const IntWrapper& a6,
+    const IntWrapper& a7,
+    const IntWrapper& a8,
+    const IntWrapper& a9,
+    const IntWrapper& a10,
+    const IntWrapper& a11,
+    const IntWrapper& a12,
+    const IntWrapper& a13,
+    const IntWrapper& a14,
+    const IntWrapper& a15,
+    const IntWrapper& a16,
+    const int& a17)
+{
+    return (int64_t)a0 + a1.Value + a2.Value + a3.Value + a4.Value + a5.Value + a6.Value + a7.Value + a8.Value + a9.Value
+        + a10.Value + a11.Value + a12.Value + a13.Value + a14.Value + a15.Value + a16.Value + a17;
+}

src/tests/Interop/IJW/NativeCallingManaged/NativeCallingManaged.cs

@@ -36,6 +36,22 @@ public static int TestEntryPoint()
             }
             TestFramework.EndTestCase();
 
+            // Regression test for https://github.com/dotnet/runtime/issues/127166:
+            // Native code calling a managed function with 17+ by-ref parameters
+            // hit an OverflowException because StubSigBuilder::EnsureEnoughQuickBytes
+            // only doubled the buffer size once, which was insufficient when the
+            // internal signature (with preserved custom modifiers) exceeded 512 bytes.
+            TestFramework.BeginTestCase("Call managed method with 18 by-ref parameters from native");
+            MethodInfo sum18Method = testType.GetMethod("ManagedEntryPointSum18ByRef");
+            long sum = (long)sum18Method.Invoke(testInstance, null);
+            const long expectedSum = 153; // 0 + (1+2+...+16) + 17
+            if (sum != expectedSum)
+            {
+                TestFramework.LogError("IJW", "Incorrect sum returned: " + sum + " (expected " + expectedSum + ")");
+                success = false;
+            }
+            TestFramework.EndTestCase();
+
             return success ? 100 : 99;
         }
     }