diff --git a/src/coreclr/vm/stubgen.cpp b/src/coreclr/vm/stubgen.cpp index 62e3cd0c5c6c34..963c47884d6cce 100644 --- a/src/coreclr/vm/stubgen.cpp +++ b/src/coreclr/vm/stubgen.cpp @@ -2003,9 +2003,13 @@ void StubSigBuilder::EnsureEnoughQuickBytes(size_t cbToAppend) STANDARD_VM_CONTRACT; SIZE_T cbBuffer = m_qbSigBuffer.Size(); - if ((m_cbSig + cbToAppend) >= cbBuffer) + if ((cbBuffer - m_cbSig) < cbToAppend) { - m_qbSigBuffer.ReSizeThrows(2 * cbBuffer); + SIZE_T cbNew = max(m_cbSig + cbToAppend, 2 * cbBuffer); + // Detect integer overflow + if ((cbNew - m_cbSig) < cbToAppend) + COMPlusThrowHR(COR_E_OVERFLOW); + m_qbSigBuffer.ReSizeThrows(cbNew); m_pbSigCursor = ((BYTE*)m_qbSigBuffer.Ptr()) + m_cbSig; } } diff --git a/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp b/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp index e581c7ae185e11..a4c1257575af51 100644 --- a/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp +++ b/src/tests/Interop/IJW/IjwNativeCallingManagedDll/IjwNativeCallingManagedDll.cpp @@ -16,6 +16,45 @@ extern "C" DLL_EXPORT int __cdecl NativeEntryPoint() return NativeFunction(); } +struct IntWrapper +{ + int Value; +}; + +#pragma managed +int64_t ManagedSum18ByRef(const int& a0, + const IntWrapper& a1, + const IntWrapper& a2, + const IntWrapper& a3, + const IntWrapper& a4, + const IntWrapper& a5, + const IntWrapper& a6, + const IntWrapper& a7, + const IntWrapper& a8, + const IntWrapper& a9, + const IntWrapper& a10, + const IntWrapper& a11, + const IntWrapper& a12, + const IntWrapper& a13, + const IntWrapper& a14, + const IntWrapper& a15, + const IntWrapper& a16, + const int& a17); + +#pragma unmanaged +int64_t NativeSum18ByRef() +{ + int a0 = 0; + IntWrapper w[16]; + for (int i = 0; i < 16; ++i) + { + w[i].Value = i + 1; + } + int a17 = 17; + return ManagedSum18ByRef(a0, w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7], + w[8], w[9], w[10], w[11], w[12], w[13], w[14], w[15], a17); +} + #pragma managed // Needed to provide a regression case for https://github.com/dotnet/runtime/issues/110365 @@ -39,6 +78,11 @@ public ref class TestClass return NativeFunction(); } + int64_t ManagedEntryPointSum18ByRef() + { + return NativeSum18ByRef(); + } + static void ChangeReturnedValue(int i) { s_valueToReturnStorage.valueToReturn = i; @@ -58,3 +102,26 @@ int ManagedCallee() { return TestClass::GetReturnValue(); } + +int64_t ManagedSum18ByRef(const int& a0, + const IntWrapper& a1, + const IntWrapper& a2, + const IntWrapper& a3, + const IntWrapper& a4, + const IntWrapper& a5, + const IntWrapper& a6, + const IntWrapper& a7, + const IntWrapper& a8, + const IntWrapper& a9, + const IntWrapper& a10, + const IntWrapper& a11, + const IntWrapper& a12, + const IntWrapper& a13, + const IntWrapper& a14, + const IntWrapper& a15, + const IntWrapper& a16, + const int& a17) +{ + return (int64_t)a0 + a1.Value + a2.Value + a3.Value + a4.Value + a5.Value + a6.Value + a7.Value + a8.Value + a9.Value + + a10.Value + a11.Value + a12.Value + a13.Value + a14.Value + a15.Value + a16.Value + a17; +} diff --git a/src/tests/Interop/IJW/NativeCallingManaged/NativeCallingManaged.cs b/src/tests/Interop/IJW/NativeCallingManaged/NativeCallingManaged.cs index b719695feef477..ced21db8f8ed01 100644 --- a/src/tests/Interop/IJW/NativeCallingManaged/NativeCallingManaged.cs +++ b/src/tests/Interop/IJW/NativeCallingManaged/NativeCallingManaged.cs @@ -36,6 +36,22 @@ public static int TestEntryPoint() } TestFramework.EndTestCase(); + // Regression test for https://github.com/dotnet/runtime/issues/127166: + // Native code calling a managed function with 17+ by-ref parameters + // hit an OverflowException because StubSigBuilder::EnsureEnoughQuickBytes + // only doubled the buffer size once, which was insufficient when the + // internal signature (with preserved custom modifiers) exceeded 512 bytes. + TestFramework.BeginTestCase("Call managed method with 18 by-ref parameters from native"); + MethodInfo sum18Method = testType.GetMethod("ManagedEntryPointSum18ByRef"); + long sum = (long)sum18Method.Invoke(testInstance, null); + const long expectedSum = 153; // 0 + (1+2+...+16) + 17 + if (sum != expectedSum) + { + TestFramework.LogError("IJW", "Incorrect sum returned: " + sum + " (expected " + expectedSum + ")"); + success = false; + } + TestFramework.EndTestCase(); + return success ? 100 : 99; } }