Improve span copy of pointers and structs containing pointers

Fixes #9161 PR dotnet#9786 fixes perf of span copy of types that don't contain references
kouvel · Mar 8, 2017 · 75cc69f · 75cc69f
1 parent afa001d
commit 75cc69f
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 16 deletions.
diff --git a/src/mscorlib/src/System/Runtime/RuntimeImports.cs b/src/mscorlib/src/System/Runtime/RuntimeImports.cs
@@ -17,7 +17,7 @@ public class RuntimeImports
     {
         // Non-inlinable wrapper around the QCall that avoids poluting the fast path
         // with P/Invoke prolog/epilog.
-        [MethodImplAttribute(MethodImplOptions.NoInlining)]
+        [MethodImpl(MethodImplOptions.NoInlining)]
         internal unsafe static void RhZeroMemory(ref byte b, nuint byteLength)
         {
             fixed (byte* bytePointer = &b)
@@ -28,5 +28,27 @@ internal unsafe static void RhZeroMemory(ref byte b, nuint byteLength)
 
         [DllImport(JitHelpers.QCall, CharSet = CharSet.Unicode)]
         extern private unsafe static void RhZeroMemory(byte* b, nuint byteLength);
+
+        // Non-inlinable wrapper around the QCall that avoids poluting the fast path
+        // with P/Invoke prolog/epilog.
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        internal unsafe static bool RhCopyMemoryWithReferences<T>(ref T destination, ref T source, int elementCount)
+        {
+            fixed (void* destinationPtr = &Unsafe.As<T, byte>(ref destination))
+            {
+                fixed (void* sourcePtr = &Unsafe.As<T, byte>(ref source))
+                {
+                    return
+                        RhCopyMemoryWithReferences(
+                            destinationPtr,
+                            sourcePtr,
+                            (nuint)elementCount * (nuint)Unsafe.SizeOf<T>());
+                }
+            }
+        }
+
+        [DllImport(JitHelpers.QCall)]
+        [return: MarshalAs(UnmanagedType.I1)]
+        extern private unsafe static bool RhCopyMemoryWithReferences(void* destination, void* source, nuint byteCount);
     }
 }
diff --git a/src/mscorlib/src/System/Span.cs b/src/mscorlib/src/System/Span.cs
@@ -211,11 +211,11 @@ public void Clear()
         {
             if (RuntimeHelpers.IsReferenceOrContainsReferences<T>())
             {
-                SpanHelper.ClearWithReferences(ref Unsafe.As<T, IntPtr>(ref _pointer.Value), (nuint)(_length * (Unsafe.SizeOf<T>() / sizeof(nuint))));
+                SpanHelper.ClearWithReferences(ref Unsafe.As<T, IntPtr>(ref _pointer.Value), (nuint)_length * (nuint)(Unsafe.SizeOf<T>() / sizeof(nuint)));
             }
             else
             {
-                SpanHelper.ClearWithoutReferences(ref Unsafe.As<T, byte>(ref _pointer.Value), (nuint)(_length * Unsafe.SizeOf<T>()));
+                SpanHelper.ClearWithoutReferences(ref Unsafe.As<T, byte>(ref _pointer.Value), (nuint)_length * (nuint)Unsafe.SizeOf<T>());
             }
         }
 
@@ -581,16 +581,18 @@ internal static unsafe void CopyTo<T>(ref T destination, ref T source, int eleme
                 {
                     fixed (byte* pSource = &Unsafe.As<T, byte>(ref source))
                     {
-#if BIT64
-                        Buffer.Memmove(pDestination, pSource, (ulong)elementsCount * (ulong)Unsafe.SizeOf<T>());
-#else
-                        Buffer.Memmove(pDestination, pSource, (uint)elementsCount * (uint)Unsafe.SizeOf<T>());
-#endif
+                        Buffer.Memmove(pDestination, pSource, (nuint)elementsCount * (nuint)Unsafe.SizeOf<T>());
                     }
                 }
             }
             else
             {
+                if ((nuint)elementsCount * (nuint)Unsafe.SizeOf<T>() >= 128 &&
+                    RuntimeImports.RhCopyMemoryWithReferences(ref destination, ref source, elementsCount))
+                {
+                    return;
+                }
+
                 if (JitHelpers.ByRefLessThan(ref destination, ref source)) // copy forward
                 {
                     for (int i = 0; i < elementsCount; i++)

diff --git a/src/vm/comutilnative.cpp b/src/vm/comutilnative.cpp
@@ -1494,13 +1494,26 @@ FCIMPL5(VOID, Buffer::InternalBlockCopy, ArrayBase *src, int srcOffset, ArrayBas
 }
 FCIMPLEND
 
-void QCALLTYPE SpanNative::SpanClear(void *dst, size_t length)
+void QCALLTYPE MemoryNative::Clear(void *dst, size_t length)
 {
     QCALL_CONTRACT;
 
     memset(dst, 0, length);
 }
 
+bool QCALLTYPE MemoryNative::CopyWithReferences(void *dst, void *src, size_t byteCount)
+{
+    QCALL_CONTRACT;
+
+    if (!IS_ALIGNED(dst, sizeof(dst)) || !IS_ALIGNED(src, sizeof(src)))
+    {
+        return false;
+    }
+
+    memmoveGCRefs(dst, src, byteCount);
+    return true;
+}
+
 void QCALLTYPE Buffer::MemMove(void *dst, void *src, size_t length)
 {
     QCALL_CONTRACT;

diff --git a/src/vm/comutilnative.h b/src/vm/comutilnative.h
@@ -110,13 +110,11 @@ class ExceptionNative
     static FCDECL0(INT32, GetExceptionCode);
 };
 
-
-//
-// SpanNative
-//
-class SpanNative {
+class MemoryNative
+{
 public:
-    static void QCALLTYPE SpanClear(void *dst, size_t length);
+    static void QCALLTYPE Clear(void *dst, size_t length);
+    static bool QCALLTYPE CopyWithReferences(void *dst, void *src, size_t byteCount);
 };
 
 //

diff --git a/src/vm/ecalllist.h b/src/vm/ecalllist.h
@@ -1287,7 +1287,8 @@ FCFuncEnd()
 #endif // ifdef FEATURE_COMINTEROP
 
 FCFuncStart(gRuntimeImportsFuncs)
-    QCFuncElement("RhZeroMemory", SpanNative::SpanClear)
+    QCFuncElement("RhZeroMemory", MemoryNative::Clear)
+    QCFuncElement("RhCopyMemoryWithReferences", MemoryNative::CopyWithReferences)
 FCFuncEnd()
 
 FCFuncStart(gWeakReferenceFuncs)