Fast file enumeration for Windows (#25426)

* Fast file enumeration for Windows

Implements an optimized, low allocation replacement for file enumeration on Windows. In preliminary tests this improves performance at least two fold, particularly for recursive searches.
Removes aggressive filter validation.
Copies and cleans up filename matching code from FileSystemWatcher.
Add length setter for ValueStringBuilder.
Remove Unix .. check for enumerable.

Author: JeremyKuhne

src/Common/src/Interop/Windows/Interop.BOOLEAN.cs

@@ -0,0 +1,21 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+internal partial class Interop
+{
+    /// <summary>
+    /// Blittable version of Windows BOOLEAN type. It is convenient in situations where
+    /// manual marshalling is required, or to avoid overhead of regular bool marshalling.
+    /// </summary>
+    /// <remarks>
+    /// Some Windows APIs return arbitrary integer values although the return type is defined
+    /// as BOOLEAN. It is best to never compare BOOLEAN to TRUE. Always use bResult != BOOLEAN.FALSE
+    /// or bResult == BOOLEAN.FALSE .
+    /// </remarks>
+    internal enum BOOLEAN : byte
+    {
+        FALSE = 0,
+        TRUE = 1,
+    }
+}

src/Common/src/Interop/Windows/Interop.LongFileTime.cs

@@ -0,0 +1,28 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+
+internal partial class Interop
+{
+    /// <summary>
+    /// 100-nanosecond intervals (ticks) since January 1, 1601 (UTC).
+    /// </summary>
+    /// <remarks>
+    /// For NT times that are defined as longs (LARGE_INTEGER, etc.).
+    /// Do NOT use for FILETIME unless you are POSITIVE it will fall on an
+    /// 8 byte boundary.
+    /// </remarks>
+    internal struct LongFileTime
+    {
+#pragma warning disable CS0649
+        /// <summary>
+        /// 100-nanosecond intervals (ticks) since January 1, 1601 (UTC).
+        /// </summary>
+        internal long TicksSince1601;
+#pragma warning restore CS0649
+
+        internal DateTime ToDateTimeUtc() => DateTime.FromFileTimeUtc(TicksSince1601);
+    }
+}

src/Common/src/Interop/Windows/NtDll/Interop.FILE_FULL_DIR_INFORMATION.cs

@@ -0,0 +1,73 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+
+internal partial class Interop
+{
+    internal partial class NtDll
+    {
+        /// <summary>
+        /// <a href="https://msdn.microsoft.com/en-us/library/windows/hardware/ff540289.aspx">FILE_FULL_DIR_INFORMATION</a> structure.
+        /// Used with GetFileInformationByHandleEx and FileIdBothDirectoryInfo/RestartInfo as well as NtQueryFileInformation.
+        /// Equivalent to <a href="https://msdn.microsoft.com/en-us/library/windows/desktop/hh447298.aspx">FILE_FULL_DIR_INFO</a> structure.
+        /// </summary>
+        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
+        public unsafe struct FILE_FULL_DIR_INFORMATION
+        {
+            /// <summary>
+            /// Offset in bytes of the next entry, if any.
+            /// </summary>
+            public uint NextEntryOffset;
+
+            /// <summary>
+            /// Byte offset within the parent directory, undefined for NTFS.
+            /// </summary>
+            public uint FileIndex;
+            public LongFileTime CreationTime;
+            public LongFileTime LastAccessTime;
+            public LongFileTime LastWriteTime;
+            public LongFileTime ChangeTime;
+            public long EndOfFile;
+            public long AllocationSize;
+
+            /// <summary>
+            /// File attributes.
+            /// </summary>
+            /// <remarks>
+            /// Note that MSDN documentation isn't correct for this- it can return
+            /// any FILE_ATTRIBUTE that is currently set on the file, not just the
+            /// ones documented.
+            /// </remarks>
+            public FileAttributes FileAttributes;
+
+            /// <summary>
+            /// The length of the file name in bytes (without null).
+            /// </summary>
+            public uint FileNameLength;
+
+            /// <summary>
+            /// The extended attribute size OR the reparse tag if a reparse point.
+            /// </summary>
+            public uint EaSize;
+
+            private char _fileName;
+            public ReadOnlySpan<char> FileName { get { fixed (char* c = &_fileName) { return new ReadOnlySpan<char>(c, (int)FileNameLength / sizeof(char)); } } }
+
+            /// <summary>
+            /// Gets the next info pointer or null if there are no more.
+            /// </summary>
+            public unsafe static FILE_FULL_DIR_INFORMATION* GetNextInfo(FILE_FULL_DIR_INFORMATION* info)
+            {
+                uint nextOffset = (*info).NextEntryOffset;
+                if (nextOffset == 0)
+                    return null;
+
+                return (FILE_FULL_DIR_INFORMATION*)((byte*)info + nextOffset);
+            }
+        }
+    }
+}

src/Common/src/Interop/Windows/NtDll/Interop.FILE_INFORMATION_CLASS.cs

@@ -0,0 +1,85 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Runtime.InteropServices;
+
+internal partial class Interop
+{
+    internal partial class NtDll
+    {
+        // https://msdn.microsoft.com/en-us/library/windows/hardware/ff728840.aspx
+        public enum FILE_INFORMATION_CLASS : uint
+        {
+            FileDirectoryInformation = 1,
+            FileFullDirectoryInformation = 2,
+            FileBothDirectoryInformation = 3,
+            FileBasicInformation = 4,
+            FileStandardInformation = 5,
+            FileInternalInformation = 6,
+            FileEaInformation = 7,
+            FileAccessInformation = 8,
+            FileNameInformation = 9,
+            FileRenameInformation = 10,
+            FileLinkInformation = 11,
+            FileNamesInformation = 12,
+            FileDispositionInformation = 13,
+            FilePositionInformation = 14,
+            FileFullEaInformation = 15,
+            FileModeInformation = 16,
+            FileAlignmentInformation = 17,
+            FileAllInformation = 18,
+            FileAllocationInformation = 19,
+            FileEndOfFileInformation = 20,
+            FileAlternateNameInformation = 21,
+            FileStreamInformation = 22,
+            FilePipeInformation = 23,
+            FilePipeLocalInformation = 24,
+            FilePipeRemoteInformation = 25,
+            FileMailslotQueryInformation = 26,
+            FileMailslotSetInformation = 27,
+            FileCompressionInformation = 28,
+            FileObjectIdInformation = 29,
+            FileCompletionInformation = 30,
+            FileMoveClusterInformation = 31,
+            FileQuotaInformation = 32,
+            FileReparsePointInformation = 33,
+            FileNetworkOpenInformation = 34,
+            FileAttributeTagInformation = 35,
+            FileTrackingInformation = 36,
+            FileIdBothDirectoryInformation = 37,
+            FileIdFullDirectoryInformation = 38,
+            FileValidDataLengthInformation = 39,
+            FileShortNameInformation = 40,
+            FileIoCompletionNotificationInformation = 41,
+            FileIoStatusBlockRangeInformation = 42,
+            FileIoPriorityHintInformation = 43,
+            FileSfioReserveInformation = 44,
+            FileSfioVolumeInformation = 45,
+            FileHardLinkInformation = 46,
+            FileProcessIdsUsingFileInformation = 47,
+            FileNormalizedNameInformation = 48,
+            FileNetworkPhysicalNameInformation = 49,
+            FileIdGlobalTxDirectoryInformation = 50,
+            FileIsRemoteDeviceInformation = 51,
+            FileUnusedInformation = 52,
+            FileNumaNodeInformation = 53,
+            FileStandardLinkInformation = 54,
+            FileRemoteProtocolInformation = 55,
+            FileRenameInformationBypassAccessCheck = 56,
+            FileLinkInformationBypassAccessCheck = 57,
+            FileVolumeNameInformation = 58,
+            FileIdInformation = 59,
+            FileIdExtdDirectoryInformation = 60,
+            FileReplaceCompletionInformation = 61,
+            FileHardLinkFullIdInformation = 62,
+            FileIdExtdBothDirectoryInformation = 63,
+            FileDispositionInformationEx = 64,
+            FileRenameInformationEx = 65,
+            FileRenameInformationExBypassAccessCheck = 66,
+            FileDesiredStorageClassInformation = 67,
+            FileStatInformation = 68
+        }
+    }
+}

src/Common/src/Interop/Windows/NtDll/Interop.IO_STATUS_BLOCK.cs

@@ -0,0 +1,47 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Runtime.InteropServices;
+
+internal partial class Interop
+{
+    internal partial class NtDll
+    {
+        // https://msdn.microsoft.com/en-us/library/windows/hardware/ff550671.aspx
+        [StructLayout(LayoutKind.Sequential)]
+        public struct IO_STATUS_BLOCK
+        {
+            /// <summary>
+            /// Status
+            /// </summary>
+            public IO_STATUS Status;
+
+            /// <summary>
+            /// Request dependent value.
+            /// </summary>
+            public IntPtr Information;
+
+            // This isn't an actual Windows type, it is a union within IO_STATUS_BLOCK. We *have* to separate it out as
+            // the size of IntPtr varies by architecture and we can't specify the size at compile time to offset the
+            // Information pointer in the status block.
+            [StructLayout(LayoutKind.Explicit)]
+            public struct IO_STATUS
+            {
+                /// <summary>
+                /// The completion status, either STATUS_SUCCESS if the operation was completed successfully or
+                /// some other informational, warning, or error status.
+                /// </summary>
+                [FieldOffset(0)]
+                public uint Status;
+
+                /// <summary>
+                /// Reserved for internal use.
+                /// </summary>
+                [FieldOffset(0)]
+                public IntPtr Pointer;
+            }
+        }
+    }
+}

src/Common/src/Interop/Windows/NtDll/Interop.NtQueryDirectoryFile.cs

@@ -0,0 +1,28 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Runtime.InteropServices;
+
+internal partial class Interop
+{
+    internal partial class NtDll
+    {
+        // https://msdn.microsoft.com/en-us/library/windows/hardware/ff556633.aspx
+        // https://msdn.microsoft.com/en-us/library/windows/hardware/ff567047.aspx
+        [DllImport(Libraries.NtDll, CharSet = CharSet.Unicode, ExactSpelling = true)]
+        public unsafe static extern int NtQueryDirectoryFile(
+            IntPtr FileHandle,
+            IntPtr Event,
+            IntPtr ApcRoutine,
+            IntPtr ApcContext,
+            out IO_STATUS_BLOCK IoStatusBlock,
+            byte[] FileInformation,
+            uint Length,
+            FILE_INFORMATION_CLASS FileInformationClass,
+            BOOLEAN ReturnSingleEntry,
+            UNICODE_STRING* FileName,
+            BOOLEAN RestartScan);
+    }
+}

src/Common/src/Interop/Windows/NtDll/Interop.NtStatus.cs

@@ -9,6 +9,7 @@ internal class StatusOptions
         // Error codes from ntstatus.h
         internal const uint STATUS_SUCCESS = 0x00000000;
         internal const uint STATUS_SOME_NOT_MAPPED = 0x00000107;
+        internal const uint STATUS_NO_MORE_FILES = 0x80000006;
         internal const uint STATUS_INVALID_PARAMETER = 0xC000000D;
         internal const uint STATUS_NO_MEMORY = 0xC0000017;
         internal const uint STATUS_OBJECT_NAME_NOT_FOUND = 0xC0000034;

src/Common/src/Interop/Windows/NtDll/Interop.RtlNtStatusToDosError.cs

@@ -0,0 +1,17 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Runtime.InteropServices;
+
+internal partial class Interop
+{
+    internal partial class NtDll
+    {
+        // https://msdn.microsoft.com/en-us/library/windows/desktop/ms680600(v=vs.85).aspx
+        [DllImport(Libraries.NtDll, ExactSpelling = true)]
+        public unsafe static extern uint RtlNtStatusToDosError(
+            int Status);
+    }
+}

src/Common/src/Interop/Windows/kernel32/Interop.CompareStringOrdinal.cs

@@ -0,0 +1,20 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Runtime.InteropServices;
+
+internal partial class Interop
+{
+    internal partial class Kernel32
+    {
+        // https://msdn.microsoft.com/en-us/library/windows/desktop/dd317762.aspx
+        [DllImport(Libraries.Kernel32, CharSet = CharSet.Unicode, SetLastError = true, ExactSpelling = true)]
+        public unsafe static extern int CompareStringOrdinal(
+            ref char lpString1,
+            int cchCount1,
+            ref char lpString2,
+            int cchCount2,
+            bool bIgnoreCase);
+    }
+}

src/Common/src/Interop/Windows/kernel32/Interop.CreateFile.cs

@@ -11,11 +11,12 @@ internal partial class Interop
 {
     internal partial class Kernel32
     {
+        // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858.aspx
         /// <summary>
         /// WARNING: The private methods do not implicitly handle long paths. Use CreateFile.
         /// </summary>
         [DllImport(Libraries.Kernel32, EntryPoint = "CreateFileW", SetLastError = true, CharSet = CharSet.Unicode, BestFitMapping = false, ExactSpelling = true)]
-        private unsafe static extern SafeFileHandle CreateFilePrivate(
+        private unsafe static extern IntPtr CreateFilePrivate(
             string lpFileName,
             int dwDesiredAccess,
             FileShare dwShareMode,
@@ -36,7 +37,16 @@ internal unsafe static SafeFileHandle CreateFile(
             lpFileName = PathInternal.EnsureExtendedPrefixOverMaxPath(lpFileName);
             fixed (SECURITY_ATTRIBUTES* sa = &securityAttrs)
             {
-                return CreateFilePrivate(lpFileName, dwDesiredAccess, dwShareMode, sa, dwCreationDisposition, dwFlagsAndAttributes, hTemplateFile);
+                IntPtr handle = CreateFilePrivate(lpFileName, dwDesiredAccess, dwShareMode, sa, dwCreationDisposition, dwFlagsAndAttributes, hTemplateFile);
+                try
+                {
+                    return new SafeFileHandle(handle, ownsHandle: true);
+                }
+                catch
+                {
+                    CloseHandle(handle);
+                    throw;
+                }
             }
         }
 
@@ -46,6 +56,25 @@ internal unsafe static SafeFileHandle CreateFile(
             FileShare dwShareMode,
             FileMode dwCreationDisposition,
             int dwFlagsAndAttributes)
+        {
+            IntPtr handle = CreateFile_IntPtr(lpFileName, dwDesiredAccess, dwShareMode, dwCreationDisposition, dwFlagsAndAttributes);
+            try
+            {
+                return new SafeFileHandle(handle, ownsHandle: true);
+            }
+            catch
+            {
+                CloseHandle(handle);
+                throw;
+            }
+        }
+
+        internal unsafe static IntPtr CreateFile_IntPtr(
+            string lpFileName,
+            int dwDesiredAccess,
+            FileShare dwShareMode,
+            FileMode dwCreationDisposition,
+            int dwFlagsAndAttributes)
         {
             lpFileName = PathInternal.EnsureExtendedPrefixOverMaxPath(lpFileName);
             return CreateFilePrivate(lpFileName, dwDesiredAccess, dwShareMode, null, dwCreationDisposition, dwFlagsAndAttributes, IntPtr.Zero);