Optimized FileMatcher.GetFiles method (#2392)

Optimizations: - Replaced Directory methods GetFiles and GetDirectories with EnumerateFiles and EnumerateDirectories - Added a custom wildcard pattern matching algorithm for matching directory and file names - Added directory caching - Added parallelization of subdirectories
dotnet · Oct 10, 2017 · 246a185 · 246a185
1 parent 0c3e87c
commit 246a185
Show file tree

Hide file tree

Showing 17 changed files with 990 additions and 229 deletions.
diff --git a/build/NuGetPackages/Microsoft.Build.Framework.nuspec b/build/NuGetPackages/Microsoft.Build.Framework.nuspec
@@ -15,6 +15,7 @@
     <tags>MSBuild</tags>
     <dependencies>
       <group targetFramework=".NETStandard1.3">
+        <dependency id="System.Threading.Tasks.Parallel" version="4.0.1" />
         <dependency id="System.Collections" version="4.0.11" />
         <dependency id="System.Diagnostics.Debug" version="4.0.11" />
         <dependency id="System.Globalization" version="4.0.11" />

diff --git a/build/NuGetPackages/Microsoft.Build.Tasks.Core.nuspec b/build/NuGetPackages/Microsoft.Build.Tasks.Core.nuspec
@@ -56,6 +56,7 @@
         <dependency id="System.Text.RegularExpressions" version="4.1.0" />
         <dependency id="System.Threading" version="4.0.11" />
         <dependency id="System.Threading.Tasks" version="4.0.11" />
+        <dependency id="System.Threading.Tasks.Parallel" version="4.0.1" />
         <dependency id="System.Threading.Thread" version="4.0.0" />
         <dependency id="System.Xml.ReaderWriter" version="4.0.11" />
         <dependency id="System.Xml.XDocument" version="4.0.11" />

diff --git a/build/NuGetPackages/Microsoft.Build.Utilities.Core.nuspec b/build/NuGetPackages/Microsoft.Build.Utilities.Core.nuspec
@@ -21,6 +21,7 @@
         <dependency id="System.AppContext" version="4.1.0" />
         <dependency id="System.Collections" version="4.0.11" />
         <dependency id="System.Collections.Concurrent" version="4.0.12" />
+        <dependency id="System.Collections.Immutable" version="1.2.0" />
         <dependency id="System.Collections.NonGeneric" version="4.0.1" />
         <dependency id="System.Console" version="4.0.0" />
         <dependency id="System.Diagnostics.Debug" version="4.0.11" />
@@ -47,13 +48,15 @@
         <dependency id="System.Text.RegularExpressions" version="4.1.0" />
         <dependency id="System.Threading" version="4.0.11" />
         <dependency id="System.Threading.Tasks" version="4.0.11" />
+        <dependency id="System.Threading.Tasks.Parallel" version="4.0.1" />
         <dependency id="System.Threading.Thread" version="4.0.0" />
         <dependency id="System.Threading.Timer" version="4.0.1" />
         <dependency id="System.Xml.ReaderWriter" version="4.0.11" />
         <dependency id="System.Xml.XmlDocument" version="4.0.1" />
       </group>
       <group targetFramework=".NETFramework4.6">
         <dependency id="Microsoft.Build.Framework" version="[$version$]" />
+        <dependency id="System.Collections.Immutable" version="1.3.1" />
         <dependency id="System.Runtime.InteropServices.RuntimeInformation" version="4.3.0" />
       </group>
     </dependencies>

diff --git a/build/NuGetPackages/Microsoft.Build.nuspec b/build/NuGetPackages/Microsoft.Build.nuspec
@@ -56,6 +56,7 @@
         <dependency id="System.Threading" version="4.0.11" />
         <dependency id="System.Threading.Tasks" version="4.0.11" />
         <dependency id="System.Threading.Tasks.Dataflow" version="4.6.0" />
+        <dependency id="System.Threading.Tasks.Parallel" version="4.0.1" />
         <dependency id="System.Threading.Thread" version="4.0.0" />
         <dependency id="System.Threading.ThreadPool" version="4.0.10" />
         <dependency id="System.Xml.ReaderWriter" version="4.0.11" />

diff --git a/src/Build.UnitTests/Microsoft.Build.Engine.UnitTests.csproj b/src/Build.UnitTests/Microsoft.Build.Engine.UnitTests.csproj
@@ -249,6 +249,7 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup Condition="'$(NetCoreBuild)' != 'true'">
+    <Reference Include="System.Collections.Immutable, Version=1.2.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a" />
     <Reference Include="System" />
     <Reference Include="System.Configuration" />
     <Reference Include="System.Xml" />

diff --git a/src/Build.UnitTests/project.json b/src/Build.UnitTests/project.json
@@ -10,13 +10,15 @@
   "frameworks": {
     "net46": {
       "Microsoft.Net.Compilers": "2.3.1",
+      "System.Collections.Immutable": "1.3.1",
       "System.Threading.Tasks.Dataflow": "4.5.24.0"
     },
     "netstandard1.3": {
       "dependencies": {
         "System.Net.NetworkInformation": "4.1.0",
         "Microsoft.NETCore": "5.0.1",
         "Microsoft.NETCore.Portable.Compatibility": "1.0.1",
+        "System.Collections.Immutable": "1.2.0",
         "System.Collections.NonGeneric": "4.0.1",
         "System.Console": "4.0.0",
         "System.Diagnostics.Contracts": "4.0.1",

diff --git a/src/Build/Evaluation/LazyItemEvaluator.IncludeOperation.cs b/src/Build/Evaluation/LazyItemEvaluator.IncludeOperation.cs
@@ -99,7 +99,8 @@ protected override ICollection<I> SelectItems(ImmutableList<ItemData>.Builder li
                         string[] includeSplitFilesEscaped = EngineFileUtilities.GetFileListEscaped(
                             _rootDirectory,
                             glob,
-                            excludePatternsForGlobs
+                            excludePatternsForGlobs,
+                            entriesCache: EntriesCache
                             );
 
                         // itemsToAdd might grow 0 or more times during the following iteration. Proactively increase its capacity to ensure only one growth happens

diff --git a/src/Build/Evaluation/LazyItemEvaluator.LazyItemOperation.cs b/src/Build/Evaluation/LazyItemEvaluator.LazyItemOperation.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Linq;
@@ -44,6 +45,11 @@ protected LazyItemOperation(OperationBuilder builder, LazyItemEvaluator<P, I, M,
                 _itemSpec.Expander = _expander;
             }
 
+            /// <summary>
+            /// Cache used for caching IO operation results
+            /// </summary>
+            protected ConcurrentDictionary<string, ImmutableArray<string>> EntriesCache => _lazyEvaluator._entriesCache;
+
             public virtual void Apply(ImmutableList<ItemData>.Builder listBuilder, ImmutableHashSet<string> globsToIgnore)
             {
                 var items = SelectItems(listBuilder, globsToIgnore).ToList();

diff --git a/src/Build/Evaluation/LazyItemEvaluator.cs b/src/Build/Evaluation/LazyItemEvaluator.cs
@@ -10,6 +10,7 @@
 using Microsoft.Build.Framework;
 using Microsoft.Build.Shared;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
@@ -45,6 +46,11 @@ internal partial class LazyItemEvaluator<P, I, M, D>
             new Dictionary<string, LazyItemList>() :
             new Dictionary<string, LazyItemList>(StringComparer.OrdinalIgnoreCase);
 
+        /// <summary>
+        /// Cache used for caching IO operation results
+        /// </summary>
+        private readonly ConcurrentDictionary<string, ImmutableArray<string>> _entriesCache = new ConcurrentDictionary<string, ImmutableArray<string>>();
+
         public LazyItemEvaluator(IEvaluatorData<P, I, M, D> data, IItemFactory<I, I> itemFactory, LoggingContext loggingContext)
         {
             _outerEvaluatorData = data;

diff --git a/src/Build/Utilities/EngineFileUtilities.cs b/src/Build/Utilities/EngineFileUtilities.cs
@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Text;
 using System.IO;
@@ -74,16 +75,18 @@ internal static string[] GetFileListUnescaped
         /// <param name="filespecEscaped">The filespec to evaluate, escaped.</param>
         /// <param name="excludeSpecsEscaped">Filespecs to exclude, escaped.</param>
         /// <param name="forceEvaluate">Whether to force file glob expansion when eager expansion is turned off</param>
+        /// <param name="entriesCache">Cache used for caching IO operation results</param>
         /// <returns>Array of file paths, escaped.</returns>
         internal static string[] GetFileListEscaped
             (
             string directoryEscaped,
             string filespecEscaped,
             IEnumerable<string> excludeSpecsEscaped = null,
-            bool forceEvaluate = false
+            bool forceEvaluate = false,
+            ConcurrentDictionary<string, ImmutableArray<string>> entriesCache = null
             )
         {
-            return GetFileList(directoryEscaped, filespecEscaped, true /* returnEscaped */, forceEvaluate, excludeSpecsEscaped);
+            return GetFileList(directoryEscaped, filespecEscaped, true /* returnEscaped */, forceEvaluate, excludeSpecsEscaped, entriesCache);
         }
 
         private static bool FilespecHasWildcards(string filespecEscaped)
@@ -125,14 +128,16 @@ private static bool FilespecHasWildcards(string filespecEscaped)
         /// <param name="returnEscaped"><code>true</code> to return escaped specs.</param>
         /// <param name="forceEvaluateWildCards">Whether to force file glob expansion when eager expansion is turned off</param>
         /// <param name="excludeSpecsEscaped">The exclude specification, escaped.</param>
+        /// <param name="entriesCache">Cache used for caching IO operation results</param>
         /// <returns>Array of file paths.</returns>
         private static string[] GetFileList
             (
             string directoryEscaped,
             string filespecEscaped,
             bool returnEscaped,
             bool forceEvaluateWildCards,
-            IEnumerable<string> excludeSpecsEscaped = null
+            IEnumerable<string> excludeSpecsEscaped = null,
+            ConcurrentDictionary<string, ImmutableArray<string>> entriesCache = null
             )
         {
             ErrorUtilities.VerifyThrowInternalLength(filespecEscaped, "filespecEscaped");
@@ -167,7 +172,7 @@ private static string[] GetFileList
                 // as a relative path, we will get back a bunch of relative paths.
                 // If the filespec started out as an absolute path, we will get
                 // back a bunch of absolute paths.
-                fileList = FileMatcher.GetFiles(directoryUnescaped, filespecUnescaped, excludeSpecsUnescaped);
+                fileList = FileMatcher.GetFiles(directoryUnescaped, filespecUnescaped, excludeSpecsUnescaped, entriesCache);
 
                 ErrorUtilities.VerifyThrow(fileList != null, "We must have a list of files here, even if it's empty.");
 

diff --git a/src/Build/project.json b/src/Build/project.json
@@ -32,6 +32,7 @@
         "System.Runtime.Serialization.Primitives": "4.1.1",
         "System.Text.Encoding.CodePages" : "4.0.1",
         "System.Threading.Tasks.Dataflow": "4.6.0.0",
+        "System.Threading.Tasks.Parallel": "4.0.1",
         "System.Threading.Thread": "4.0.0",
         "System.Threading.ThreadPool": "4.0.10",
         "System.Xml.XmlDocument": "4.0.1",

diff --git a/src/Framework/project.json b/src/Framework/project.json
@@ -8,6 +8,7 @@
         "SourceLink.Create.CommandLine": "2.1.2",
         "System.Runtime.Serialization.Primitives": "4.1.1",
         "System.Text.Encoding.CodePages" : "4.0.1",
+        "System.Threading.Tasks.Parallel": "4.0.1",
         "System.Threading.Thread": "4.0.0"
       }
     }