Unify threading between crossgen2 and ilc (#63282)

They don't have to differ in the `--parallelism` vs `--singlethreaded` argument.

docs/workflow/debugging/coreclr/debugging-aot-compilers.md

@@ -19,7 +19,7 @@ Built in debugging aids in the managed compilers
 ---------------------------------
 
 - When debugging a multi-threaded component of the compiler and not investigating a multi-threading issue itself, it is generally advisable to disable the use of multiple threads.
-To do this use the `--parallelism 1` switch (for crossgen2) or `--singlethreaded` (for ILC) to specify that the maximum parallelism of the process shall be 1.
+To do this use the `--parallelism 1` switch to specify that the maximum parallelism of the process shall be 1.
 
 - When debugging the behavior of compiling a single method, the compiler may be instructed to only compile a single method. This is done via the various --singlemethod options
 

src/coreclr/tools/Common/Compiler/CompilationBuilder.cs

@@ -23,6 +23,7 @@ public abstract partial class CompilationBuilder
         private DependencyTrackingLevel _dependencyTrackingLevel = DependencyTrackingLevel.None;
         protected IEnumerable<ICompilationRootProvider> _compilationRoots = Array.Empty<ICompilationRootProvider>();
         protected OptimizationMode _optimizationMode = OptimizationMode.None;
+        protected int _parallelism = -1;
 
         public CompilationBuilder(CompilerTypeSystemContext context, CompilationModuleGroup compilationGroup, NameMangler nameMangler)
         {
@@ -41,6 +42,12 @@ public CompilationBuilder UseLogger(Logger logger)
             return this;
         }
 
+        public CompilationBuilder UseParallelism(int parallelism)
+        {
+            _parallelism = parallelism;
+            return this;
+        }
+
         public CompilationBuilder UseCompilationUnitPrefix(string prefix)
         {
             _nameMangler.CompilationUnitPrefix = prefix;

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilationBuilder.Aot.cs

@@ -22,7 +22,6 @@ partial class CompilationBuilder
         protected MethodImportationErrorProvider _methodImportationErrorProvider = new MethodImportationErrorProvider();
         protected IInliningPolicy _inliningPolicy;
         protected bool _methodBodyFolding;
-        protected bool _singleThreaded;
         protected InstructionSetSupport _instructionSetSupport;
         protected SecurityMitigationOptions _mitigationOptions;
 
@@ -92,12 +91,6 @@ public CompilationBuilder UseMethodBodyFolding(bool enable)
             return this;
         }
 
-        public CompilationBuilder UseSingleThread(bool enable)
-        {
-            _singleThreaded = enable;
-            return this;
-        }
-
         public CompilationBuilder UsePreinitializationManager(PreinitializationManager manager)
         {
             _preinitializationManager = manager;

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ILScanner.cs

@@ -4,18 +4,18 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
-using System.Threading;
+using System.Threading.Tasks;
 
 using ILCompiler.DependencyAnalysis;
 using ILCompiler.DependencyAnalysisFramework;
 
 using Internal.IL;
 using Internal.IL.Stubs;
+using Internal.JitInterface;
 using Internal.TypeSystem;
 using Internal.ReadyToRunConstants;
 
 using Debug = System.Diagnostics.Debug;
-using Internal.JitInterface;
 
 namespace ILCompiler
 {
@@ -26,8 +26,7 @@ namespace ILCompiler
     /// </summary>
     internal sealed class ILScanner : Compilation, IILScanner
     {
-        private CountdownEvent _compilationCountdown;
-        private readonly bool _singleThreaded;
+        private readonly int _parallelism;
 
         internal ILScanner(
             DependencyAnalyzerBase<NodeFactory> dependencyGraph,
@@ -36,11 +35,11 @@ internal ILScanner(
             ILProvider ilProvider,
             DebugInformationProvider debugInformationProvider,
             Logger logger,
-            bool singleThreaded)
+            int parallelism)
             : base(dependencyGraph, nodeFactory, roots, ilProvider, debugInformationProvider, null, nodeFactory.CompilationModuleGroup, logger)
         {
             _helperCache = new HelperCache(this);
-            _singleThreaded = singleThreaded;
+            _parallelism = parallelism;
         }
 
         protected override void CompileInternal(string outputFile, ObjectDumper dumper)
@@ -78,7 +77,7 @@ protected override void ComputeDependencyNodeDependencies(List<DependencyNodeCor
                 methodsToCompile.Add(methodCodeNodeNeedingCode);
             }
 
-            if (_singleThreaded)
+            if (_parallelism == 1)
             {
                 CompileSingleThreaded(methodsToCompile);
             }
@@ -95,18 +94,10 @@ private void CompileMultiThreaded(List<ScannedMethodNode> methodsToCompile)
                 Logger.Writer.WriteLine($"Scanning {methodsToCompile.Count} methods...");
             }
 
-            WaitCallback compileSingleMethodDelegate = m => CompileSingleMethod((ScannedMethodNode)m);
-
-            using (_compilationCountdown = new CountdownEvent(methodsToCompile.Count))
-            {
-                foreach (ScannedMethodNode methodCodeNodeNeedingCode in methodsToCompile)
-                {
-                    ThreadPool.QueueUserWorkItem(compileSingleMethodDelegate, methodCodeNodeNeedingCode);
-                }
-
-                _compilationCountdown.Wait();
-                _compilationCountdown = null;
-            }
+            Parallel.ForEach(
+                methodsToCompile,
+                new ParallelOptions { MaxDegreeOfParallelism = _parallelism },
+                CompileSingleMethod);
         }
 
         private void CompileSingleThreaded(List<ScannedMethodNode> methodsToCompile)
@@ -142,11 +133,6 @@ private void CompileSingleMethod(ScannedMethodNode methodCodeNodeNeedingCode)
             {
                 throw new CodeGenerationFailedException(method, ex);
             }
-            finally
-            {
-                if (_compilationCountdown != null)
-                    _compilationCountdown.Signal();
-            }
         }
 
         ILScanResults IILScanner.Scan()

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ILScannerBuilder.cs

@@ -27,7 +27,7 @@ public sealed class ILScannerBuilder
         private IEnumerable<ICompilationRootProvider> _compilationRoots = Array.Empty<ICompilationRootProvider>();
         private MetadataManager _metadataManager;
         private InteropStubManager _interopStubManager = new EmptyInteropStubManager();
-        private bool _singleThreaded;
+        private int _parallelism = -1;
 
         internal ILScannerBuilder(CompilerTypeSystemContext context, CompilationModuleGroup compilationGroup, NameMangler mangler, ILProvider ilProvider, PreinitializationManager preinitializationManager)
         {
@@ -63,9 +63,9 @@ public ILScannerBuilder UseInteropStubManager(InteropStubManager interopStubMana
             return this;
         }
 
-        public ILScannerBuilder UseSingleThread(bool enable)
+        public ILScannerBuilder UseParallelism(int parallelism)
         {
-            _singleThreaded = enable;
+            _parallelism = parallelism;
             return this;
         }
 
@@ -74,7 +74,7 @@ public IILScanner ToILScanner()
             var nodeFactory = new ILScanNodeFactory(_context, _compilationGroup, _metadataManager, _interopStubManager, _nameMangler, _preinitializationManager);
             DependencyAnalyzerBase<NodeFactory> graph = _dependencyTrackingLevel.CreateDependencyGraph(nodeFactory);
 
-            return new ILScanner(graph, nodeFactory, _compilationRoots, _ilProvider, new NullDebugInformationProvider(), _logger, _singleThreaded);
+            return new ILScanner(graph, nodeFactory, _compilationRoots, _ilProvider, new NullDebugInformationProvider(), _logger, _parallelism);
         }
     }
 }

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs

@@ -33,7 +33,6 @@ public sealed class ReadyToRunCodegenCompilationBuilder : CompilationBuilder
         private string _perfMapPath;
         private int _perfMapFormatVersion;
         private bool _generateProfileFile;
-        private int _parallelism;
         Func<MethodDesc, string> _printReproInstructions;
         private InstructionSetSupport _instructionSetSupport;
         private ProfileDataManager _profileData;
@@ -170,12 +169,6 @@ public ReadyToRunCodegenCompilationBuilder UseProfileFile(bool generateProfileFi
             return this;
         }
 
-        public ReadyToRunCodegenCompilationBuilder UseParallelism(int parallelism)
-        {
-            _parallelism = parallelism;
-            return this;
-        }
-
         public ReadyToRunCodegenCompilationBuilder UsePrintReproInstructions(Func<MethodDesc, string> printReproInstructions)
         {
             _printReproInstructions = printReproInstructions;

src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/RyuJitCompilation.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
 using System.Threading;
+using System.Threading.Tasks;
 
 using ILCompiler.DependencyAnalysis;
 using ILCompiler.DependencyAnalysisFramework;
@@ -21,10 +22,10 @@ public sealed class RyuJitCompilation : Compilation
         private readonly ConditionalWeakTable<Thread, CorInfoImpl> _corinfos = new ConditionalWeakTable<Thread, CorInfoImpl>();
         internal readonly RyuJitCompilationOptions _compilationOptions;
         private readonly ExternSymbolMappedField _hardwareIntrinsicFlags;
-        private CountdownEvent _compilationCountdown;
         private readonly Dictionary<string, InstructionSet> _instructionSetMap;
         private readonly ProfileDataManager _profileDataManager;
         private readonly MethodImportationErrorProvider _methodImportationErrorProvider;
+        private readonly int _parallelism;
 
         public InstructionSetSupport InstructionSetSupport { get; }
 
@@ -40,7 +41,8 @@ internal RyuJitCompilation(
             InstructionSetSupport instructionSetSupport,
             ProfileDataManager profileDataManager,
             MethodImportationErrorProvider errorProvider,
-            RyuJitCompilationOptions options)
+            RyuJitCompilationOptions options,
+            int parallelism)
             : base(dependencyGraph, nodeFactory, roots, ilProvider, debugInformationProvider, devirtualizationManager, inliningPolicy, logger)
         {
             _compilationOptions = options;
@@ -59,6 +61,8 @@ internal RyuJitCompilation(
             _profileDataManager = profileDataManager;
 
             _methodImportationErrorProvider = errorProvider;
+
+            _parallelism = parallelism;
         }
 
         public ProfileDataManager ProfileData => _profileDataManager;
@@ -125,7 +129,7 @@ protected override void ComputeDependencyNodeDependencies(List<DependencyNodeCor
                 methodsToCompile.Add(methodCodeNodeNeedingCode);
             }
 
-            if ((_compilationOptions & RyuJitCompilationOptions.SingleThreadedCompilation) != 0)
+            if (_parallelism == 1)
             {
                 CompileSingleThreaded(methodsToCompile);
             }
@@ -141,23 +145,10 @@ private void CompileMultiThreaded(List<MethodCodeNode> methodsToCompile)
                 Logger.Writer.WriteLine($"Compiling {methodsToCompile.Count} methods...");
             }
 
-            WaitCallback compileSingleMethodDelegate = m =>
-            {
-                CorInfoImpl corInfo = _corinfos.GetValue(Thread.CurrentThread, thread => new CorInfoImpl(this));
-                CompileSingleMethod(corInfo, (MethodCodeNode)m);
-            };
-
-            using (_compilationCountdown = new CountdownEvent(methodsToCompile.Count))
-            {
-
-                foreach (MethodCodeNode methodCodeNodeNeedingCode in methodsToCompile)
-                {
-                    ThreadPool.QueueUserWorkItem(compileSingleMethodDelegate, methodCodeNodeNeedingCode);
-                }
-
-                _compilationCountdown.Wait();
-                _compilationCountdown = null;
-            }
+            Parallel.ForEach(
+                methodsToCompile,
+                new ParallelOptions { MaxDegreeOfParallelism = _parallelism },
+                CompileSingleMethod);
         }
 
 
@@ -176,52 +167,50 @@ private void CompileSingleThreaded(List<MethodCodeNode> methodsToCompile)
             }
         }
 
+        private void CompileSingleMethod(MethodCodeNode methodCodeNodeNeedingCode)
+        {
+            CorInfoImpl corInfo = _corinfos.GetValue(Thread.CurrentThread, thread => new CorInfoImpl(this));
+            CompileSingleMethod(corInfo, methodCodeNodeNeedingCode);
+        }
+
         private void CompileSingleMethod(CorInfoImpl corInfo, MethodCodeNode methodCodeNodeNeedingCode)
         {
-            try
-            {
-                MethodDesc method = methodCodeNodeNeedingCode.Method;
+            MethodDesc method = methodCodeNodeNeedingCode.Method;
 
-                TypeSystemException exception = _methodImportationErrorProvider.GetCompilationError(method);
+            TypeSystemException exception = _methodImportationErrorProvider.GetCompilationError(method);
 
-                // If we previously failed to import the method, do not try to import it again and go
-                // directly to the error path.
-                if (exception == null)
+            // If we previously failed to import the method, do not try to import it again and go
+            // directly to the error path.
+            if (exception == null)
+            {
+                try
                 {
-                    try
-                    {
-                        corInfo.CompileMethod(methodCodeNodeNeedingCode);
-                    }
-                    catch (TypeSystemException ex)
-                    {
-                        exception = ex;
-                    }
+                    corInfo.CompileMethod(methodCodeNodeNeedingCode);
                 }
-
-                if (exception != null)
+                catch (TypeSystemException ex)
                 {
-                    // TODO: fail compilation if a switch was passed
-
-                    // Try to compile the method again, but with a throwing method body this time.
-                    MethodIL throwingIL = TypeSystemThrowingILEmitter.EmitIL(method, exception);
-                    corInfo.CompileMethod(methodCodeNodeNeedingCode, throwingIL);
-
-                    if (exception is TypeSystemException.InvalidProgramException
-                        && method.OwningType is MetadataType mdOwningType
-                        && mdOwningType.HasCustomAttribute("System.Runtime.InteropServices", "ClassInterfaceAttribute"))
-                    {
-                        Logger.LogWarning("COM interop is not supported with full ahead of time compilation", 3052, method, MessageSubCategory.AotAnalysis);
-                    }
-                    else
-                    {
-                        Logger.LogWarning($"Method will always throw because: {exception.Message}", 1005, method, MessageSubCategory.AotAnalysis);
-                    }
+                    exception = ex;
                 }
             }
-            finally
+
+            if (exception != null)
             {
-                if (_compilationCountdown != null)
-                    _compilationCountdown.Signal();
+                // TODO: fail compilation if a switch was passed
+
+                // Try to compile the method again, but with a throwing method body this time.
+                MethodIL throwingIL = TypeSystemThrowingILEmitter.EmitIL(method, exception);
+                corInfo.CompileMethod(methodCodeNodeNeedingCode, throwingIL);
+
+                if (exception is TypeSystemException.InvalidProgramException
+                    && method.OwningType is MetadataType mdOwningType
+                    && mdOwningType.HasCustomAttribute("System.Runtime.InteropServices", "ClassInterfaceAttribute"))
+                {
+                    Logger.LogWarning("COM interop is not supported with full ahead of time compilation", 3052, method, MessageSubCategory.AotAnalysis);
+                }
+                else
+                {
+                    Logger.LogWarning($"Method will always throw because: {exception.Message}", 1005, method, MessageSubCategory.AotAnalysis);
+                }
             }
         }
 
@@ -252,7 +241,6 @@ public override MethodIL GetMethodIL(MethodDesc method)
     public enum RyuJitCompilationOptions
     {
         MethodBodyFolding = 0x1,
-        SingleThreadedCompilation = 0x2,
-        ControlFlowGuardAnnotations = 0x4,
+        ControlFlowGuardAnnotations = 0x2,
     }
 }

src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/RyuJitCompilationBuilder.cs

@@ -107,17 +107,14 @@ public override ICompilation ToCompilation()
             if (_methodBodyFolding)
                 options |= RyuJitCompilationOptions.MethodBodyFolding;
 
-            if (_singleThreaded)
-                options |= RyuJitCompilationOptions.SingleThreadedCompilation;
-
             if ((_mitigationOptions & SecurityMitigationOptions.ControlFlowGuardAnnotations) != 0)
                 options |= RyuJitCompilationOptions.ControlFlowGuardAnnotations;
 
             var factory = new RyuJitNodeFactory(_context, _compilationGroup, _metadataManager, _interopStubManager, _nameMangler, _vtableSliceProvider, _dictionaryLayoutProvider, GetPreinitializationManager());
 
             JitConfigProvider.Initialize(_context.Target, jitFlagBuilder.ToArray(), _ryujitOptions);
             DependencyAnalyzerBase<NodeFactory> graph = CreateDependencyGraph(factory, new ObjectNode.ObjectNodeComparer(new CompilerComparer()));
-            return new RyuJitCompilation(graph, factory, _compilationRoots, _ilProvider, _debugInformationProvider, _logger, _devirtualizationManager, _inliningPolicy ?? _compilationGroup, _instructionSetSupport, _profileDataManager, _methodImportationErrorProvider, options);
+            return new RyuJitCompilation(graph, factory, _compilationRoots, _ilProvider, _debugInformationProvider, _logger, _devirtualizationManager, _inliningPolicy ?? _compilationGroup, _instructionSetSupport, _profileDataManager, _methodImportationErrorProvider, options, _parallelism);
         }
     }
 }

src/coreclr/tools/aot/ILCompiler.RyuJit/ILCompiler.RyuJit.csproj

@@ -3,7 +3,7 @@
     <OutputType>Library</OutputType>
     <RootNamespace>ILCompiler</RootNamespace>
     <AssemblyName>ILCompiler.RyuJit</AssemblyName>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <EnableDefaultCompileItems>false</EnableDefaultCompileItems>
     <Platforms>x64;x86</Platforms>