diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/CompilationWorklist.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/CompilationWorklist.cs new file mode 100644 index 00000000000000..8d93b9b8c8675d --- /dev/null +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/CompilationWorklist.cs @@ -0,0 +1,198 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.ExceptionServices; +using System.Threading; + +using ILCompiler.DependencyAnalysis; +using ILCompiler.DependencyAnalysisFramework; + +namespace ILCompiler +{ + internal delegate void CompilationWorkerAction( + DependencyNodeCore item, + ref TWorkerState workerState); + + internal sealed class CompilationWorklist : IDisposable + { + private sealed class Worker + { + public readonly SemaphoreSlim Start = new(0); + public readonly Thread Thread; + public TWorkerState State; + private readonly CompilationWorklist _worklist; + + public Worker(CompilationWorklist worklist) + { + _worklist = worklist; + Thread = new Thread(Run); + } + + private void Run() => _worklist.RunWorkerLoop(this); + } + + private readonly ManualResetEventSlim _complete = new(); + private readonly int _parallelism; + private readonly CompilationWorkerAction _processItem; + private TWorkerState _mainWorkerState; + private IReadOnlyList> _items; + private ExceptionDispatchInfo _exception; + private Worker[] _workers; + private int _nextIndex; + private int _startedWorkerCount; + private int _workersRemaining; + private volatile bool _stopping; + + public CompilationWorklist(int parallelism, CompilationWorkerAction processItem) + { + _parallelism = parallelism; + _processItem = processItem; + } + + public void Run(IReadOnlyList> items) + { + EnsureWorkers(); + Debug.Assert(_items is null); + + _items = items; + _exception = null; + _nextIndex = -1; + _workersRemaining = _parallelism; + _complete.Reset(); + + foreach (Worker worker in _workers) + { + worker.Start.Release(); + } + + try + { + ProcessItems(ref _mainWorkerState); + } + finally + { + WaitForCompletion()?.Throw(); + } + } + + public void Dispose() + { + if (_stopping) + { + return; + } + + _stopping = true; + + if (_workers is not null) + { + for (int i = 0; i < _startedWorkerCount; i++) + { + _workers[i].Start.Release(); + } + + for (int i = 0; i < _startedWorkerCount; i++) + { + Worker worker = _workers[i]; + worker.Thread.Join(); + worker.Start.Dispose(); + worker.State = default; + } + } + + _mainWorkerState = default; + _complete.Dispose(); + } + + private void EnsureWorkers() + { + if (_workers is not null) + { + return; + } + + _workers = new Worker[_parallelism - 1]; + for (int i = 0; i < _workers.Length; i++) + { + Worker worker = new(this); + _workers[i] = worker; + worker.Thread.Start(); + _startedWorkerCount++; + } + } + + private void RunWorkerLoop(Worker worker) + { + while (true) + { + worker.Start.Wait(); + if (_stopping) + { + return; + } + + ProcessItems(ref worker.State); + } + } + + private void ProcessItems(ref TWorkerState workerState) + { + try + { + while (TryTake(out DependencyNodeCore item)) + { + _processItem(item, ref workerState); + } + } + catch (Exception ex) + { + lock (this) + { + _exception ??= ExceptionDispatchInfo.Capture(ex); + } + } + finally + { + if (Interlocked.Decrement(ref _workersRemaining) == 0) + { + _complete.Set(); + } + } + } + + private bool TryTake(out DependencyNodeCore item) + { + lock (this) + { + if (_exception is not null) + { + item = null; + return false; + } + + int index = ++_nextIndex; + IReadOnlyList> items = _items; + if ((uint)index >= (uint)items.Count) + { + item = null; + return false; + } + + item = items[index]; + return true; + } + } + + private ExceptionDispatchInfo WaitForCompletion() + { + _complete.Wait(); + _items = null; + ExceptionDispatchInfo exception = _exception; + _exception = null; + return exception; + } + } +} diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs index 18d4f644407ddb..431e7258449b85 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs @@ -283,7 +283,6 @@ public sealed class ReadyToRunCodegenCompilation : Compilation private readonly bool _resilient; private readonly int _parallelism; - private readonly CorInfoImpl[] _corInfoImpls; private readonly bool _generateMapFile; private readonly bool _generateMapCsvFile; @@ -358,7 +357,7 @@ internal ReadyToRunCodegenCompilation( _computedFixedLayoutTypesUncached = IsLayoutFixedInCurrentVersionBubbleInternal; _resilient = resilient; _parallelism = parallelism; - _corInfoImpls = new CorInfoImpl[_parallelism]; + _compilationWorklist = new CompilationWorklist(_parallelism, CompileOneMethod); _generateMapFile = generateMapFile; _generateMapCsvFile = generateMapCsvFile; _generatePdbFile = generatePdbFile; @@ -402,9 +401,8 @@ public override void Compile(string outputFile) { _dependencyGraph.ComputeMarkedNodes(); - _doneAllCompiling = true; - Array.Clear(_corInfoImpls); - _compilationThreadSemaphore.Release(_parallelism); + // Release per-worker JIT state before object emission to reduce peak memory usage. + _compilationWorklist.Dispose(); var nodes = _dependencyGraph.MarkedNodeList; @@ -706,15 +704,14 @@ public void PrepareForCompilationRetry(MethodWithGCInfo methodToBeRecompiled, IE } } - [ThreadStatic] - private static int s_methodsCompiledPerThread = 0; + private struct WorkerState + { + public CorInfoImpl CorInfoImpl; + public int MethodsCompiled; + } - private SemaphoreSlim _compilationThreadSemaphore = new(0); - private volatile IEnumerator> _currentCompilationMethodList; - private volatile bool _doneAllCompiling; - private int _finishedThreadCount; - private ManualResetEventSlim _compilationSessionComplete = new ManualResetEventSlim(); - private bool _hasCreatedCompilationThreads = false; + private readonly CompilationWorklist _compilationWorklist; + private int _compilationSessionGeneratedColdCode; private bool _hasAddedAsyncReferences = false; protected override void ComputeDependencyNodeDependencies(List> obj) @@ -762,7 +759,7 @@ protected override void ComputeDependencyNodeDependencies(List 0) { @@ -776,7 +773,7 @@ protected override void ComputeDependencyNodeDependencies(List 1000 || _methodILCache.ILProvider.Version != _methodILCache.ExpectedILProviderVersion) _methodILCache = new ILCache(_methodILCache.ILProvider, NodeFactory.CompilationModuleGroup); } + } - void CompileMethodList(IEnumerable> methodList) - { - // Disable generation of new tokens across the multi-threaded compile - NodeFactory.ManifestMetadataTable._mutableModule.DisableNewTokens = true; - - if (_parallelism == 1) - { - foreach (var dependency in methodList) - CompileOneMethod(dependency, 0); - } - else - { - _currentCompilationMethodList = methodList.GetEnumerator(); - _finishedThreadCount = 0; - _compilationSessionComplete.Reset(); - - if (!_hasCreatedCompilationThreads) - { - for (int compilationThreadId = 1; compilationThreadId < _parallelism; compilationThreadId++) - { - new Thread(CompilationThread).Start((object)compilationThreadId); - } - _hasCreatedCompilationThreads = true; - } - - _compilationThreadSemaphore.Release(_parallelism - 1); - CompileOnThread(0); - _compilationSessionComplete.Wait(); - } + private bool CompileMethodList(IReadOnlyList> methodList) + { + _compilationSessionGeneratedColdCode = 0; - // Re-enable generation of new tokens after the multi-threaded compile + NodeFactory.ManifestMetadataTable._mutableModule.DisableNewTokens = true; + try + { + _compilationWorklist.Run(methodList); + return Volatile.Read(ref _compilationSessionGeneratedColdCode) != 0; + } + finally + { NodeFactory.ManifestMetadataTable._mutableModule.DisableNewTokens = false; } + } - void CompilationThread(object objThreadId) + private void CompileOneMethod(DependencyNodeCore dependency, ref WorkerState workerState) + { + MethodWithGCInfo methodCodeNodeNeedingCode = dependency as MethodWithGCInfo; + if (methodCodeNodeNeedingCode == null) { - while (true) + if (dependency is DeferredTillPhaseNode deferredPhaseNode) { - _compilationThreadSemaphore.Wait(); - lock (this) - { - if (_doneAllCompiling) - return; - } - CompileOnThread((int)objThreadId); + if (Logger.IsVerbose) + _logger.Writer.WriteLine($"Moved to phase {_nodeFactory.CompilationCurrentPhase}"); + deferredPhaseNode.NotifyCurrentPhase(_nodeFactory.CompilationCurrentPhase); + return; } } - void CompileOnThread(int compilationThreadId) - { - var compilationMethodList = _currentCompilationMethodList; - while (true) - { - DependencyNodeCore dependency; - lock (compilationMethodList) - { - if (!compilationMethodList.MoveNext()) - { - if (Interlocked.Increment(ref _finishedThreadCount) == _parallelism) - _compilationSessionComplete.Set(); + Debug.Assert((_nodeFactory.CompilationCurrentPhase == 0) || ((_nodeFactory.CompilationCurrentPhase == 2) && !_finishedFirstCompilationRunInPhase2)); - return; - } - dependency = compilationMethodList.Current; - } + MethodDesc method = methodCodeNodeNeedingCode.Method; - CompileOneMethod(dependency, compilationThreadId); - } + if (Logger.IsVerbose) + { + string methodName = method.ToString(); + Logger.Writer.WriteLine("Compiling " + methodName); } - void CompileOneMethod(DependencyNodeCore dependency, int compileThreadId) + if (_nodeFactory.OptimizationFlags.PrintReproArgs) { - MethodWithGCInfo methodCodeNodeNeedingCode = dependency as MethodWithGCInfo; - if (methodCodeNodeNeedingCode == null) + Logger.Writer.WriteLine($"Single method repro args:{GetReproInstructions(method)}"); + } + + try + { + using (PerfEventSource.StartStopEvents.JitMethodEvents()) { - if (dependency is DeferredTillPhaseNode deferredPhaseNode) + workerState.MethodsCompiled++; + if (workerState.CorInfoImpl is null || + (_parallelism != 1 && (workerState.MethodsCompiled % 3000) == 0)) { - if (Logger.IsVerbose) - _logger.Writer.WriteLine($"Moved to phase {_nodeFactory.CompilationCurrentPhase}"); - deferredPhaseNode.NotifyCurrentPhase(_nodeFactory.CompilationCurrentPhase); - return; + // Periodically create a new CorInfoImpl to clear out stale caches. For single-threaded + // compilation, reuse one instance so SuperPMI can rely on non-reuse of ObjectToHandle handles. + workerState.CorInfoImpl = new CorInfoImpl(this); } - } - - Debug.Assert((_nodeFactory.CompilationCurrentPhase == 0) || ((_nodeFactory.CompilationCurrentPhase == 2) && !_finishedFirstCompilationRunInPhase2)); - - MethodDesc method = methodCodeNodeNeedingCode.Method; - if (Logger.IsVerbose) - { - string methodName = method.ToString(); - Logger.Writer.WriteLine("Compiling " + methodName); - } - - if (_nodeFactory.OptimizationFlags.PrintReproArgs) - { - Logger.Writer.WriteLine($"Single method repro args:{GetReproInstructions(method)}"); - } - - try - { - using (PerfEventSource.StartStopEvents.JitMethodEvents()) + CorInfoImpl corInfoImpl = workerState.CorInfoImpl; + corInfoImpl.CompileMethod(methodCodeNodeNeedingCode, Logger); + if (corInfoImpl.HasColdCode) { - s_methodsCompiledPerThread++; - bool createNewCorInfoImpl = false; - - if (_corInfoImpls[compileThreadId] == null) - createNewCorInfoImpl = true; - else - { - if (_parallelism == 1) - { - // Create only 1 CorInfoImpl if not using parallelism - // This allows SuperPMI to rely on non-reuse of handles in ObjectToHandle - } - else - { - // Periodically create a new CorInfoImpl to clear out stale caches - // This is done as the CorInfoImpl holds a cache of data structures visible to the JIT - // Those data structures include both structures which will last for the lifetime of the compilation - // process, as well as various temporary structures that would really be better off with thread lifetime. - if ((s_methodsCompiledPerThread % 3000) == 0) - { - createNewCorInfoImpl = true; - } - } - } - - if (createNewCorInfoImpl) - _corInfoImpls[compileThreadId] = new CorInfoImpl(this); - - CorInfoImpl corInfoImpl = _corInfoImpls[compileThreadId]; - corInfoImpl.CompileMethod(methodCodeNodeNeedingCode, Logger); - if (corInfoImpl.HasColdCode) - { - generatedColdCode = true; - } + Volatile.Write(ref _compilationSessionGeneratedColdCode, 1); } } - catch (TypeSystemException ex) - { - // If compilation fails, don't emit code for this method. It will be Jitted at runtime - if (Logger.IsVerbose) - Logger.Writer.WriteLine($"Warning: Method `{method}` was not compiled because: {ex.Message}"); - } - catch (RequiresRuntimeJitException ex) - { - if (Logger.IsVerbose) - Logger.Writer.WriteLine($"Info: Method `{method}` was not compiled because `{ex.Message}` requires runtime JIT"); - } - catch (CodeGenerationFailedException ex) when (_resilient) - { - if (Logger.IsVerbose) - Logger.Writer.WriteLine($"Warning: Method `{method}` was not compiled because `{ex.Message}` requires runtime JIT"); - } + } + catch (TypeSystemException ex) + { + // If compilation fails, don't emit code for this method. It will be Jitted at runtime + if (Logger.IsVerbose) + Logger.Writer.WriteLine($"Warning: Method `{method}` was not compiled because: {ex.Message}"); + } + catch (RequiresRuntimeJitException ex) + { + if (Logger.IsVerbose) + Logger.Writer.WriteLine($"Info: Method `{method}` was not compiled because `{ex.Message}` requires runtime JIT"); + } + catch (CodeGenerationFailedException ex) when (_resilient) + { + if (Logger.IsVerbose) + Logger.Writer.WriteLine($"Warning: Method `{method}` was not compiled because `{ex.Message}` requires runtime JIT"); } } @@ -1086,6 +1010,8 @@ public ISymbolNode GetFieldRvaData(FieldDesc field) public override void Dispose() { + _compilationWorklist.Dispose(); + // Workaround for https://github.com/dotnet/runtime/issues/23103. // ManifestMetadataTable.Dispose() allows to break circular reference // ConcurrentBag -> EcmaModule -> EcmaAssembly -> ReadyToRunCompilerContext -> ... -> ConcurrentBag. diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj index 23c99268bc5330..546a92e6bb9714 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj @@ -217,6 +217,7 @@ + diff --git a/src/coreclr/tools/aot/crossgen2/Program.cs b/src/coreclr/tools/aot/crossgen2/Program.cs index 691add6dd78b2a..68fe3325976669 100644 --- a/src/coreclr/tools/aot/crossgen2/Program.cs +++ b/src/coreclr/tools/aot/crossgen2/Program.cs @@ -351,360 +351,385 @@ private void RunSingleCompilation(Dictionary inFilePaths, Instru using (PerfEventSource.StartStopEvents.CompilationEvents()) { - ICompilation compilation; - using (PerfEventSource.StartStopEvents.LoadingEvents()) - { - List inputModules = new List(); - List rootingModules = new List(); - HashSet crossModuleInlineableCode = new HashSet(); + bool determinismCheckFailed; + using var compilation = (ReadyToRunCodegenCompilation)BuildCompilation( + inFilePaths, + versionBubbleModulesHash, + unrootedInputFilePaths, + compositeRootPath, + typeSystemContext, + dgmlLogFileName, + outFile, + logger, + instructionSetSupport + ); + compilation.Compile(outFile); - foreach (var inputFile in inFilePaths) - { - EcmaModule module = typeSystemContext.GetModuleFromPath(inputFile.Value); - inputModules.Add(module); - rootingModules.Add(module); - versionBubbleModulesHash.Add(module); + if (dgmlLogFileName != null) + compilation.WriteDependencyLog(dgmlLogFileName); + determinismCheckFailed = compilation.DeterminismCheckFailed; - if (!_command.CompositeOrInputBubble) - { - break; - } - } + if (determinismCheckFailed) + throw new Exception("Determinism Check Failed"); + } + } + + private ICompilation BuildCompilation( + Dictionary inFilePaths, + HashSet versionBubbleModulesHash, + Dictionary unrootedInputFilePaths, + string compositeRootPath, + ReadyToRunCompilerContext typeSystemContext, + string dgmlLogFileName, + string outFile, + Logger logger, + InstructionSetSupport instructionSetSupport + ) + { + using (PerfEventSource.StartStopEvents.LoadingEvents()) + { + List inputModules = new List(); + List rootingModules = new List(); + HashSet crossModuleInlineableCode = new HashSet(); - foreach (var unrootedInputFile in unrootedInputFilePaths) + foreach (var inputFile in inFilePaths) + { + EcmaModule module = typeSystemContext.GetModuleFromPath(inputFile.Value); + inputModules.Add(module); + rootingModules.Add(module); + versionBubbleModulesHash.Add(module); + + + if (!_command.CompositeOrInputBubble) { - EcmaModule module = typeSystemContext.GetModuleFromPath(unrootedInputFile.Value); - inputModules.Add(module); - versionBubbleModulesHash.Add(module); + break; } + } + + foreach (var unrootedInputFile in unrootedInputFilePaths) + { + EcmaModule module = typeSystemContext.GetModuleFromPath(unrootedInputFile.Value); + inputModules.Add(module); + versionBubbleModulesHash.Add(module); + } - string[] crossModuleInlining = Get(_command.CrossModuleInlining); - if (crossModuleInlining.Length > 0) + string[] crossModuleInlining = Get(_command.CrossModuleInlining); + if (crossModuleInlining.Length > 0) + { + foreach (var crossModulePgoAssemblyName in crossModuleInlining) { - foreach (var crossModulePgoAssemblyName in crossModuleInlining) + foreach (var module in _referenceableModules) { - foreach (var module in _referenceableModules) + if (!versionBubbleModulesHash.Contains(module)) { - if (!versionBubbleModulesHash.Contains(module)) + if (crossModulePgoAssemblyName == "*" || + (String.Compare(crossModulePgoAssemblyName, module.Assembly.GetName().Name, StringComparison.OrdinalIgnoreCase) == 0)) { - if (crossModulePgoAssemblyName == "*" || - (String.Compare(crossModulePgoAssemblyName, module.Assembly.GetName().Name, StringComparison.OrdinalIgnoreCase) == 0)) - { - crossModuleInlineableCode.Add((EcmaModule)module); - } + crossModuleInlineableCode.Add((EcmaModule)module); } } } } + } - // - // Initialize compilation group and compilation roots - // + // + // Initialize compilation group and compilation roots + // - // Single method mode? - MethodDesc singleMethod = CheckAndParseSingleMethodModeArguments(typeSystemContext); + // Single method mode? + MethodDesc singleMethod = CheckAndParseSingleMethodModeArguments(typeSystemContext); - List mibcFiles = new List(); - foreach (var file in Get(_command.MibcFilePaths)) - { - mibcFiles.Add(file); - } + List mibcFiles = new List(); + foreach (var file in Get(_command.MibcFilePaths)) + { + mibcFiles.Add(file); + } - List versionBubbleModules = new List(versionBubbleModulesHash); - bool composite = Get(_command.Composite); - if (!composite && inputModules.Count != 1) - { - throw new Exception(string.Format(SR.ErrorMultipleInputFilesCompositeModeOnly, string.Join("; ", inputModules))); - } + List versionBubbleModules = new List(versionBubbleModulesHash); + bool composite = Get(_command.Composite); + if (!composite && inputModules.Count != 1) + { + throw new Exception(string.Format(SR.ErrorMultipleInputFilesCompositeModeOnly, string.Join("; ", inputModules))); + } + + string rtrHeaderSymbolName = Get(_command.ReadyToRunHeaderSymbolName); - string rtrHeaderSymbolName = Get(_command.ReadyToRunHeaderSymbolName); + ReadyToRunContainerFormat format = Get(_command.OutputFormat); + if (format == ReadyToRunContainerFormat.PE && typeSystemContext.Target.Architecture == TargetArchitecture.Wasm32) + { + format = ReadyToRunContainerFormat.Wasm; + } + if (!composite && format != ReadyToRunContainerFormat.PE && format != ReadyToRunContainerFormat.Wasm) + { + throw new Exception(string.Format(SR.ErrorContainerFormatRequiresComposite, format)); + } - ReadyToRunContainerFormat format = Get(_command.OutputFormat); - if (format == ReadyToRunContainerFormat.PE && typeSystemContext.Target.Architecture == TargetArchitecture.Wasm32) + if (rtrHeaderSymbolName is not null) + { + if (!composite) { - format = ReadyToRunContainerFormat.Wasm; + throw new Exception(SR.ErrorReadyToRunHeaderSymbolNameRequiresComposite); } - if (!composite && format != ReadyToRunContainerFormat.PE && format != ReadyToRunContainerFormat.Wasm) + + if (string.IsNullOrWhiteSpace(rtrHeaderSymbolName)) { - throw new Exception(string.Format(SR.ErrorContainerFormatRequiresComposite, format)); + throw new Exception(SR.ErrorReadyToRunHeaderSymbolNameEmpty); } + } - if (rtrHeaderSymbolName is not null) - { - if (!composite) - { - throw new Exception(SR.ErrorReadyToRunHeaderSymbolNameRequiresComposite); - } + bool compileBubbleGenerics = Get(_command.CompileBubbleGenerics); + ReadyToRunCompilationModuleGroupBase compilationGroup; + List compilationRoots = new List(); + ReadyToRunCompilationModuleGroupConfig groupConfig = new ReadyToRunCompilationModuleGroupConfig(); + groupConfig.Context = typeSystemContext; + groupConfig.IsCompositeBuildMode = composite; + groupConfig.IsInputBubble = _inputBubble; + groupConfig.CompilationModuleSet = inputModules; + groupConfig.VersionBubbleModuleSet = versionBubbleModules; + groupConfig.CompileGenericDependenciesFromVersionBubbleModuleSet = compileBubbleGenerics; + groupConfig.CrossModuleGenericCompilation = crossModuleInlineableCode.Count > 0; + groupConfig.CrossModuleInlining = groupConfig.CrossModuleGenericCompilation; // Currently we set these flags to the same values + groupConfig.CrossModuleInlineable = crossModuleInlineableCode; + groupConfig.CompileAllPossibleCrossModuleCode = false; + groupConfig.InstructionSetSupport = instructionSetSupport; + + // Handle non-local generics command line option + ModuleDesc nonLocalGenericsHome = compileBubbleGenerics ? inputModules[0] : null; + string nonLocalGenericsModule = Get(_command.NonLocalGenericsModule); + if (nonLocalGenericsModule == "*") + { + groupConfig.CompileAllPossibleCrossModuleCode = true; + nonLocalGenericsHome = inputModules[0]; + } + else if (nonLocalGenericsModule == "") + { + // Nothing was specified + } + else + { + bool matchFound = false; - if (string.IsNullOrWhiteSpace(rtrHeaderSymbolName)) + // Allow module to be specified by assembly name or by filename + if (nonLocalGenericsModule.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)) + nonLocalGenericsModule = Path.GetFileNameWithoutExtension(nonLocalGenericsModule); + foreach (var module in inputModules) + { + if (String.Compare(module.Assembly.GetName().Name, nonLocalGenericsModule, StringComparison.OrdinalIgnoreCase) == 0) { - throw new Exception(SR.ErrorReadyToRunHeaderSymbolNameEmpty); + matchFound = true; + nonLocalGenericsHome = module; + groupConfig.CompileAllPossibleCrossModuleCode = true; + break; } } - bool compileBubbleGenerics = Get(_command.CompileBubbleGenerics); - ReadyToRunCompilationModuleGroupBase compilationGroup; - List compilationRoots = new List(); - ReadyToRunCompilationModuleGroupConfig groupConfig = new ReadyToRunCompilationModuleGroupConfig(); - groupConfig.Context = typeSystemContext; - groupConfig.IsCompositeBuildMode = composite; - groupConfig.IsInputBubble = _inputBubble; - groupConfig.CompilationModuleSet = inputModules; - groupConfig.VersionBubbleModuleSet = versionBubbleModules; - groupConfig.CompileGenericDependenciesFromVersionBubbleModuleSet = compileBubbleGenerics; - groupConfig.CrossModuleGenericCompilation = crossModuleInlineableCode.Count > 0; - groupConfig.CrossModuleInlining = groupConfig.CrossModuleGenericCompilation; // Currently we set these flags to the same values - groupConfig.CrossModuleInlineable = crossModuleInlineableCode; - groupConfig.CompileAllPossibleCrossModuleCode = false; - groupConfig.InstructionSetSupport = instructionSetSupport; - - // Handle non-local generics command line option - ModuleDesc nonLocalGenericsHome = compileBubbleGenerics ? inputModules[0] : null; - string nonLocalGenericsModule = Get(_command.NonLocalGenericsModule); - if (nonLocalGenericsModule == "*") - { - groupConfig.CompileAllPossibleCrossModuleCode = true; - nonLocalGenericsHome = inputModules[0]; - } - else if (nonLocalGenericsModule == "") + if (!matchFound) { - // Nothing was specified - } - else - { - bool matchFound = false; - - // Allow module to be specified by assembly name or by filename - if (nonLocalGenericsModule.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)) - nonLocalGenericsModule = Path.GetFileNameWithoutExtension(nonLocalGenericsModule); - foreach (var module in inputModules) + foreach (var module in _referenceableModules) { if (String.Compare(module.Assembly.GetName().Name, nonLocalGenericsModule, StringComparison.OrdinalIgnoreCase) == 0) { matchFound = true; - nonLocalGenericsHome = module; - groupConfig.CompileAllPossibleCrossModuleCode = true; break; } } if (!matchFound) { - foreach (var module in _referenceableModules) - { - if (String.Compare(module.Assembly.GetName().Name, nonLocalGenericsModule, StringComparison.OrdinalIgnoreCase) == 0) - { - matchFound = true; - break; - } - } - - if (!matchFound) - { - throw new CommandLineException(string.Format(SR.ErrorNonLocalGenericsModule, nonLocalGenericsModule)); - } + throw new CommandLineException(string.Format(SR.ErrorNonLocalGenericsModule, nonLocalGenericsModule)); } } + } - bool compileNoMethods = Get(_command.CompileNoMethods); - if (singleMethod != null) - { - // Compiling just a single method - compilationGroup = new SingleMethodCompilationModuleGroup( - groupConfig, - singleMethod); - compilationRoots.Add(new SingleMethodRootProvider(singleMethod)); - } - else if (compileNoMethods) - { - compilationGroup = new NoMethodsCompilationModuleGroup(groupConfig); - } - else - { - // Single assembly compilation. - compilationGroup = new ReadyToRunSingleAssemblyCompilationModuleGroup(groupConfig); - } + bool compileNoMethods = Get(_command.CompileNoMethods); + if (singleMethod != null) + { + // Compiling just a single method + compilationGroup = new SingleMethodCompilationModuleGroup( + groupConfig, + singleMethod); + compilationRoots.Add(new SingleMethodRootProvider(singleMethod)); + } + else if (compileNoMethods) + { + compilationGroup = new NoMethodsCompilationModuleGroup(groupConfig); + } + else + { + // Single assembly compilation. + compilationGroup = new ReadyToRunSingleAssemblyCompilationModuleGroup(groupConfig); + } - // R2R field layout needs compilation group information - typeSystemContext.SetCompilationGroup(compilationGroup); + // R2R field layout needs compilation group information + typeSystemContext.SetCompilationGroup(compilationGroup); - // Load any profiles generated by method call chain analyis - CallChainProfile jsonProfile = null; - string callChainProfileFile = Get(_command.CallChainProfileFile); - if (!string.IsNullOrEmpty(callChainProfileFile)) - { - jsonProfile = new CallChainProfile(callChainProfileFile, typeSystemContext, _referenceableModules); - } + // Load any profiles generated by method call chain analyis + CallChainProfile jsonProfile = null; + string callChainProfileFile = Get(_command.CallChainProfileFile); + if (!string.IsNullOrEmpty(callChainProfileFile)) + { + jsonProfile = new CallChainProfile(callChainProfileFile, typeSystemContext, _referenceableModules); + } - // Examine profile guided information as appropriate - MIbcProfileParser.MibcGroupParseRules parseRule; - if (nonLocalGenericsHome != null) - { - parseRule = MIbcProfileParser.MibcGroupParseRules.VersionBubbleWithCrossModule2; - } - else - { - parseRule = MIbcProfileParser.MibcGroupParseRules.VersionBubbleWithCrossModule1; - } + // Examine profile guided information as appropriate + MIbcProfileParser.MibcGroupParseRules parseRule; + if (nonLocalGenericsHome != null) + { + parseRule = MIbcProfileParser.MibcGroupParseRules.VersionBubbleWithCrossModule2; + } + else + { + parseRule = MIbcProfileParser.MibcGroupParseRules.VersionBubbleWithCrossModule1; + } - ProfileDataManager profileDataManager = - new ProfileDataManager(logger, - _referenceableModules, - inputModules, - versionBubbleModules, - crossModuleInlineableCode, - nonLocalGenericsHome, - mibcFiles, - parseRule, - jsonProfile, - typeSystemContext, - compilationGroup, - Get(_command.EmbedPgoData), - Get(_command.SupportIbc), - crossModuleInlineableCode.Count == 0 ? compilationGroup.VersionsWithMethodBody : compilationGroup.CrossModuleInlineable, - Get(_command.SynthesizeRandomMibc)); - - bool partial = Get(_command.Partial); - compilationGroup.ApplyProfileGuidedOptimizationData(profileDataManager, partial); - - if ((singleMethod == null) && !compileNoMethods) + ProfileDataManager profileDataManager = + new ProfileDataManager(logger, + _referenceableModules, + inputModules, + versionBubbleModules, + crossModuleInlineableCode, + nonLocalGenericsHome, + mibcFiles, + parseRule, + jsonProfile, + typeSystemContext, + compilationGroup, + Get(_command.EmbedPgoData), + Get(_command.SupportIbc), + crossModuleInlineableCode.Count == 0 ? compilationGroup.VersionsWithMethodBody : compilationGroup.CrossModuleInlineable, + Get(_command.SynthesizeRandomMibc)); + + bool partial = Get(_command.Partial); + compilationGroup.ApplyProfileGuidedOptimizationData(profileDataManager, partial); + + if ((singleMethod == null) && !compileNoMethods) + { + // For normal compilations add compilation roots. + foreach (var module in rootingModules) { - // For normal compilations add compilation roots. - foreach (var module in rootingModules) + compilationRoots.Add(new ReadyToRunProfilingRootProvider(module, profileDataManager)); + // If we're doing partial precompilation, only use profile data. + if (!partial) { - compilationRoots.Add(new ReadyToRunProfilingRootProvider(module, profileDataManager)); - // If we're doing partial precompilation, only use profile data. - if (!partial) + if (ReadyToRunVisibilityRootProvider.UseVisibilityBasedRootProvider(module)) { - if (ReadyToRunVisibilityRootProvider.UseVisibilityBasedRootProvider(module)) - { - compilationRoots.Add(new ReadyToRunVisibilityRootProvider(module)); + compilationRoots.Add(new ReadyToRunVisibilityRootProvider(module)); - if (ReadyToRunXmlRootProvider.TryCreateRootProviderFromEmbeddedDescriptorFile(module, out ReadyToRunXmlRootProvider xmlProvider)) - { - compilationRoots.Add(xmlProvider); - } - } - else + if (ReadyToRunXmlRootProvider.TryCreateRootProviderFromEmbeddedDescriptorFile(module, out ReadyToRunXmlRootProvider xmlProvider)) { - compilationRoots.Add(new ReadyToRunLibraryRootProvider(module)); + compilationRoots.Add(xmlProvider); } } - - if (!_command.CompositeOrInputBubble) + else { - break; + compilationRoots.Add(new ReadyToRunLibraryRootProvider(module)); } } - } - if (!typeSystemContext.TargetAllowsRuntimeCodeGeneration && typeSystemContext.BubbleIncludesCoreModule) - { - // For some platforms, we cannot JIT. - // As a result, we need to ensure that we have a fallback implementation for all hardware intrinsics - // that are marked as supported. - // Otherwise, the interpreter won't have an implementation it can call for any non-ReadyToRun code. - compilationRoots.Add(new ReadyToRunHardwareIntrinsicRootProvider(typeSystemContext)); - } - - // In single-file compilation mode, use the assembly's DebuggableAttribute to determine whether to optimize - // or produce debuggable code if an explicit optimization level was not specified on the command line - OptimizationMode optimizationMode = _command.OptimizationMode; - if (optimizationMode == OptimizationMode.None && !Get(_command.OptimizeDisabled) && !composite) - { - System.Diagnostics.Debug.Assert(inputModules.Count == 1); - optimizationMode = ((EcmaAssembly)inputModules[0].Assembly).HasOptimizationsDisabled() ? OptimizationMode.None : OptimizationMode.Blended; - } - - CompositeImageSettings compositeImageSettings = new CompositeImageSettings(); - string compositeKeyFile = Get(_command.CompositeKeyFile); - if (compositeKeyFile != null) - { - byte[] compositeStrongNameKey = File.ReadAllBytes(compositeKeyFile); - if (!IsValidPublicKey(compositeStrongNameKey)) + if (!_command.CompositeOrInputBubble) { - throw new Exception(string.Format(SR.ErrorCompositeKeyFileNotPublicKey)); + break; } - - compositeImageSettings.PublicKey = compositeStrongNameKey.ToImmutableArray(); } + } - if (rtrHeaderSymbolName != null) + if (!typeSystemContext.TargetAllowsRuntimeCodeGeneration && typeSystemContext.BubbleIncludesCoreModule) + { + // For some platforms, we cannot JIT. + // As a result, we need to ensure that we have a fallback implementation for all hardware intrinsics + // that are marked as supported. + // Otherwise, the interpreter won't have an implementation it can call for any non-ReadyToRun code. + compilationRoots.Add(new ReadyToRunHardwareIntrinsicRootProvider(typeSystemContext)); + } + + // In single-file compilation mode, use the assembly's DebuggableAttribute to determine whether to optimize + // or produce debuggable code if an explicit optimization level was not specified on the command line + OptimizationMode optimizationMode = _command.OptimizationMode; + if (optimizationMode == OptimizationMode.None && !Get(_command.OptimizeDisabled) && !composite) + { + System.Diagnostics.Debug.Assert(inputModules.Count == 1); + optimizationMode = ((EcmaAssembly)inputModules[0].Assembly).HasOptimizationsDisabled() ? OptimizationMode.None : OptimizationMode.Blended; + } + + CompositeImageSettings compositeImageSettings = new CompositeImageSettings(); + string compositeKeyFile = Get(_command.CompositeKeyFile); + if (compositeKeyFile != null) + { + byte[] compositeStrongNameKey = File.ReadAllBytes(compositeKeyFile); + if (!IsValidPublicKey(compositeStrongNameKey)) { - compositeImageSettings.ReadyToRunHeaderSymbolName = rtrHeaderSymbolName; + throw new Exception(string.Format(SR.ErrorCompositeKeyFileNotPublicKey)); } - // - // Compile - // - - ReadyToRunCodegenCompilationBuilder builder = new ReadyToRunCodegenCompilationBuilder( - typeSystemContext, compilationGroup, _allInputFilePaths.Values, compositeRootPath); - string compilationUnitPrefix = ""; - builder.UseCompilationUnitPrefix(compilationUnitPrefix); - - ILProvider ilProvider = new ReadyToRunILProvider(compilationGroup); - - DependencyTrackingLevel trackingLevel = dgmlLogFileName == null ? - DependencyTrackingLevel.None : (Get(_command.GenerateFullDgmlLog) ? DependencyTrackingLevel.All : DependencyTrackingLevel.First); - - NodeFactoryOptimizationFlags nodeFactoryFlags = new NodeFactoryOptimizationFlags(); - nodeFactoryFlags.OptimizeAsyncMethods = Get(_command.AsyncMethodOptimization); - nodeFactoryFlags.TypeValidation = Get(_command.TypeValidation); - nodeFactoryFlags.DeterminismStress = Get(_command.DeterminismStress); - nodeFactoryFlags.PrintReproArgs = Get(_command.PrintReproInstructions); - nodeFactoryFlags.EnableCachedInterfaceDispatchSupport = Get(_command.EnableCachedInterfaceDispatchSupport) ?? !typeSystemContext.TargetAllowsRuntimeCodeGeneration; - nodeFactoryFlags.StripInliningInfo = Get(_command.StripInliningInfo); - nodeFactoryFlags.StripDebugInfo = Get(_command.StripDebugInfo); - nodeFactoryFlags.StripILBodies = Get(_command.StripILBodies); - - builder - .UseMapFile(Get(_command.Map)) - .UseMapCsvFile(Get(_command.MapCsv)) - .UsePdbFile(Get(_command.Pdb), Get(_command.PdbPath)) - .UsePerfMapFile(Get(_command.PerfMap), Get(_command.PerfMapPath), Get(_command.PerfMapFormatVersion)) - .UseProfileFile(jsonProfile != null) - .UseProfileData(profileDataManager) - .UseNodeFactoryOptimizationFlags(nodeFactoryFlags) - .FileLayoutAlgorithms(Get(_command.MethodLayout), Get(_command.FileLayout)) - .UseCompositeImageSettings(compositeImageSettings) - .UseJitPath(Get(_command.JitPath)) - .UseInstructionSetSupport(instructionSetSupport) - .UseCustomPESectionAlignment(Get(_command.CustomPESectionAlignment)) - .UseVerifyTypeAndFieldLayout(Get(_command.VerifyTypeAndFieldLayout)) - .UseHotColdSplitting(Get(_command.HotColdSplitting)) - .GenerateOutputFile(outFile) - .UseImageBase(_imageBase) - .UseContainerFormat(format) - .UseILProvider(ilProvider) - .UseBackendOptions(Get(_command.CodegenOptions)) - .UseLogger(logger) - .UseParallelism(Get(_command.Parallelism)) - .UseResilience(Get(_command.Resilient)) - .UseDependencyTracking(trackingLevel) - .UseCompilationRoots(compilationRoots) - .UseOptimizationMode(optimizationMode); - - builder.UseGenericCycleDetection( - depthCutoff: Get(_command.GenericCycleDepthCutoff), - breadthCutoff: Get(_command.GenericCycleBreadthCutoff)); - - builder.UsePrintReproInstructions(CreateReproArgumentString); - - compilation = builder.ToCompilation(); - + compositeImageSettings.PublicKey = compositeStrongNameKey.ToImmutableArray(); } - compilation.Compile(outFile); - if (dgmlLogFileName != null) - compilation.WriteDependencyLog(dgmlLogFileName); + if (rtrHeaderSymbolName != null) + { + compositeImageSettings.ReadyToRunHeaderSymbolName = rtrHeaderSymbolName; + } - compilation.Dispose(); + // + // Compile + // - if (((ReadyToRunCodegenCompilation)compilation).DeterminismCheckFailed) - throw new Exception("Determinism Check Failed"); + ReadyToRunCodegenCompilationBuilder builder = new ReadyToRunCodegenCompilationBuilder( + typeSystemContext, compilationGroup, _allInputFilePaths.Values, compositeRootPath); + string compilationUnitPrefix = ""; + builder.UseCompilationUnitPrefix(compilationUnitPrefix); + + ILProvider ilProvider = new ReadyToRunILProvider(compilationGroup); + + DependencyTrackingLevel trackingLevel = dgmlLogFileName == null ? + DependencyTrackingLevel.None : (Get(_command.GenerateFullDgmlLog) ? DependencyTrackingLevel.All : DependencyTrackingLevel.First); + + NodeFactoryOptimizationFlags nodeFactoryFlags = new NodeFactoryOptimizationFlags(); + nodeFactoryFlags.OptimizeAsyncMethods = Get(_command.AsyncMethodOptimization); + nodeFactoryFlags.TypeValidation = Get(_command.TypeValidation); + nodeFactoryFlags.DeterminismStress = Get(_command.DeterminismStress); + nodeFactoryFlags.PrintReproArgs = Get(_command.PrintReproInstructions); + nodeFactoryFlags.EnableCachedInterfaceDispatchSupport = Get(_command.EnableCachedInterfaceDispatchSupport) ?? !typeSystemContext.TargetAllowsRuntimeCodeGeneration; + nodeFactoryFlags.StripInliningInfo = Get(_command.StripInliningInfo); + nodeFactoryFlags.StripDebugInfo = Get(_command.StripDebugInfo); + nodeFactoryFlags.StripILBodies = Get(_command.StripILBodies); + + builder + .UseMapFile(Get(_command.Map)) + .UseMapCsvFile(Get(_command.MapCsv)) + .UsePdbFile(Get(_command.Pdb), Get(_command.PdbPath)) + .UsePerfMapFile(Get(_command.PerfMap), Get(_command.PerfMapPath), Get(_command.PerfMapFormatVersion)) + .UseProfileFile(jsonProfile != null) + .UseProfileData(profileDataManager) + .UseNodeFactoryOptimizationFlags(nodeFactoryFlags) + .FileLayoutAlgorithms(Get(_command.MethodLayout), Get(_command.FileLayout)) + .UseCompositeImageSettings(compositeImageSettings) + .UseJitPath(Get(_command.JitPath)) + .UseInstructionSetSupport(instructionSetSupport) + .UseCustomPESectionAlignment(Get(_command.CustomPESectionAlignment)) + .UseVerifyTypeAndFieldLayout(Get(_command.VerifyTypeAndFieldLayout)) + .UseHotColdSplitting(Get(_command.HotColdSplitting)) + .GenerateOutputFile(outFile) + .UseImageBase(_imageBase) + .UseContainerFormat(format) + .UseILProvider(ilProvider) + .UseBackendOptions(Get(_command.CodegenOptions)) + .UseLogger(logger) + .UseParallelism(Get(_command.Parallelism)) + .UseResilience(Get(_command.Resilient)) + .UseDependencyTracking(trackingLevel) + .UseCompilationRoots(compilationRoots) + .UseOptimizationMode(optimizationMode); + + builder.UseGenericCycleDetection( + depthCutoff: Get(_command.GenericCycleDepthCutoff), + breadthCutoff: Get(_command.GenericCycleBreadthCutoff)); + + builder.UsePrintReproInstructions(CreateReproArgumentString); + + return builder.ToCompilation(); } + } private static bool GetTargetAllowsRuntimeCodeGeneration(TargetOS operatingSystem, TargetArchitecture architecture)