From 3733041e39f9bddafb0da306cf1388810dc77df2 Mon Sep 17 00:00:00 2001 From: Serge Mera Date: Wed, 17 Jul 2019 10:23:37 -0700 Subject: [PATCH] Enable the MSBuildResolver to work under dotnet core (#561) Enable the MSBuildResolver to work under dotnet core --- .../Sdk/Public/Managed/Frameworks/helpers.dsc | 5 +- Public/Sdk/Public/Managed/Shared/types.dsc | 4 +- .../Managed/Testing/XUnit/xunitframework.dsc | 2 + Public/Sdk/Public/Managed/helpers.dsc | 28 +++-- Public/Sdk/Public/Managed/managedSdk.dsc | 2 +- .../Sdk/Public/Managed/runtimeConfigFiles.dsc | 7 +- .../Prelude.Configuration.Resolvers.dsc | 18 ++- .../DotNet-Runtime.linux-x64.dsc | 11 ++ .../DotNet-Runtime.osx-x64.dsc | 11 ++ .../DotNet-Runtime.win-x64.dsc | 11 ++ .../Dotnet-Runtime-External/common.dsc | 22 ++++ .../Dotnet-Runtime-External/module.config.dsc | 30 +++++ .../SelfHost/Libraries/MSBuild/msbuild.dsc | 37 ++++-- Public/Sdk/SelfHost/MacOS/coreRT.dsc | 2 +- Public/Src/Deployment/buildXL.dsc | 9 +- .../MSBuildGraphBuilderArguments.cs | 12 +- .../ProjectGraphWithPredictionsResult.cs | 34 +++-- .../Src/FrontEnd/MsBuild/MsBuildFrontEnd.cs | 6 +- .../Src/FrontEnd/MsBuild/MsBuildResolver.cs | 3 +- .../MsBuild/MsBuildWorkspaceResolver.cs | 117 +++++++++++++++--- Public/Src/FrontEnd/MsBuild/PipConstructor.cs | 26 ++-- .../FrontEnd/MsBuild/PipGraphConstructor.cs | 8 +- .../FrontEnd/MsBuild/ProjectGraphResult.cs | 12 +- Public/Src/FrontEnd/MsBuild/Tracing/Log.cs | 4 +- .../Libs/Prelude.Configuration.Resolvers.ts | 15 +++ .../MsBuildDotNetRuntimeTests.cs | 72 +++++++++++ .../ExecutionTests/MsBuildIntegrationTests.cs | 4 +- .../MsBuildPipExecutionTestBase.cs | 63 ++++++++-- .../MsBuildPipSchedulingTestBase.cs | 16 ++- .../MsBuild/Test.BuildXL.FrontEnd.MsBuild.dsc | 23 ++-- .../IMsBuildAssemblyLoader.cs | 7 +- .../MsBuildAssemblyLoader.cs | 81 ++++++------ .../MsBuildGraphBuilder.cs | 2 +- .../Tool.MsBuildGraphBuilder.dsc | 31 ++++- .../MsBuildAssemblyLoaderTests.cs | 9 +- .../MsBuildGraphConstructionTests.cs | 21 ++-- .../MsBuildGraphDecorationTests.cs | 9 +- .../MsBuildGraphProgressTests.cs | 9 +- .../MsBuildGraphProjectPredictionTests.cs | 10 +- .../MsBuildPredictionCollectorTests.cs | 14 +-- .../Test.Tool.MsBuildGraphBuilder.dsc | 15 +-- .../Utilities/GraphBuilderToolTestBase.cs | 47 +++++++ .../Resolvers/IMsBuildResolverSettings.cs | 29 +++++ .../Mutable/MsBuildResolverSettings.cs | 8 ++ config.dsc | 37 ++++-- 45 files changed, 729 insertions(+), 214 deletions(-) create mode 100644 Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.linux-x64.dsc create mode 100644 Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.osx-x64.dsc create mode 100644 Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.win-x64.dsc create mode 100644 Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/common.dsc create mode 100644 Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/module.config.dsc create mode 100644 Public/Src/FrontEnd/UnitTests/MsBuild/ExecutionTests/MsBuildDotNetRuntimeTests.cs create mode 100644 Public/Src/Tools/UnitTests/MsBuildGraphBuilder/Utilities/GraphBuilderToolTestBase.cs diff --git a/Public/Sdk/Public/Managed/Frameworks/helpers.dsc b/Public/Sdk/Public/Managed/Frameworks/helpers.dsc index 94d9c585f7..8a21a7e817 100644 --- a/Public/Sdk/Public/Managed/Frameworks/helpers.dsc +++ b/Public/Sdk/Public/Managed/Frameworks/helpers.dsc @@ -4,7 +4,8 @@ namespace Helpers { export declare const qualifier : {}; - function getToolTemplate() : Transformer.ExecuteArgumentsComposible { + @@public + export function getDotNetToolTemplate() : Transformer.ExecuteArgumentsComposible { const host = Context.getCurrentHost(); Contract.assert(host.cpuArchitecture === "x64", "The current DotNetCore Runtime package only has x64 version of Node. Ensure this runs on a 64-bit OS -or- update PowerShell.Core package to have other architectures embedded and fix this logic"); @@ -40,7 +41,7 @@ namespace Helpers }; } - const toolTemplate = getToolTemplate(); + const toolTemplate = getDotNetToolTemplate(); @@public export function wrapInDotNetExeForCurrentOs(args: Transformer.ExecuteArguments) : Transformer.ExecuteArguments { diff --git a/Public/Sdk/Public/Managed/Shared/types.dsc b/Public/Sdk/Public/Managed/Shared/types.dsc index 7eecaa6730..03405caa3f 100644 --- a/Public/Sdk/Public/Managed/Shared/types.dsc +++ b/Public/Sdk/Public/Managed/Shared/types.dsc @@ -129,12 +129,12 @@ export interface LinkResource { } @@public -export function isBinary(item: Reference) : item is Binary { +export function isBinary(item: any) : item is Binary { return item["binary"] !== undefined; } @@public -export function isAssembly(item: Reference) : item is Assembly { +export function isAssembly(item: any) : item is Assembly { return item["name"] !== undefined && (item["compile"] !== undefined || item["runtime"] !== undefined) && item["contents"] === undefined; // Exclude nuget packages diff --git a/Public/Sdk/Public/Managed/Testing/XUnit/xunitframework.dsc b/Public/Sdk/Public/Managed/Testing/XUnit/xunitframework.dsc index 9f01c1b87c..b322e6edb5 100644 --- a/Public/Sdk/Public/Managed/Testing/XUnit/xunitframework.dsc +++ b/Public/Sdk/Public/Managed/Testing/XUnit/xunitframework.dsc @@ -47,6 +47,7 @@ const xunitNetStandardRuntimeConfigFiles: File[] = Managed.RuntimeConfigFiles.cr "xunit.console", Managed.Factory.createBinary(xunitNetCoreConsolePackage, r`/lib/netcoreapp2.0/xunit.console.dll`), xunitReferences, + undefined, // runtimeContentToSkip undefined, // appconfig true); @@ -62,6 +63,7 @@ function additionalRuntimeContent(args: Managed.TestArguments) : Deployment.Depl "xunit.console", Managed.Factory.createBinary(xunitNetCoreConsolePackage, r`/lib/netcoreapp2.0/xunit.console.dll`), xunitReferences, + args.runtimeContentToSkip, undefined, // appConfig true)), xunitConsolePackage.getFile(r`/tools/netcoreapp2.0/xunit.runner.utility.netcoreapp10.dll`), diff --git a/Public/Sdk/Public/Managed/helpers.dsc b/Public/Sdk/Public/Managed/helpers.dsc index bb4ba47d5a..f203720173 100644 --- a/Public/Sdk/Public/Managed/helpers.dsc +++ b/Public/Sdk/Public/Managed/helpers.dsc @@ -3,6 +3,7 @@ import * as Shared from "Sdk.Managed.Shared"; import * as Csc from "Sdk.Managed.Tools.Csc"; +import * as Deployment from "Sdk.Deployment"; namespace Helpers { @@ -53,11 +54,13 @@ namespace Helpers { * You can control whether to follow the compile or the runtime axis using the optional compile argument. */ @@public - export function computeTransitiveReferenceClosure(framework: Shared.Framework, references: Shared.Reference[], compile?: boolean) : Shared.Binary[] { + export function computeTransitiveReferenceClosure(framework: Shared.Framework, references: Shared.Reference[], runtimeContentToSkip: Deployment.DeployableItem[], compile?: boolean) : Shared.Binary[] { return computeTransitiveClosure([ ...references, ...framework.standardReferences - ], compile); + ], + runtimeContentToSkip, + compile); } /** @@ -65,7 +68,7 @@ namespace Helpers { * You can control whether to follow the compile or the runtime axis using the optional compile argument. */ @@public - export function computeTransitiveClosure(references: Shared.Reference[], compile?: boolean) : Shared.Binary[] { + export function computeTransitiveClosure(references: Shared.Reference[], referencesToSkip: Deployment.DeployableItem[], compile?: boolean) : Shared.Binary[] { let results = MutableSet.empty(); let visitedReferences = MutableSet.empty(); @@ -75,14 +78,15 @@ namespace Helpers { for (let ref of allReferences) { - computeTransitiveReferenceClosureHelper(ref, results, visitedReferences, compile); + const referencesToSkipSet = Set.empty().add(...(referencesToSkip ? referencesToSkip : [])); + computeTransitiveReferenceClosureHelper(ref, referencesToSkipSet, results, visitedReferences, compile); } return results.toArray(); } - function computeTransitiveReferenceClosureHelper(ref: Shared.Reference, results: MutableSet, visitedReferences: MutableSet, compile?: boolean) { - if (visitedReferences.contains(ref)) + function computeTransitiveReferenceClosureHelper(ref: Shared.Reference, referencesToSkip: Set,results: MutableSet, visitedReferences: MutableSet, compile?: boolean) { + if (visitedReferences.contains(ref) || referencesToSkip.contains(ref)) { return; } @@ -95,17 +99,17 @@ namespace Helpers { } else if (Shared.isAssembly(ref)) { - if (compile && ref.compile) { + if (compile && ref.compile && !referencesToSkip.contains(ref.compile)) { results.add(ref.compile); } - if (!compile && ref.runtime) { + if (!compile && ref.runtime && !referencesToSkip.contains(ref.runtime)) { results.add(ref.runtime); } if (ref.references) { for (let nestedRef of ref.references) { - computeTransitiveReferenceClosureHelper(nestedRef, results, visitedReferences, compile); + computeTransitiveReferenceClosureHelper(nestedRef, referencesToSkip, results, visitedReferences, compile); } } } @@ -113,18 +117,18 @@ namespace Helpers { { if (compile) { - results.add(...ref.compile); + results.add(...ref.compile.filter(c => !referencesToSkip.contains(c))); } else { - results.add(...ref.runtime); + results.add(...ref.runtime.filter(c => !referencesToSkip.contains(c))); } for (let dependency of ref.dependencies) { if (Shared.isManagedPackage(dependency)) { - computeTransitiveReferenceClosureHelper(dependency, results, visitedReferences, compile); + computeTransitiveReferenceClosureHelper(dependency, referencesToSkip, results, visitedReferences, compile); } } diff --git a/Public/Sdk/Public/Managed/managedSdk.dsc b/Public/Sdk/Public/Managed/managedSdk.dsc index 7c6e014842..7d772b861a 100644 --- a/Public/Sdk/Public/Managed/managedSdk.dsc +++ b/Public/Sdk/Public/Managed/managedSdk.dsc @@ -209,7 +209,7 @@ export function assembly(args: Arguments, targetType: Csc.TargetType) : Result { if (targetType === "exe") { - runtimeConfigFiles = RuntimeConfigFiles.createFiles(framework, name, runtimeBinary, references, appConfig); + runtimeConfigFiles = RuntimeConfigFiles.createFiles(framework, name, runtimeBinary, references, args.runtimeContentToSkip, appConfig); if (framework.applicationDeploymentStyle === "selfContained") { const frameworkRuntimeFiles = framework.runtimeContentProvider(qualifier.targetRuntime); diff --git a/Public/Sdk/Public/Managed/runtimeConfigFiles.dsc b/Public/Sdk/Public/Managed/runtimeConfigFiles.dsc index 3baacc150b..6111a65ba6 100644 --- a/Public/Sdk/Public/Managed/runtimeConfigFiles.dsc +++ b/Public/Sdk/Public/Managed/runtimeConfigFiles.dsc @@ -5,6 +5,7 @@ import {Transformer} from "Sdk.Transformers"; import * as Json from "Sdk.Json"; import * as Shared from "Sdk.Managed.Shared"; +import * as Deployment from "Sdk.Deployment"; namespace RuntimeConfigFiles { @@ -18,6 +19,7 @@ namespace RuntimeConfigFiles { assemblyName: string, runtimeBinary: Shared.Binary, references: Shared.Reference[], + runtimeContentToSkip: Deployment.DeployableItem[], appConfig: File, testRunnerDeployment?: boolean ) : File[] { @@ -44,7 +46,7 @@ namespace RuntimeConfigFiles { } return [ - createDependenciesJson(framework, assemblyName, runtimeBinary, references, testRunnerDeployment), + createDependenciesJson(framework, assemblyName, runtimeBinary, references, runtimeContentToSkip, testRunnerDeployment), createRuntimeConfigJson(framework, assemblyName, runtimeConfigFolder, testRunnerDeployment), ]; case "none": @@ -68,12 +70,13 @@ namespace RuntimeConfigFiles { assemblyName: string, runtimeBinary: Shared.Binary, references: Shared.Reference[], + runtimeContentToSkip: Deployment.DeployableItem[], testRunnerDeployment?: boolean ): File { const specFileOutput = Context.getNewOutputDirectory("DotNetSpecFiles"); - const runtimeReferences = Helpers.computeTransitiveReferenceClosure(framework, references, false); + const runtimeReferences = Helpers.computeTransitiveReferenceClosure(framework, references, runtimeContentToSkip, false); const dependencySpecExtension = `${assemblyName}.deps.json`; const dependencySpecPath = p`${specFileOutput}/${dependencySpecExtension}`; diff --git a/Public/Sdk/Public/Prelude/Prelude.Configuration.Resolvers.dsc b/Public/Sdk/Public/Prelude/Prelude.Configuration.Resolvers.dsc index 562af7b593..3611c63f5e 100644 --- a/Public/Sdk/Public/Prelude/Prelude.Configuration.Resolvers.dsc +++ b/Public/Sdk/Public/Prelude/Prelude.Configuration.Resolvers.dsc @@ -136,12 +136,28 @@ interface MsBuildResolver extends ResolverBase, UntrackingSettings { runInContainer?: boolean; /** - * Collection of directories to search for the required MsBuild assemblies and MsBuild.exe (a.k.a. MSBuild toolset). + * Collection of directories to search for the required MsBuild assemblies and MsBuild.exe/MSBuild.dll (a.k.a. MSBuild toolset). * If not specified, locations in %PATH% are used. * Locations are traversed in specification order. */ msBuildSearchLocations?: Directory[]; + /** + * Whether to use the full framework or dotnet core version of MSBuild. Selected runtime is used both for build evaluation and execution. + * Default is full framework. + * Observe that using the full framework version means that msbuild.exe is expected to be found in msbuildSearchLocations + * (or PATH if not specified). If using the dotnet core version, the same logic applies but to msbuild.dll + */ + msBuildRuntime?: "FullFramework" | "DotNetCore"; + + /** + * Collection of directories to search for dotnet.exe, when DotNetCore is specified as the msBuildRuntime. If not + * specified, locations in %PATH% are used. + * Locations are traversed in specification order. + * It has no effect if the specified MSBuild runtime is full framework. + */ + dotNetSearchLocations?: Directory[]; + /** * Optional file paths for the projects or solutions that should be used to start parsing. These are relative * paths with respect to the root traversal. diff --git a/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.linux-x64.dsc b/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.linux-x64.dsc new file mode 100644 index 0000000000..0c2fd088ac --- /dev/null +++ b/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.linux-x64.dsc @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import {Transformer} from "Sdk.Transformers"; +import {createPublicDotNetRuntime} from "DotNet-Runtime.Common"; + +const v3 = importFrom("DotNet-Runtime.linux-x64.3.0.0-preview5").extracted; +const v2 = importFrom("DotNet-Runtime.linux-x64.2.2.2").extracted; + +@@public +export const extracted = createPublicDotNetRuntime(v3, v2); \ No newline at end of file diff --git a/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.osx-x64.dsc b/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.osx-x64.dsc new file mode 100644 index 0000000000..03be93be3b --- /dev/null +++ b/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.osx-x64.dsc @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import {Transformer} from "Sdk.Transformers"; +import {createPublicDotNetRuntime} from "DotNet-Runtime.Common"; + +const v3 = importFrom("DotNet-Runtime.osx-x64.3.0.0-preview5").extracted; +const v2 = importFrom("DotNet-Runtime.osx-x64.2.2.2").extracted; + +@@public +export const extracted = createPublicDotNetRuntime(v3, v2); \ No newline at end of file diff --git a/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.win-x64.dsc b/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.win-x64.dsc new file mode 100644 index 0000000000..01c664ca92 --- /dev/null +++ b/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/DotNet-Runtime.win-x64.dsc @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import {Transformer} from "Sdk.Transformers"; +import {createPublicDotNetRuntime} from "DotNet-Runtime.Common"; + +const v3 = importFrom("DotNet-Runtime.win-x64.3.0.0-preview5").extracted; +const v2 = importFrom("DotNet-Runtime.win-x64.2.2.2").extracted; + +@@public +export const extracted = createPublicDotNetRuntime(v3, v2); \ No newline at end of file diff --git a/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/common.dsc b/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/common.dsc new file mode 100644 index 0000000000..e64c8fc4d0 --- /dev/null +++ b/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/common.dsc @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import {Transformer} from "Sdk.Transformers"; + +@@public +export function createPublicDotNetRuntime(v3Runtime : StaticDirectory, v2Runtime: StaticDirectory) : StaticDirectory { + const netCoreAppV2 = v2Runtime.contents.filter(file => file.path.isWithin(d`${v2Runtime.root}/shared`)); + + // We create a package that has dotnet executable and related SDK, plus the V2 SDK + const dotNetRuntimeRoot = Context.getNewOutputDirectory("DotNet-Runtime"); + + const sealDirectory = Transformer.sealDirectory( + dotNetRuntimeRoot, + [ + ...v3Runtime.contents.map(file => Transformer.copyFile(file, file.path.relocate(v3Runtime.root, dotNetRuntimeRoot))), + ...netCoreAppV2.map(file => Transformer.copyFile(file, file.path.relocate(v2Runtime.root, dotNetRuntimeRoot))) + ] + ); + + return sealDirectory; +} diff --git a/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/module.config.dsc b/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/module.config.dsc new file mode 100644 index 0000000000..530d32005c --- /dev/null +++ b/Public/Sdk/SelfHost/Libraries/Dotnet-Runtime-External/module.config.dsc @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// These are the external versions of the architecture-specific DotNet-Runtime packages. +// They are created from the 3.0.0-preview5 version, plus the 2.2 NetCore.App SDK, which is deployed side-by-side +// The latter is required for MSBuild tests + +module({ + name: "DotNet-Runtime.win-x64", + nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences, + projects: [f`DotNet-Runtime.win-x64.dsc`] +}); + +module({ + name: "DotNet-Runtime.osx-x64", + nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences, + projects: [f`DotNet-Runtime.osx-x64.dsc`] +}); + +module({ + name: "DotNet-Runtime.linux-x64", + nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences, + projects: [f`DotNet-Runtime.linux-x64.dsc`] +}); + +module({ + name: "DotNet-Runtime.Common", + nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences, + projects: [f`common.dsc`] +}); diff --git a/Public/Sdk/SelfHost/Libraries/MSBuild/msbuild.dsc b/Public/Sdk/SelfHost/Libraries/MSBuild/msbuild.dsc index 6bc4c8a5b4..5ed6b9f616 100644 --- a/Public/Sdk/SelfHost/Libraries/MSBuild/msbuild.dsc +++ b/Public/Sdk/SelfHost/Libraries/MSBuild/msbuild.dsc @@ -3,16 +3,8 @@ import * as Managed from "Sdk.Managed"; import * as BuildXLSdk from "Sdk.BuildXL"; -// import * as Deployment from "Sdk.Deployment"; -@@public -export interface MSBuildQualifier extends Qualifier { - configuration: "debug" | "release"; - targetFramework: "net472" ; - targetRuntime: "win-x64" | "osx-x64"; -}; - -export declare const qualifier : MSBuildQualifier; +export declare const qualifier: BuildXLSdk.DefaultQualifier; @@public export const msbuildReferences: Managed.ManagedNugetPackage[] = [ @@ -22,12 +14,16 @@ export const msbuildReferences: Managed.ManagedNugetPackage[] = [ importFrom("Microsoft.Build.Tasks.Core").pkg, ]; +/** Runtime content for tests */ @@public export const msbuildRuntimeContent = [ + BuildXLSdk.isDotNetCoreBuild ? importFrom("System.Threading.Tasks.Dataflow").pkg : importFrom("DataflowForMSBuild").pkg, importFrom("System.Numerics.Vectors").pkg, - importFrom("DataflowForMSBuildRuntime").pkg, - // importFrom("System.Collections.Immutable").pkg, - ...BuildXLSdk.isTargetRuntimeOsx ? [ + importFrom("Microsoft.Build.Runtime").pkg, + ...BuildXLSdk.isDotNetCoreBuild ? [ + importFrom("Microsoft.NETCore.App.210").pkg, + importFrom("System.Text.Encoding.CodePages").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("Microsoft.Build.Tasks.Core").withQualifier({targetFramework: "netstandard2.0"}).pkg, importFrom("Microsoft.Build.Runtime").Contents.all.getFile(r`contentFiles/any/netcoreapp2.1/MSBuild.dll`), importFrom("Microsoft.Build.Runtime").Contents.all.getFile(r`contentFiles/any/netcoreapp2.1/MSBuild.runtimeconfig.json`), ] @@ -36,3 +32,20 @@ export const msbuildRuntimeContent = [ importFrom("Microsoft.Build.Runtime").Contents.all.getFile(r`contentFiles/any/net472/MSBuild.exe.config`), ], ]; + +function getFrameworkFolder() { + return BuildXLSdk.isDotNetCoreBuild ? "dotnetcore" : qualifier.targetFramework; +} + +@@public +export const deployment = [ + { + subfolder: a`msbuild`, + contents: [{ + subfolder: getFrameworkFolder(), + contents: [ + ...msbuildRuntimeContent, + ...msbuildReferences,] + }] + }, +]; \ No newline at end of file diff --git a/Public/Sdk/SelfHost/MacOS/coreRT.dsc b/Public/Sdk/SelfHost/MacOS/coreRT.dsc index ce40a738e2..68f2e06c0c 100644 --- a/Public/Sdk/SelfHost/MacOS/coreRT.dsc +++ b/Public/Sdk/SelfHost/MacOS/coreRT.dsc @@ -17,7 +17,7 @@ export namespace CoreRT { @@public export function compileToNative(asm: Shared.Assembly): NativeExecutableResult { /** Compile to native object file */ - const referencesClosure = Managed.Helpers.computeTransitiveClosure(asm.references, /*compile*/ false); + const referencesClosure = Managed.Helpers.computeTransitiveClosure(asm.references, asm.runtimeContentToSkip, /*compile*/ false); const ilcResult = Ilc.compile({ out: `${asm.name}.o`, inputs: [ diff --git a/Public/Src/Deployment/buildXL.dsc b/Public/Src/Deployment/buildXL.dsc index 54009f6776..8219cb1e13 100644 --- a/Public/Src/Deployment/buildXL.dsc +++ b/Public/Src/Deployment/buildXL.dsc @@ -45,14 +45,7 @@ namespace BuildXL { ).exe ] } ] ), - { - subfolder: r`MsBuildGraphBuilder`, - contents: BuildXLSdk.isDotNetCoreBuild ? [] : [ - // If the current qualifier is full framework, this tool has to be built with 472 - importFrom("BuildXL.Tools").MsBuildGraphBuilder.withQualifier( - Object.merge<(typeof qualifier) & {targetFramework: "net472"}>(qualifier, {targetFramework: "net472"})).exe - ] - }, + importFrom("BuildXL.Tools").MsBuildGraphBuilder.deployment, { subfolder: r`bvfs`, contents: qualifier.targetRuntime !== "win-x64" ? [] : [ diff --git a/Public/Src/FrontEnd/MsBuild.Serialization/MSBuildGraphBuilderArguments.cs b/Public/Src/FrontEnd/MsBuild.Serialization/MSBuildGraphBuilderArguments.cs index 3ade79353e..7dde3756d4 100644 --- a/Public/Src/FrontEnd/MsBuild.Serialization/MSBuildGraphBuilderArguments.cs +++ b/Public/Src/FrontEnd/MsBuild.Serialization/MSBuildGraphBuilderArguments.cs @@ -59,6 +59,11 @@ public sealed class MSBuildGraphBuilderArguments /// public bool AllowProjectsWithoutTargetProtocol { get; } + /// + /// Whether the MSBuild runtime is DotNet core (as opposed to full framework) + /// + public bool MsBuildRuntimeIsDotNetCore { get; } + /// public MSBuildGraphBuilderArguments( IReadOnlyCollection projectsToParse, @@ -67,7 +72,8 @@ public sealed class MSBuildGraphBuilderArguments IReadOnlyCollection mSBuildSearchLocations, IReadOnlyCollection entryPointTargets, IReadOnlyCollection requestedQualifiers, - bool allowProjectsWithoutTargetProtocol) + bool allowProjectsWithoutTargetProtocol, + bool msBuildRuntimeIsDotNetCore) { Contract.Requires(projectsToParse?.Count > 0); Contract.Requires(!string.IsNullOrEmpty(outputPath)); @@ -83,6 +89,7 @@ public sealed class MSBuildGraphBuilderArguments EntryPointTargets = entryPointTargets; RequestedQualifiers = requestedQualifiers; AllowProjectsWithoutTargetProtocol = allowProjectsWithoutTargetProtocol; + MsBuildRuntimeIsDotNetCore = msBuildRuntimeIsDotNetCore; } /// @@ -94,7 +101,8 @@ public override string ToString() Global properties: {string.Join(" ", GlobalProperties.Select(kvp => $"[{kvp.Key}]={kvp.Value}"))} Search locations: {string.Join(" ", MSBuildSearchLocations)} Requested qualifiers: {string.Join(" ", RequestedQualifiers.Select(qualifier => string.Join(";", qualifier.Select(kvp => $"[{kvp.Key}]={kvp.Value}"))))} -Allow projects without target protocol: {AllowProjectsWithoutTargetProtocol}"; +Allow projects without target protocol: {AllowProjectsWithoutTargetProtocol} +MSBuild runtime is DotNetCore: {MsBuildRuntimeIsDotNetCore}"; } } } diff --git a/Public/Src/FrontEnd/MsBuild.Serialization/ProjectGraphWithPredictionsResult.cs b/Public/Src/FrontEnd/MsBuild.Serialization/ProjectGraphWithPredictionsResult.cs index 8537e0d167..685a640c9a 100644 --- a/Public/Src/FrontEnd/MsBuild.Serialization/ProjectGraphWithPredictionsResult.cs +++ b/Public/Src/FrontEnd/MsBuild.Serialization/ProjectGraphWithPredictionsResult.cs @@ -29,39 +29,57 @@ namespace BuildXL.FrontEnd.MsBuild.Serialization public IReadOnlyDictionary MsBuildAssemblyPaths { get; } /// - /// Path to MsBuild.exe that was found + /// Path to MsBuild that was found /// /// /// The path may not be valid if is false. + /// The last component of the path could be either MSBuild.exe or MSBuild.dll, depending on the selected runtime. /// - public TPathType PathToMsBuildExe { get; } + public TPathType PathToMsBuild { get; } + + /// + /// Path to dotnet.exe, if the dotnet core version of MSBuild was specified to run + /// + /// + /// This is not actually populated by the graph construction tool, but by bxl + /// + public TPathType PathToDotNetExe { get; } /// public bool Succeeded { get; } /// - public static ProjectGraphWithPredictionsResult CreateSuccessfulGraph(ProjectGraphWithPredictions projectGraphWithPredictions, IReadOnlyDictionary assemblyPathsToLoad, TPathType pathToMsBuildExe) + public static ProjectGraphWithPredictionsResult CreateSuccessfulGraph(ProjectGraphWithPredictions projectGraphWithPredictions, IReadOnlyDictionary assemblyPathsToLoad, TPathType pathToMsBuild) { Contract.Requires(projectGraphWithPredictions != null); Contract.Requires(assemblyPathsToLoad != null); - return new ProjectGraphWithPredictionsResult(projectGraphWithPredictions, failure: default, msBuildAssemblyPaths: assemblyPathsToLoad, pathToMsBuildExe: pathToMsBuildExe, succeeded: true); + return new ProjectGraphWithPredictionsResult(projectGraphWithPredictions, failure: default, msBuildAssemblyPaths: assemblyPathsToLoad, pathToMsBuild: pathToMsBuild, pathToDotNetExe: default(TPathType), succeeded: true); } /// - public static ProjectGraphWithPredictionsResult CreateFailure(GraphConstructionError failure, IReadOnlyDictionary assemblyPathsToLoad, TPathType pathToMsBuildExe) + public static ProjectGraphWithPredictionsResult CreateFailure(GraphConstructionError failure, IReadOnlyDictionary assemblyPathsToLoad, TPathType pathToMsBuild) { Contract.Requires(failure != null); Contract.Requires(assemblyPathsToLoad != null); - return new ProjectGraphWithPredictionsResult(default, failure, assemblyPathsToLoad, pathToMsBuildExe, succeeded: false); + return new ProjectGraphWithPredictionsResult(default, failure, assemblyPathsToLoad, pathToMsBuild, pathToDotNetExe: default(TPathType), succeeded: false); + } + + /// + /// Returns a new instance of this with a specific path to dotnet.exe + /// + public ProjectGraphWithPredictionsResult WithPathToDotNetExe(TPathType pathToDotNetExe) + { + return new ProjectGraphWithPredictionsResult(Result, Failure, MsBuildAssemblyPaths, PathToMsBuild, pathToDotNetExe, Succeeded); } [JsonConstructor] - private ProjectGraphWithPredictionsResult(ProjectGraphWithPredictions result, GraphConstructionError failure, IReadOnlyDictionary msBuildAssemblyPaths, TPathType pathToMsBuildExe, bool succeeded) + private ProjectGraphWithPredictionsResult(ProjectGraphWithPredictions result, GraphConstructionError failure, IReadOnlyDictionary msBuildAssemblyPaths, TPathType pathToMsBuild, TPathType pathToDotNetExe, bool succeeded) { Result = result; Failure = failure; Succeeded = succeeded; - PathToMsBuildExe = pathToMsBuildExe; + PathToMsBuild = pathToMsBuild; + PathToDotNetExe = pathToDotNetExe; MsBuildAssemblyPaths = msBuildAssemblyPaths; } } diff --git a/Public/Src/FrontEnd/MsBuild/MsBuildFrontEnd.cs b/Public/Src/FrontEnd/MsBuild/MsBuildFrontEnd.cs index d8460cef40..6083fa8d1f 100644 --- a/Public/Src/FrontEnd/MsBuild/MsBuildFrontEnd.cs +++ b/Public/Src/FrontEnd/MsBuild/MsBuildFrontEnd.cs @@ -64,11 +64,7 @@ internal bool TryValidateMsBuildAssemblyLocationsAreCoordinated(IEnumerable EvaluateAllFilesAsync(IReadOnlySet evaluationGo m_host, result.ModuleDefinition, m_msBuildResolverSettings, - result.MsBuildExeLocation, + result.MsBuildLocation, + result.DotNetExeLocation, m_frontEndName, m_msBuildWorkspaceResolver.UserDefinedEnvironment, m_msBuildWorkspaceResolver.UserDefinedPassthroughVariables); diff --git a/Public/Src/FrontEnd/MsBuild/MsBuildWorkspaceResolver.cs b/Public/Src/FrontEnd/MsBuild/MsBuildWorkspaceResolver.cs index 45e243825c..a66d0ac469 100644 --- a/Public/Src/FrontEnd/MsBuild/MsBuildWorkspaceResolver.cs +++ b/Public/Src/FrontEnd/MsBuild/MsBuildWorkspaceResolver.cs @@ -84,7 +84,10 @@ public class MsBuildWorkspaceResolver : IWorkspaceModuleResolver /// /// Keep in sync with the BuildXL deployment spec that places the tool /// - private RelativePath RelativePathToGraphConstructionTool => RelativePath.Create(m_context.StringTable, @"tools\MsBuildGraphBuilder\ProjectGraphBuilder.exe"); + private RelativePath RelativePathToGraphConstructionTool => + RelativePath.Create(m_context.StringTable, m_resolverSettings.ShouldRunDotNetCoreMSBuild() ? + @"tools\MsBuildGraphBuilder\dotnetcore\ProjectGraphBuilder.dll" : + @"tools\MsBuildGraphBuilder\net472\ProjectGraphBuilder.exe"); /// /// The result of computing the build graph @@ -268,7 +271,7 @@ private async Task> TryComputeBuildGraphIfNeededAsy if (m_projectGraph == null) { // Get the locations where the MsBuild assemblies should be searched - if (!TryRetrieveMsBuildSearchLocations(out IEnumerable searchLocations)) + if (!TryRetrieveMsBuildSearchLocations(out IEnumerable msBuildSearchLocations)) { // Errors should have been logged return new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable); @@ -280,9 +283,21 @@ private async Task> TryComputeBuildGraphIfNeededAsy return new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable); } + // If we should run the dotnet core version of MSBuild, let's retrieve the locations where dotnet.exe + // should be found + IEnumerable dotNetSearchLocations = null; + if (m_resolverSettings.ShouldRunDotNetCoreMSBuild()) + { + if (!TryRetrieveDotNetSearchLocations(out dotNetSearchLocations)) + { + // Errors should have been logged + return new MsBuildGraphConstructionFailure(m_resolverSettings, m_context.PathTable); + } + } + BuildParameters.IBuildParameters buildParameters = RetrieveBuildParameters(); - m_projectGraph = await TryComputeBuildGraphAsync(searchLocations, parsingEntryPoints, buildParameters); + m_projectGraph = await TryComputeBuildGraphAsync(msBuildSearchLocations, dotNetSearchLocations, parsingEntryPoints, buildParameters); } return m_projectGraph.Value; @@ -352,7 +367,7 @@ private bool TryRetrieveParsingEntryPoint(out IEnumerable parsingE } - private async Task> TryComputeBuildGraphAsync(IEnumerable searchLocations, IEnumerable parsingEntryPoints, BuildParameters.IBuildParameters buildParameters) + private async Task> TryComputeBuildGraphAsync(IEnumerable msBuildSearchLocations, IEnumerable dotnetSearchLocations, IEnumerable parsingEntryPoints, BuildParameters.IBuildParameters buildParameters) { // We create a unique output file on the obj folder associated with the current front end, and using a GUID as the file name AbsolutePath outputDirectory = m_host.GetFolderForFrontEnd(MsBuildFrontEnd.Name); @@ -363,7 +378,7 @@ private async Task> TryComputeBuildGraphAsync(IEnum // Make sure the directories are there FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable)); - Possible> maybeProjectGraphResult = await ComputeBuildGraphAsync(responseFile, parsingEntryPoints, outputFile, searchLocations, buildParameters); + Possible> maybeProjectGraphResult = await ComputeBuildGraphAsync(responseFile, parsingEntryPoints, outputFile, msBuildSearchLocations, dotnetSearchLocations, buildParameters); if (!maybeProjectGraphResult.Succeeded) { @@ -409,7 +424,7 @@ private async Task> TryComputeBuildGraphAsync(IEnum allowedModuleDependencies: null, // no module policies cyclicalFriendModules: null); // no whitelist of cycles - return new ProjectGraphResult(projectGraph, moduleDefinition, projectGraphResult.PathToMsBuildExe); + return new ProjectGraphResult(projectGraph, moduleDefinition, projectGraphResult.PathToMsBuild, projectGraphResult.PathToDotNetExe); } private void DeleteGraphBuilderRelatedFiles(AbsolutePath outputFile, AbsolutePath responseFile) @@ -451,7 +466,26 @@ private bool TryRetrieveMsBuildSearchLocations(out IEnumerable sea m_host.Engine, m_resolverSettings.MsBuildSearchLocations?.SelectList(directoryLocation => directoryLocation.Path), out searchLocations, - () => Tracing.Logger.Log.NoSearchLocationsSpecified(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable)), + () => Tracing.Logger.Log.NoSearchLocationsSpecified(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), "msBuildSearchLocations"), + paths => Tracing.Logger.Log.CannotParseBuildParameterPath(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), paths) + ); + } + + /// + /// Retrieves a list of search locations for dotnet.exe + /// + /// + /// First inspects the resolver configuration to check if these are defined explicitly. Otherwise, uses PATH environment variable. + /// + private bool TryRetrieveDotNetSearchLocations(out IEnumerable searchLocations) + { + return FrontEndUtilities.TryRetrieveExecutableSearchLocations( + MsBuildFrontEnd.Name, + m_context, + m_host.Engine, + m_resolverSettings.DotNetSearchLocations?.SelectList(directoryLocation => directoryLocation.Path), + out searchLocations, + () => Tracing.Logger.Log.NoSearchLocationsSpecified(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), "dotnetSearchLocations"), paths => Tracing.Logger.Log.CannotParseBuildParameterPath(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), paths) ); } @@ -460,10 +494,21 @@ private bool TryRetrieveMsBuildSearchLocations(out IEnumerable sea AbsolutePath responseFile, IEnumerable projectEntryPoints, AbsolutePath outputFile, - IEnumerable searchLocations, + IEnumerable msBuidSearchLocations, + IEnumerable dotnetSearchLocations, BuildParameters.IBuildParameters buildParameters) { - SandboxedProcessResult result = await RunMsBuildGraphBuilderAsync(responseFile, projectEntryPoints, outputFile, searchLocations, buildParameters); + AbsolutePath dotnetExeLocation = AbsolutePath.Invalid; + if (m_resolverSettings.ShouldRunDotNetCoreMSBuild()) + { + if (!TryFindDotNetExe(dotnetSearchLocations, out dotnetExeLocation, out string failure)) + { + return ProjectGraphWithPredictionsResult.CreateFailure( + GraphConstructionError.CreateFailureWithoutLocation(failure), + CollectionUtilities.EmptyDictionary(), AbsolutePath.Invalid); + } + } + SandboxedProcessResult result = await RunMsBuildGraphBuilderAsync(responseFile, projectEntryPoints, outputFile, msBuidSearchLocations, dotnetExeLocation, buildParameters); string standardError = result.StandardError.CreateReader().ReadToEndAsync().GetAwaiter().GetResult(); @@ -505,7 +550,7 @@ private bool TryRetrieveMsBuildSearchLocations(out IEnumerable sea var projectGraphWithPredictionsResult = serializer.Deserialize>(reader); // A successfully constructed graph should always have a valid path to MsBuild - Contract.Assert(!projectGraphWithPredictionsResult.Succeeded || projectGraphWithPredictionsResult.PathToMsBuildExe.IsValid); + Contract.Assert(!projectGraphWithPredictionsResult.Succeeded || projectGraphWithPredictionsResult.PathToMsBuild.IsValid); // A successfully constructed graph should always have at least one project node Contract.Assert(!projectGraphWithPredictionsResult.Succeeded || projectGraphWithPredictionsResult.Result.ProjectNodes.Length > 0); // A failed construction should always have a failure set @@ -515,10 +560,30 @@ private bool TryRetrieveMsBuildSearchLocations(out IEnumerable sea Tracing.Logger.Log.GraphConstructionToolCompleted( m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), string.Join(",\n", projectGraphWithPredictionsResult.MsBuildAssemblyPaths.Select(kvp => I($"[{kvp.Key}]:{kvp.Value.ToString(m_context.PathTable)}"))), - projectGraphWithPredictionsResult.PathToMsBuildExe.ToString(m_context.PathTable)); + projectGraphWithPredictionsResult.PathToMsBuild.ToString(m_context.PathTable)); + + return m_resolverSettings.ShouldRunDotNetCoreMSBuild() ? projectGraphWithPredictionsResult.WithPathToDotNetExe(dotnetExeLocation) : projectGraphWithPredictionsResult; + } + } + + private bool TryFindDotNetExe(IEnumerable dotnetSearchLocations, out AbsolutePath dotnetExeLocation, out string failure) + { + dotnetExeLocation = AbsolutePath.Invalid; + failure = string.Empty; - return projectGraphWithPredictionsResult; + foreach (AbsolutePath location in dotnetSearchLocations) + { + AbsolutePath dotnetExeCandidate = location.Combine(m_context.PathTable, "dotnet.exe"); + if (m_host.Engine.FileExists(dotnetExeCandidate)) + { + dotnetExeLocation = dotnetExeCandidate; + return true; + } } + + failure = $"Cannot find dotnet.exe. " + + $"This is required because the dotnet core version of MSBuild was specified to run. Searched locations: [{string.Join(", ", dotnetSearchLocations.Select(location => location.ToString(m_context.PathTable)))}]"; + return false; } private void TrackFilesAndEnvironment(ISet fileAccesses, AbsolutePath frontEndFolder) @@ -539,11 +604,13 @@ private void TrackFilesAndEnvironment(ISet fileAccesses, Abs AbsolutePath responseFile, IEnumerable projectEntryPoints, AbsolutePath outputFile, - IEnumerable searchLocations, + IEnumerable msBuildSearchLocations, + AbsolutePath dotnetExeLocation, BuildParameters.IBuildParameters buildParameters) { + Contract.Assert(!m_resolverSettings.ShouldRunDotNetCoreMSBuild() || dotnetExeLocation.IsValid); + AbsolutePath toolDirectory = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).GetParent(m_context.PathTable); - string pathToTool = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).ToString(m_context.PathTable); string outputDirectory = outputFile.GetParent(m_context.PathTable).ToString(m_context.PathTable); string outputFileString = outputFile.ToString(m_context.PathTable); IReadOnlyCollection entryPointTargets = m_resolverSettings.InitialTargets ?? CollectionUtilities.EmptyArray(); @@ -554,14 +621,30 @@ private void TrackFilesAndEnvironment(ISet fileAccesses, Abs projectEntryPoints.Select(entryPoint => entryPoint.ToString(m_context.PathTable)).ToList(), outputFileString, new GlobalProperties(m_resolverSettings.GlobalProperties ?? CollectionUtilities.EmptyDictionary()), - searchLocations.Select(location => location.ToString(m_context.PathTable)).ToList(), + msBuildSearchLocations.Select(location => location.ToString(m_context.PathTable)).ToList(), entryPointTargets, requestedQualifiers, - m_resolverSettings.AllowProjectsToNotSpecifyTargetProtocol == true); + m_resolverSettings.AllowProjectsToNotSpecifyTargetProtocol == true, + m_resolverSettings.ShouldRunDotNetCoreMSBuild()); var responseFilePath = responseFile.ToString(m_context.PathTable); SerializeResponseFile(responseFilePath, arguments); + string graphConstructionToolPath = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePathToGraphConstructionTool).ToString(m_context.PathTable); + string pathToTool; + string toolArguments; + // if we should call the dotnet core version of MSBuild, we need to actually call dotnet.exe and pass the tool itself as its first argument + if (m_resolverSettings.ShouldRunDotNetCoreMSBuild()) + { + pathToTool = dotnetExeLocation.ToString(m_context.PathTable); + toolArguments = I($"\"{graphConstructionToolPath}\" \"{responseFilePath}\""); + } + else + { + pathToTool = graphConstructionToolPath; + toolArguments = I($"\"{responseFilePath}\""); + } + Tracing.Logger.Log.LaunchingGraphConstructionTool(m_context.LoggingContext, m_resolverSettings.Location(m_context.PathTable), arguments.ToString(), pathToTool); // Just being defensive, make sure there is not an old output file lingering around @@ -572,7 +655,7 @@ private void TrackFilesAndEnvironment(ISet fileAccesses, Abs pathToTool, buildStorageDirectory: outputDirectory, fileAccessManifest: GenerateFileAccessManifest(toolDirectory, outputFile), - arguments: I($"\"{responseFilePath}\""), + arguments: toolArguments, workingDirectory: outputDirectory, description: "MsBuild graph builder", buildParameters, diff --git a/Public/Src/FrontEnd/MsBuild/PipConstructor.cs b/Public/Src/FrontEnd/MsBuild/PipConstructor.cs index a3088f424b..691dda0f77 100644 --- a/Public/Src/FrontEnd/MsBuild/PipConstructor.cs +++ b/Public/Src/FrontEnd/MsBuild/PipConstructor.cs @@ -55,7 +55,8 @@ internal sealed class PipConstructor private AbsolutePath Root => m_resolverSettings.Root; - private readonly AbsolutePath m_msBuildExePath; + private readonly AbsolutePath m_msBuildPath; + private readonly AbsolutePath m_dotnetExePath; private readonly string m_frontEndName; private readonly IEnumerable> m_userDefinedEnvironment; private readonly IEnumerable m_userDefinedPassthroughVariables; @@ -81,7 +82,8 @@ internal sealed class PipConstructor FrontEndHost frontEndHost, ModuleDefinition moduleDefinition, IMsBuildResolverSettings resolverSettings, - AbsolutePath pathToMsBuildExe, + AbsolutePath pathToMsBuild, + AbsolutePath pathToDotnetExe, string frontEndName, IEnumerable> userDefinedEnvironment, IEnumerable userDefinedPassthroughVariables) @@ -90,7 +92,8 @@ internal sealed class PipConstructor Contract.Requires(frontEndHost != null); Contract.Requires(moduleDefinition != null); Contract.Requires(resolverSettings != null); - Contract.Requires(pathToMsBuildExe.IsValid); + Contract.Requires(pathToMsBuild.IsValid); + Contract.Requires(!resolverSettings.ShouldRunDotNetCoreMSBuild() || pathToDotnetExe.IsValid); Contract.Requires(!string.IsNullOrEmpty(frontEndName)); Contract.Requires(userDefinedEnvironment != null); Contract.Requires(userDefinedPassthroughVariables != null); @@ -99,7 +102,8 @@ internal sealed class PipConstructor m_frontEndHost = frontEndHost; m_moduleDefinition = moduleDefinition; m_resolverSettings = resolverSettings; - m_msBuildExePath = pathToMsBuildExe; + m_msBuildPath = pathToMsBuild; + m_dotnetExePath = pathToDotnetExe; m_frontEndName = frontEndName; m_userDefinedEnvironment = userDefinedEnvironment; m_userDefinedPassthroughVariables = userDefinedPassthroughVariables; @@ -767,7 +771,17 @@ private AbsolutePath GetLogDirectory(ProjectWithPredictions projectFile, Qualifi ProcessBuilder processBuilder, ProjectWithPredictions project) { - FileArtifact cmdExeArtifact = FileArtifact.CreateSourceFile(m_msBuildExePath); + // If we should use the dotnet core version of msbuild, the executable for the pip is dotnet.exe instead of msbuild.exe, and + // the first argument is msbuild.dll + FileArtifact cmdExeArtifact; + if (m_resolverSettings.ShouldRunDotNetCoreMSBuild()) + { + cmdExeArtifact = FileArtifact.CreateSourceFile(m_dotnetExePath); + processBuilder.ArgumentsBuilder.Add(PipDataAtom.FromAbsolutePath(m_msBuildPath)); + } + else { + cmdExeArtifact = FileArtifact.CreateSourceFile(m_msBuildPath); + } processBuilder.Executable = cmdExeArtifact; processBuilder.AddInputFile(cmdExeArtifact); @@ -780,8 +794,6 @@ private AbsolutePath GetLogDirectory(ProjectWithPredictions projectFile, Qualifi // ensure environment value (and hence pip hash) consistency. processBuilder.EnableTempDirectory(); - AbsolutePath toolDir = m_msBuildExePath.GetParent(PathTable); - processBuilder.ToolDescription = StringId.Create(m_context.StringTable, I($"{m_moduleDefinition.Descriptor.Name} - {project.FullPath.ToString(PathTable)}")); return true; diff --git a/Public/Src/FrontEnd/MsBuild/PipGraphConstructor.cs b/Public/Src/FrontEnd/MsBuild/PipGraphConstructor.cs index 1aa9d435ee..6d04476a0f 100644 --- a/Public/Src/FrontEnd/MsBuild/PipGraphConstructor.cs +++ b/Public/Src/FrontEnd/MsBuild/PipGraphConstructor.cs @@ -35,7 +35,8 @@ public sealed class PipGraphConstructor FrontEndHost frontEndHost, ModuleDefinition moduleDefinition, IMsBuildResolverSettings resolverSettings, - AbsolutePath pathToMsBuildExe, + AbsolutePath pathToMsBuild, + AbsolutePath pathToDotnetExe, string frontEndName, IEnumerable> userDefinedEnvironment, IEnumerable userDefinedPassthroughVariables) @@ -44,14 +45,15 @@ public sealed class PipGraphConstructor Contract.Requires(frontEndHost != null); Contract.Requires(moduleDefinition != null); Contract.Requires(resolverSettings != null); - Contract.Requires(pathToMsBuildExe.IsValid); + Contract.Requires(pathToMsBuild.IsValid); + Contract.Requires(!resolverSettings.ShouldRunDotNetCoreMSBuild() || pathToDotnetExe.IsValid); Contract.Requires(!string.IsNullOrEmpty(frontEndName)); Contract.Requires(userDefinedEnvironment != null); Contract.Requires(userDefinedPassthroughVariables != null); m_context = context; m_frontEndHost = frontEndHost; - m_pipConstructor = new PipConstructor(context, frontEndHost, moduleDefinition, resolverSettings, pathToMsBuildExe, frontEndName, userDefinedEnvironment, userDefinedPassthroughVariables); + m_pipConstructor = new PipConstructor(context, frontEndHost, moduleDefinition, resolverSettings, pathToMsBuild, pathToDotnetExe, frontEndName, userDefinedEnvironment, userDefinedPassthroughVariables); } /// diff --git a/Public/Src/FrontEnd/MsBuild/ProjectGraphResult.cs b/Public/Src/FrontEnd/MsBuild/ProjectGraphResult.cs index 89d8e8a2e2..1739af0962 100644 --- a/Public/Src/FrontEnd/MsBuild/ProjectGraphResult.cs +++ b/Public/Src/FrontEnd/MsBuild/ProjectGraphResult.cs @@ -21,18 +21,22 @@ namespace BuildXL.FrontEnd.MsBuild public ModuleDefinition ModuleDefinition { get; } /// - public AbsolutePath MsBuildExeLocation { get; } + public AbsolutePath MsBuildLocation { get; } /// - public ProjectGraphResult(ProjectGraphWithPredictions projectGraphWithPredictions, ModuleDefinition moduleDefinition, AbsolutePath msBuildExeLocation) + public AbsolutePath DotNetExeLocation { get; } + + /// + public ProjectGraphResult(ProjectGraphWithPredictions projectGraphWithPredictions, ModuleDefinition moduleDefinition, AbsolutePath msBuildLocation, AbsolutePath dotnetExeLocation) { Contract.Requires(projectGraphWithPredictions != null); Contract.Requires(moduleDefinition != null); - Contract.Requires(msBuildExeLocation.IsValid); + Contract.Requires(msBuildLocation.IsValid); ProjectGraph = projectGraphWithPredictions; ModuleDefinition = moduleDefinition; - MsBuildExeLocation = msBuildExeLocation; + MsBuildLocation = msBuildLocation; + DotNetExeLocation = dotnetExeLocation; } } } diff --git a/Public/Src/FrontEnd/MsBuild/Tracing/Log.cs b/Public/Src/FrontEnd/MsBuild/Tracing/Log.cs index 125cfe7b4c..564db4a4b8 100644 --- a/Public/Src/FrontEnd/MsBuild/Tracing/Log.cs +++ b/Public/Src/FrontEnd/MsBuild/Tracing/Log.cs @@ -69,9 +69,9 @@ internal Logger() EventGenerators = EventGenerators.LocalOnly, EventLevel = Level.Error, EventTask = (ushort)Tasks.Parser, - Message = EventConstants.LabeledProvenancePrefix + "Build parameter 'PATH' is not specified, and no explicit locations were defined in the resolver settings via 'MsBuildAssemblyLocations'.", + Message = EventConstants.LabeledProvenancePrefix + "Build parameter 'PATH' is not specified, and no explicit locations were defined in the resolver settings via '{specifiedVia}'.", Keywords = (int)(Keywords.UserMessage | Keywords.Diagnostics))] - public abstract void NoSearchLocationsSpecified(LoggingContext context, Location location); + public abstract void NoSearchLocationsSpecified(LoggingContext context, Location location, string specifiedVia); [GeneratedEvent( (ushort)LogEventId.CannotParseBuildParameterPath, diff --git a/Public/Src/FrontEnd/UnitTests/Libs/Prelude.Configuration.Resolvers.ts b/Public/Src/FrontEnd/UnitTests/Libs/Prelude.Configuration.Resolvers.ts index cdfa33df04..f4f3f9aebb 100644 --- a/Public/Src/FrontEnd/UnitTests/Libs/Prelude.Configuration.Resolvers.ts +++ b/Public/Src/FrontEnd/UnitTests/Libs/Prelude.Configuration.Resolvers.ts @@ -122,6 +122,21 @@ interface MsBuildResolver { */ msBuildSearchLocations?: Directory[]; + /** + * Whether to use the full framework or dotnet core version of MSBuild. Selected runtime is used both for build evaluation and execution. + * Default is full framework. + * Observe that using the full framework version means that msbuild.exe is expected to be found in msbuildSearchLocations + * (or PATH if not specified). If using the dotnet core version, the same logic applies but to msbuild.dll + */ + msBuildRuntime?: "FullFramework" | "DotNetCore"; + + /** + * Collection of directories to search for dotnet.exe, when DotNetCore is specified as the msBuildRuntime. If not + * specified, locations in %PATH% are used. + * Locations are traversed in specification order. + */ + dotNetSearchLocations?: Directory[]; + /** * Targets to execute on the entry point project. * If not provided, the default targets are used. diff --git a/Public/Src/FrontEnd/UnitTests/MsBuild/ExecutionTests/MsBuildDotNetRuntimeTests.cs b/Public/Src/FrontEnd/UnitTests/MsBuild/ExecutionTests/MsBuildDotNetRuntimeTests.cs new file mode 100644 index 0000000000..b016dd39e0 --- /dev/null +++ b/Public/Src/FrontEnd/UnitTests/MsBuild/ExecutionTests/MsBuildDotNetRuntimeTests.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BuildXL.Utilities.Configuration.Mutable; +using BuildXL.Engine.Tracing; +using BuildXL.Pips.Operations; +using BuildXL.Scheduler.Graph; +using Test.BuildXL.EngineTestUtilities; +using Xunit; +using Xunit.Abstractions; +using System; +using BuildXL.Utilities; +using BuildXL.Utilities.Configuration; + +namespace Test.BuildXL.FrontEnd.MsBuild +{ + /// + /// Uses an MSBuild resolver to schedule and execute pips based on MSBuild. + /// + /// + /// These tests actually execute pips, and are therefore expensive + /// + public class MsBuildDotNetRuntimeTests : MsBuildPipExecutionTestBase + { + public MsBuildDotNetRuntimeTests(ITestOutputHelper output) + : base(output) + { + } + + [Theory] + [InlineData("DotNetCore")] + [InlineData("FullFramework")] + public void RuntimeSelectionIsEffective(string msBuildRuntime) + { + const string TestProj1 = "test1.csproj"; + var pathToTestProj1 = R("public", "dir1", TestProj1); + + const string Dirs = "dirs.proj"; + var config = (CommandLineConfiguration)Build( + msBuildRuntime: msBuildRuntime, + dotnetSearchLocations: $"[d`{TestDeploymentDir}/{RelativePathToDotnetExe}`]") + .AddSpec(Dirs, CreateDirsProject(pathToTestProj1)) + .AddSpec(pathToTestProj1, CreateEmptyProject()) + .PersistSpecsAndGetConfiguration(); + + config.Sandbox.FileSystemMode = FileSystemMode.RealAndMinimalPipGraph; + + var engineResult = RunEngineWithConfig(config); + Assert.True(engineResult.IsSuccess); + + var pipGraph = engineResult.EngineState.PipGraph; + + var processPips = pipGraph.RetrievePipsOfType(PipType.Process).ToList(); + var testProj1 = (Process)processPips.Find(pip => pip.Provenance.OutputValueSymbol.ToString(engineResult.EngineState.SymbolTable).Contains(TestProj1)); + Assert.True(testProj1 != null); + + if (msBuildRuntime == "DotNetCore") + { + // The main executable has to be dotnet.exe (or dotnet in the mac/unix case) + Assert.Contains("DOTNET", testProj1.Executable.Path.ToString(PathTable).ToUpperInvariant()); + } + else + { + // The main executable has to be msbuild.exe + Assert.Contains("MSBUILD.EXE", testProj1.Executable.Path.ToString(PathTable).ToUpperInvariant()); + } + } + } +} diff --git a/Public/Src/FrontEnd/UnitTests/MsBuild/ExecutionTests/MsBuildIntegrationTests.cs b/Public/Src/FrontEnd/UnitTests/MsBuild/ExecutionTests/MsBuildIntegrationTests.cs index 70facdc91a..d960f73186 100644 --- a/Public/Src/FrontEnd/UnitTests/MsBuild/ExecutionTests/MsBuildIntegrationTests.cs +++ b/Public/Src/FrontEnd/UnitTests/MsBuild/ExecutionTests/MsBuildIntegrationTests.cs @@ -136,7 +136,9 @@ public void PassthroughVariablesAreHonored(bool isPassThrough) runInContainer: false, environment: environment, globalProperties: null, - filenameEntryPoint: pathToTestProj1) + filenameEntryPoint: pathToTestProj1, + msBuildRuntime: null, + dotnetSearchLocations: null) .AddSpec(pathToTestProj1, CreateWriteFileTestProject("MyFile")) .PersistSpecsAndGetConfiguration(); diff --git a/Public/Src/FrontEnd/UnitTests/MsBuild/Infrastructure/MsBuildPipExecutionTestBase.cs b/Public/Src/FrontEnd/UnitTests/MsBuild/Infrastructure/MsBuildPipExecutionTestBase.cs index fd9c22f420..856bc6a901 100644 --- a/Public/Src/FrontEnd/UnitTests/MsBuild/Infrastructure/MsBuildPipExecutionTestBase.cs +++ b/Public/Src/FrontEnd/UnitTests/MsBuild/Infrastructure/MsBuildPipExecutionTestBase.cs @@ -36,6 +36,16 @@ public abstract class MsBuildPipExecutionTestBase : DsTestWithCacheBase // By default the engine runs e2e protected virtual EnginePhases Phase => EnginePhases.Execute; + // Keep the paths below in sync with Public\Src\FrontEnd\UnitTests\MsBuild\Test.BuildXL.FrontEnd.MsBuild.dsc + /// + protected string RelativePathToFullframeworkMSBuild => "msbuild/net472"; + + /// + protected string RelativePathToDotnetCoreMSBuild => "msbuild/dotnetcore"; + + /// + protected string RelativePathToDotnetExe => "dotnet"; + protected MsBuildPipExecutionTestBase(ITestOutputHelper output) : base(output, true) { RegisterEventSource(global::BuildXL.Engine.ETWLogger.Log); @@ -59,19 +69,40 @@ protected SpecEvaluationBuilder BuildWithEnvironment(Dictionary environment = null, Dictionary globalProperties = null, string filenameEntryPoint = null) + protected SpecEvaluationBuilder Build( + bool runInContainer = false, + Dictionary environment = null, + Dictionary globalProperties = null, + string filenameEntryPoint = null, + string msBuildRuntime = null, + string dotnetSearchLocations = null) { return Build(runInContainer, environment != null? environment.ToDictionary(kvp => kvp.Key, kvp => new DiscriminatingUnion(kvp.Value)) : null, globalProperties, - filenameEntryPoint); + filenameEntryPoint, + msBuildRuntime, + dotnetSearchLocations); } /// - protected SpecEvaluationBuilder Build(bool runInContainer, Dictionary> environment, Dictionary globalProperties, string filenameEntryPoint) + protected SpecEvaluationBuilder Build( + bool runInContainer, + Dictionary> environment, + Dictionary globalProperties, + string filenameEntryPoint, + string msBuildRuntime, + string dotnetSearchLocations) { // Let's explicitly pass an empty environment, so the process environment won't affect tests by default - return base.Build().Configuration(DefaultMsBuildPrelude(runInContainer, environment: environment ?? new Dictionary>(), globalProperties, filenameEntryPoint: filenameEntryPoint)); + return base.Build().Configuration( + DefaultMsBuildPrelude( + runInContainer, + environment: environment ?? new Dictionary>(), + globalProperties, + filenameEntryPoint: filenameEntryPoint, + msBuildRuntime: msBuildRuntime, + dotnetSearchLocations: dotnetSearchLocations)); } /// @@ -111,6 +142,18 @@ protected BuildXLEngineResult RunEngineWithConfig(ICommandLineConfiguration conf } } + /// + /// Returns an empty project + /// + protected string CreateEmptyProject() + { + return +$@" + + +"; + } + /// /// Returns a project that just echoes 'Hello World' /// @@ -180,7 +223,7 @@ protected string CreateWriteFileTestProject(string outputFilename, string projec private string GetWriteFileTask() { return -$@" +$@" @@ -229,14 +272,16 @@ protected string CreateDirsProject(params string[] projectNames) bool enableEngineTracing = false, string logVerbosity = null, bool allowProjectsToNotSpecifyTargetProtocol = true, - string filenameEntryPoint = null) => $@" + string filenameEntryPoint = null, + string msBuildRuntime = null, + string dotnetSearchLocations = null) => $@" config({{ disableDefaultSourceResolver: true, resolvers: [ {{ kind: 'MsBuild', moduleName: 'Test', - msBuildSearchLocations: [d`{TestDeploymentDir}`], + msBuildSearchLocations: [d`{TestDeploymentDir}/{(msBuildRuntime == "DotNetCore" ? RelativePathToDotnetCoreMSBuild : RelativePathToFullframeworkMSBuild)}`], root: d`.`, allowProjectsToNotSpecifyTargetProtocol: {(allowProjectsToNotSpecifyTargetProtocol ? "true" : "false")}, runInContainer: {(runInContainer ? "true" : "false")}, @@ -246,6 +291,8 @@ protected string CreateDirsProject(params string[] projectNames) enableEngineTracing: {(enableEngineTracing? "true" : "false")}, {(logVerbosity != null ? $"logVerbosity: {logVerbosity}," : string.Empty)} {(filenameEntryPoint != null ? $"fileNameEntryPoints: [r`{filenameEntryPoint}`]," : string.Empty)} + {(msBuildRuntime != null ? $"msBuildRuntime: \"{msBuildRuntime}\"," : string.Empty)} + {(dotnetSearchLocations != null ? $"dotNetSearchLocations: {dotnetSearchLocations}," : string.Empty)} }}, ], }});"; @@ -258,7 +305,7 @@ protected string CreateDirsProject(params string[] projectNames) {{ kind: 'MsBuild', moduleName: 'Test', - msBuildSearchLocations: [d`{TestDeploymentDir}`], + msBuildSearchLocations: [d`{TestDeploymentDir}/{RelativePathToFullframeworkMSBuild}`], root: d`.`, allowProjectsToNotSpecifyTargetProtocol: true, {DictionaryToExpression("environment", new Dictionary())} diff --git a/Public/Src/FrontEnd/UnitTests/MsBuild/Infrastructure/MsBuildPipSchedulingTestBase.cs b/Public/Src/FrontEnd/UnitTests/MsBuild/Infrastructure/MsBuildPipSchedulingTestBase.cs index 6f9dffdbc6..4efb0280ff 100644 --- a/Public/Src/FrontEnd/UnitTests/MsBuild/Infrastructure/MsBuildPipSchedulingTestBase.cs +++ b/Public/Src/FrontEnd/UnitTests/MsBuild/Infrastructure/MsBuildPipSchedulingTestBase.cs @@ -40,6 +40,19 @@ public abstract class MsBuildPipSchedulingTestBase : DsTestWithCacheBase protected AbsolutePath TestPath { get; } + // Keep the paths below in sync with Public\Src\FrontEnd\UnitTests\MsBuild\Test.BuildXL.FrontEnd.MsBuild.dsc + private AbsolutePath FullframeworkMSBuild => AbsolutePath.Create(PathTable, TestDeploymentDir) + .Combine(PathTable, "msbuild") + .Combine(PathTable, "net472") + .Combine(PathTable, "MSBuild.exe"); + private AbsolutePath DotnetCoreMSBuild => AbsolutePath.Create(PathTable, TestDeploymentDir) + .Combine(PathTable, "msbuild") + .Combine(PathTable, "dotnetcore") + .Combine(PathTable, "MSBuild.dll"); + private AbsolutePath DotnetExe => AbsolutePath.Create(PathTable, TestDeploymentDir) + .Combine(PathTable, "dotnet") + .Combine(PathTable, OperatingSystemHelper.IsUnixOS ? "dotnet" : "dotnet.exe"); + /// public MsBuildPipSchedulingTestBase(ITestOutputHelper output, bool usePassThroughFileSystem = false) : base(output, usePassThroughFileSystem) { @@ -134,7 +147,8 @@ internal MsBuildSchedulingResult ScheduleAll(MsBuildResolverSettings resolverSet controller, m_testModule, resolverSettings, - AbsolutePath.Create(PathTable, TestDeploymentDir).Combine(PathTable, "MSBuild.exe"), + resolverSettings.ShouldRunDotNetCoreMSBuild()? DotnetCoreMSBuild : FullframeworkMSBuild, + resolverSettings.ShouldRunDotNetCoreMSBuild()? DotnetExe : AbsolutePath.Invalid, nameof(MsBuildFrontEnd), trackedEnv, passthroughVars); diff --git a/Public/Src/FrontEnd/UnitTests/MsBuild/Test.BuildXL.FrontEnd.MsBuild.dsc b/Public/Src/FrontEnd/UnitTests/MsBuild/Test.BuildXL.FrontEnd.MsBuild.dsc index fbf50cbe51..6e4cfff872 100644 --- a/Public/Src/FrontEnd/UnitTests/MsBuild/Test.BuildXL.FrontEnd.MsBuild.dsc +++ b/Public/Src/FrontEnd/UnitTests/MsBuild/Test.BuildXL.FrontEnd.MsBuild.dsc @@ -3,9 +3,10 @@ import * as Managed from "Sdk.Managed"; import * as MSBuild from "Sdk.Selfhost.MSBuild"; +import * as Frameworks from "Sdk.Managed.Frameworks"; namespace Test.MsBuild { - export declare const qualifier: MSBuild.MSBuildQualifier; + export declare const qualifier: BuildXLSdk.DefaultQualifier; @@public export const dll = BuildXLSdk.test({ @@ -38,21 +39,19 @@ namespace Test.MsBuild { importFrom("BuildXL.Utilities.Instrumentation").Common.dll, ...BuildXLSdk.tplPackages, ], + + // We need both the full framework and dotnet core versions of MSBuild, plus dotnet.exe for the dotnet core case runtimeContent: [ - ...MSBuild.msbuildRuntimeContent, - ...MSBuild.msbuildReferences, + ...importFrom("Sdk.Selfhost.MSBuild").withQualifier(Object.merge(qualifier, {targetFramework: "net472"})).deployment, + ...importFrom("Sdk.Selfhost.MSBuild").withQualifier(Object.merge(qualifier, {targetFramework: "netcoreapp3.0"})).deployment, + { + subfolder: "dotnet", + contents: Frameworks.Helpers.getDotNetToolTemplate().dependencies + }, { subfolder: a`tools`, - contents: [{ - subfolder: a`MsBuildGraphBuilder`, - contents: [ - importFrom("BuildXL.Tools").MsBuildGraphBuilder.exe, - ]} - ] + contents: [importFrom("BuildXL.Tools").MsBuildGraphBuilder.deployment] } ], - runtimeContentToSkip : [ - importFrom("System.Threading.Tasks.Dataflow").pkg - ] }); } diff --git a/Public/Src/Tools/Tool.MsBuildGraphBuilder/IMsBuildAssemblyLoader.cs b/Public/Src/Tools/Tool.MsBuildGraphBuilder/IMsBuildAssemblyLoader.cs index 774df5d61e..a57e530fe4 100644 --- a/Public/Src/Tools/Tool.MsBuildGraphBuilder/IMsBuildAssemblyLoader.cs +++ b/Public/Src/Tools/Tool.MsBuildGraphBuilder/IMsBuildAssemblyLoader.cs @@ -19,6 +19,11 @@ public interface IMsBuildAssemblyLoader /// A collection of full paths to directories, representing search locations on disk /// On failure, the reason for why the assemblies were not found /// A dictionary from assembly names to paths to the required assemblies that were found. May not be complete if the invocation failed. - bool TryLoadMsBuildAssemblies(IEnumerable searchLocations, GraphBuilderReporter reporter, out string failureReason, out IReadOnlyDictionary locatedAssemblyPaths, out string locatedMsBuildExePath); + bool TryLoadMsBuildAssemblies( + IEnumerable searchLocations, + GraphBuilderReporter reporter, + out string failureReason, + out IReadOnlyDictionary locatedAssemblyPaths, + out string locatedMsBuildExePath); } } \ No newline at end of file diff --git a/Public/Src/Tools/Tool.MsBuildGraphBuilder/MsBuildAssemblyLoader.cs b/Public/Src/Tools/Tool.MsBuildGraphBuilder/MsBuildAssemblyLoader.cs index 5afbf1d6d5..cfde234361 100644 --- a/Public/Src/Tools/Tool.MsBuildGraphBuilder/MsBuildAssemblyLoader.cs +++ b/Public/Src/Tools/Tool.MsBuildGraphBuilder/MsBuildAssemblyLoader.cs @@ -10,6 +10,7 @@ using System.Linq; using System.Reflection; using System.Text; +using Microsoft.Build.Tasks; namespace MsBuildGraphBuilderTool { @@ -31,48 +32,56 @@ public sealed class MsBuildAssemblyLoader : IMsBuildAssemblyLoader private const string SystemCollectionsImmutable = "System.Collections.Immutable.dll"; private const string SystemThreadingDataflow = "System.Threading.Tasks.Dataflow.dll"; - // MsBuild.exe is not a required to be loaded but is required to be found - private const string MsBuildExe = "MsBuild.exe"; + // MsBuild is not a required to be loaded but is required to be found + // Under DotNetCore there is not an msbuild.exe but a msbuild.dll + private string m_msbuild; - private static readonly string[] s_assemblyNamesToLoad = new[] - { - MsBuildExe, - MicrosoftBuild, - MicrosoftBuildFramework, - MicrosoftBuildUtilities, - SystemCollectionsImmutable, - SystemThreadingDataflow - }; + private readonly string[] m_assemblyNamesToLoad; /// /// The expected public token for each required assembly /// - private static readonly Dictionary s_assemblyPublicTokens = new Dictionary - { - [MsBuildExe] = MSBuildPublicKeyToken, - [MicrosoftBuild] = MSBuildPublicKeyToken, - [MicrosoftBuildFramework] = MSBuildPublicKeyToken, - [MicrosoftBuildUtilities] = MSBuildPublicKeyToken, - [SystemCollectionsImmutable] = DotNetPublicToken, - [SystemThreadingDataflow] = DotNetPublicToken, - }; + private readonly Dictionary m_assemblyPublicTokens; private static ResolveEventHandler s_msBuildHandler; // Assembly resolution is multi-threaded - private readonly ConcurrentDictionary m_loadedAssemblies = new ConcurrentDictionary(5, s_assemblyNamesToLoad.Length, StringComparer.OrdinalIgnoreCase); - - private static readonly Lazy s_singleton = new Lazy(() => new MsBuildAssemblyLoader()); + private readonly ConcurrentDictionary m_loadedAssemblies; - private MsBuildAssemblyLoader() + public MsBuildAssemblyLoader(bool msBuildRuntimeIsDotNetCore) { - } + m_msbuild = msBuildRuntimeIsDotNetCore ? "MSBuild.dll" : "MSBuild.exe"; - /// - public static MsBuildAssemblyLoader Instance => s_singleton.Value; + m_assemblyNamesToLoad = new[] + { + m_msbuild, + MicrosoftBuild, + MicrosoftBuildFramework, + MicrosoftBuildUtilities, + SystemCollectionsImmutable, + SystemThreadingDataflow + }; + + m_assemblyPublicTokens = new Dictionary + { + [m_msbuild] = MSBuildPublicKeyToken, + [MicrosoftBuild] = MSBuildPublicKeyToken, + [MicrosoftBuildFramework] = MSBuildPublicKeyToken, + [MicrosoftBuildUtilities] = MSBuildPublicKeyToken, + [SystemCollectionsImmutable] = DotNetPublicToken, + [SystemThreadingDataflow] = DotNetPublicToken, + }; + + m_loadedAssemblies = new ConcurrentDictionary(5, m_assemblyNamesToLoad.Length, StringComparer.OrdinalIgnoreCase); + } /// - public bool TryLoadMsBuildAssemblies(IEnumerable searchLocations, GraphBuilderReporter reporter, out string failureReason, out IReadOnlyDictionary locatedAssemblyPaths, out string locatedMsBuildExePath) + public bool TryLoadMsBuildAssemblies( + IEnumerable searchLocations, + GraphBuilderReporter reporter, + out string failureReason, + out IReadOnlyDictionary locatedAssemblyPaths, + out string locatedMsBuildExePath) { // We want to make sure the provided locations actually contain all the needed assemblies if (!TryGetAssemblyLocations(searchLocations, reporter, out locatedAssemblyPaths, out IReadOnlyDictionary missingAssemblyNames, out locatedMsBuildExePath)) @@ -104,7 +113,7 @@ public bool TryLoadMsBuildAssemblies(IEnumerable searchLocations, GraphB // Automatically un-register the handler once all supported assemblies have been loaded. // No need to synchronize threads here, if the handler was already removed, nothing bad happens - if (m_loadedAssemblies.Count == s_assemblyNamesToLoad.Length) + if (m_loadedAssemblies.Count == m_assemblyNamesToLoad.Length) { AppDomain.CurrentDomain.AssemblyResolve -= s_msBuildHandler; } @@ -128,10 +137,10 @@ public bool TryLoadMsBuildAssemblies(IEnumerable searchLocations, GraphB out IReadOnlyDictionary missingAssemblies, out string locatedMsBuildExePath) { - var foundAssemblies = new Dictionary(s_assemblyNamesToLoad.Length, StringComparer.OrdinalIgnoreCase); - var notFoundAssemblies = new Dictionary(s_assemblyNamesToLoad.Length, StringComparer.OrdinalIgnoreCase); + var foundAssemblies = new Dictionary(m_assemblyNamesToLoad.Length, StringComparer.OrdinalIgnoreCase); + var notFoundAssemblies = new Dictionary(m_assemblyNamesToLoad.Length, StringComparer.OrdinalIgnoreCase); - var assembliesToFind = new HashSet(s_assemblyNamesToLoad, StringComparer.OrdinalIgnoreCase); + var assembliesToFind = new HashSet(m_assemblyNamesToLoad, StringComparer.OrdinalIgnoreCase); locatedMsBuildExePath = string.Empty; foreach (var location in locations) @@ -142,7 +151,7 @@ public bool TryLoadMsBuildAssemblies(IEnumerable searchLocations, GraphB try { var dlls = Directory.EnumerateFiles(location, "*.dll", SearchOption.TopDirectoryOnly); - var msBuildExe = Directory.EnumerateFiles(location, MsBuildExe, SearchOption.TopDirectoryOnly); + var msBuildExe = Directory.EnumerateFiles(location, m_msbuild, SearchOption.TopDirectoryOnly); foreach (string fullPath in dlls.Union(msBuildExe)) { @@ -158,7 +167,7 @@ public bool TryLoadMsBuildAssemblies(IEnumerable searchLocations, GraphB // If the file found is msbuild.exe, we update the associated location but don't store the path // in foundAssemblies, since this is not an assembly we really want to load - if (file.Equals(MsBuildExe, StringComparison.OrdinalIgnoreCase)) + if (file.Equals(m_msbuild, StringComparison.OrdinalIgnoreCase)) { locatedMsBuildExePath = fullPath; } @@ -222,7 +231,7 @@ public bool TryLoadMsBuildAssemblies(IEnumerable searchLocations, GraphB return assembliesToFind.Count == 0; } - private static bool IsMsBuildAssembly(AssemblyName assemblyName, string fullPathToAssembly, out string notFoundReason) + private bool IsMsBuildAssembly(AssemblyName assemblyName, string fullPathToAssembly, out string notFoundReason) { if (string.Equals($"{assemblyName.Name}.dll", MicrosoftBuild, StringComparison.OrdinalIgnoreCase)) { @@ -258,7 +267,7 @@ private static bool IsMsBuildAssembly(AssemblyName assemblyName, string fullPath sb.Append($"{b:x2}"); } - if (s_assemblyPublicTokens.TryGetValue(assemblyName.Name, out string publicToken) && !sb.ToString().Equals(publicToken, StringComparison.OrdinalIgnoreCase)) + if (m_assemblyPublicTokens.TryGetValue(assemblyName.Name, out string publicToken) && !sb.ToString().Equals(publicToken, StringComparison.OrdinalIgnoreCase)) { notFoundReason = $"The assembly under '{fullPathToAssembly}' is expected to contain public token '{publicToken}' but '{sb}' was found."; return false; diff --git a/Public/Src/Tools/Tool.MsBuildGraphBuilder/MsBuildGraphBuilder.cs b/Public/Src/Tools/Tool.MsBuildGraphBuilder/MsBuildGraphBuilder.cs index aa6a045d3c..da29364146 100644 --- a/Public/Src/Tools/Tool.MsBuildGraphBuilder/MsBuildGraphBuilder.cs +++ b/Public/Src/Tools/Tool.MsBuildGraphBuilder/MsBuildGraphBuilder.cs @@ -54,7 +54,7 @@ public static class MsBuildGraphBuilder // The output file is used as a unique name to identify the pipe using (var reporter = new GraphBuilderReporter(Path.GetFileName(arguments.OutputPath))) { - DoBuildGraphAndSerialize(MsBuildAssemblyLoader.Instance, reporter, arguments); + DoBuildGraphAndSerialize(new MsBuildAssemblyLoader(arguments.MsBuildRuntimeIsDotNetCore), reporter, arguments); } } diff --git a/Public/Src/Tools/Tool.MsBuildGraphBuilder/Tool.MsBuildGraphBuilder.dsc b/Public/Src/Tools/Tool.MsBuildGraphBuilder/Tool.MsBuildGraphBuilder.dsc index e3bb4ad021..ff0120a13f 100644 --- a/Public/Src/Tools/Tool.MsBuildGraphBuilder/Tool.MsBuildGraphBuilder.dsc +++ b/Public/Src/Tools/Tool.MsBuildGraphBuilder/Tool.MsBuildGraphBuilder.dsc @@ -6,10 +6,11 @@ import * as BuildXLSdk from "Sdk.BuildXL"; import {Transformer} from "Sdk.Transformers"; import * as Deployment from "Sdk.Deployment"; import * as MSBuild from "Sdk.Selfhost.MSBuild"; +import * as Frameworks from "Sdk.Managed.Frameworks"; +import * as Shared from "Sdk.Managed.Shared"; namespace MsBuildGraphBuilder { - // TODO: We want this to be netstandard too but since Build Prediction is not, we have to keep it net47 only - export declare const qualifier: MSBuild.MSBuildQualifier; + export declare const qualifier: BuildXLSdk.DefaultQualifier; @@public export const exe = BuildXLSdk.executable({ @@ -23,8 +24,8 @@ namespace MsBuildGraphBuilder { importFrom("BuildXL.Utilities").Collections.dll, importFrom("BuildXL.Utilities").Native.dll, importFrom("BuildXL.Utilities.Instrumentation").Common.dll, - importFrom("System.Collections.Immutable").pkg, - importFrom("DataflowForMSBuildRuntime").pkg, + ...addIf(BuildXLSdk.isFullFramework, importFrom("System.Collections.Immutable").pkg), + ...addIf(BuildXLSdk.isFullFramework, importFrom("System.Threading.Tasks.Dataflow").pkg), importFrom("Microsoft.Build.Prediction").pkg, NetFx.System.Threading.Tasks.dll, ...MSBuild.msbuildReferences, @@ -35,10 +36,30 @@ namespace MsBuildGraphBuilder { runtimeContentToSkip: [ // don't add msbuild dlls because assembly resolvers will resolve msbuild from other MSBuild installations ...MSBuild.msbuildReferences, - importFrom("System.Threading.Tasks.Dataflow").pkg ], internalsVisibleTo: [ "Test.Tool.ProjectGraphBuilder", ] }); + + @@public + export const deployment : Deployment.Definition = { contents: [{ + subfolder: r`MsBuildGraphBuilder`, + contents: [ + { + subfolder: r`net472`, + contents: [ + $.withQualifier(Object.merge(qualifier, {targetFramework: "net472"})) + .MsBuildGraphBuilder.exe + ] + }, + { + subfolder: r`dotnetcore`, + contents: [ + $.withQualifier(Object.merge(qualifier, {targetFramework: "netcoreapp3.0"})) + .MsBuildGraphBuilder.exe + ] + } + ] + }]}; } diff --git a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildAssemblyLoaderTests.cs b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildAssemblyLoaderTests.cs index e4ca065ec6..5a0616e3fd 100644 --- a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildAssemblyLoaderTests.cs +++ b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildAssemblyLoaderTests.cs @@ -5,12 +5,13 @@ using System.Linq; using MsBuildGraphBuilderTool; using Test.BuildXL.TestUtilities.Xunit; +using Test.ProjectGraphBuilder.Utilities; using Xunit; using Xunit.Abstractions; namespace Test.ProjectGraphBuilder { - public class MsBuildAssemblyLoaderTests : XunitBuildXLTest + public class MsBuildAssemblyLoaderTests : GraphBuilderToolTestBase { public MsBuildAssemblyLoaderTests(ITestOutputHelper output): base(output) { @@ -21,8 +22,7 @@ public void CorrectAssembliesAreSuccessfullyLoaded() { using (var reporter = new GraphBuilderReporter(Guid.NewGuid().ToString())) { - var assemblyLoader = MsBuildAssemblyLoader.Instance; - var succeed = assemblyLoader.TryLoadMsBuildAssemblies( + var succeed = AssemblyLoader.TryLoadMsBuildAssemblies( // The test deployment dir should have all assemblies needed by the loader new [] {TestDeploymentDir}, reporter, @@ -44,8 +44,7 @@ public void NotFoundAssembliesGetReported() { using (var reporter = new GraphBuilderReporter(Guid.NewGuid().ToString())) { - var assemblyLoader = MsBuildAssemblyLoader.Instance; - var succeed = assemblyLoader.TryLoadMsBuildAssemblies( + var succeed = AssemblyLoader.TryLoadMsBuildAssemblies( // An empty location should result in not finding any of the required assemblies new string[] {}, reporter, diff --git a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphConstructionTests.cs b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphConstructionTests.cs index ebf5d18f94..c00eda9dce 100644 --- a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphConstructionTests.cs +++ b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphConstructionTests.cs @@ -14,7 +14,7 @@ namespace Test.ProjectGraphBuilder { - public class MsBuildGraphConstructionTests : TemporaryStorageTestBase + public class MsBuildGraphConstructionTests : GraphBuilderToolTestBase { private readonly MsBuildProjectBuilder m_builder; @@ -233,14 +233,13 @@ private ProjectGraphWithPredictionsResult BuildGraphAndDeserialize(IRead { string outputFile = Path.Combine(TemporaryDirectory, Guid.NewGuid().ToString()); - var arguments = new MSBuildGraphBuilderArguments( - projectEntryPoints, - outputFile, - globalProperties: GlobalProperties.Empty, - mSBuildSearchLocations: new[] {TestDeploymentDir}, - entryPointTargets: new string[0], - requestedQualifiers: new GlobalProperties[] { GlobalProperties.Empty }, - allowProjectsWithoutTargetProtocol: true); + var arguments = GetStandardBuilderArguments( + projectEntryPoints, + outputFile, + GlobalProperties.Empty, + new string[0], + new GlobalProperties[] { GlobalProperties.Empty }, + allowProjectsWithoutTargetProtocol: true); return BuildGraphAndDeserialize(arguments); @@ -249,14 +248,14 @@ private ProjectGraphWithPredictionsResult BuildGraphAndDeserialize(IRead private MSBuildGraphBuilderArguments CreateBuilderArguments(string entryPointPath, GlobalProperties[] requestedQualifiers = null, GlobalProperties globalProperties = null, bool allowProjectsWithoutTargetProtocol = true) { string outputFile = Path.Combine(TemporaryDirectory, Guid.NewGuid().ToString()); - var arguments = new MSBuildGraphBuilderArguments( + var arguments = GetStandardBuilderArguments( new string[] { entryPointPath }, outputFile, globalProperties: globalProperties ?? GlobalProperties.Empty, - mSBuildSearchLocations: new[] { TestDeploymentDir }, entryPointTargets: new string[0], requestedQualifiers: requestedQualifiers ?? new GlobalProperties[] { GlobalProperties.Empty }, allowProjectsWithoutTargetProtocol: allowProjectsWithoutTargetProtocol); + return arguments; } } diff --git a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphDecorationTests.cs b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphDecorationTests.cs index 9a6ab0b0f1..a12bdf2baf 100644 --- a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphDecorationTests.cs +++ b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphDecorationTests.cs @@ -16,7 +16,7 @@ namespace Test.ProjectGraphBuilder /// /// Tests related to the decorations associated to the project graph (e.g. failures, location of assemblies, etc.) /// - public class MsBuildGraphDecorationTests : TemporaryStorageTestBase + public class MsBuildGraphDecorationTests : GraphBuilderToolTestBase { public MsBuildGraphDecorationTests(ITestOutputHelper output): base(output) { @@ -56,13 +56,13 @@ public void ValidProjectFileSucceedsAndAssemblyLocationsAreSetProperly() // We expect the result to succeed Assert.True(projectGraphWithPredictionsResult.Succeeded); // The locations for MSBuild.exe and its assemblies should be properly set - Assert.Contains(TestDeploymentDir, projectGraphWithPredictionsResult.PathToMsBuildExe); + Assert.Contains(TestDeploymentDir, projectGraphWithPredictionsResult.PathToMsBuild); Assert.All(projectGraphWithPredictionsResult.MsBuildAssemblyPaths.Values, assemblyPath => assemblyPath.Contains(TestDeploymentDir)); } private ProjectGraphWithPredictionsResult BuildGraphAndDeserialize(string projectEntryPointContent = null) { - return BuildGraphAndDeserialize(MsBuildAssemblyLoader.Instance, projectEntryPointContent); + return BuildGraphAndDeserialize(AssemblyLoader, projectEntryPointContent); } private ProjectGraphWithPredictionsResult BuildGraphAndDeserialize(IMsBuildAssemblyLoader assemblyLoader, string projectEntryPointContent = null) @@ -77,11 +77,10 @@ private ProjectGraphWithPredictionsResult BuildGraphAndDeserialize(IMsBu using (var reporter = new GraphBuilderReporter(Guid.NewGuid().ToString())) { - var arguments = new MSBuildGraphBuilderArguments( + var arguments = GetStandardBuilderArguments( new[] { entryPoint }, outputFile, globalProperties: GlobalProperties.Empty, - mSBuildSearchLocations: new string[] {TestDeploymentDir}, entryPointTargets: new string[0], requestedQualifiers: new GlobalProperties[] { GlobalProperties.Empty}, allowProjectsWithoutTargetProtocol: false); diff --git a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphProgressTests.cs b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphProgressTests.cs index fc61dfd03a..bc4b3fad30 100644 --- a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphProgressTests.cs +++ b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphProgressTests.cs @@ -8,14 +8,13 @@ using System.Threading.Tasks; using BuildXL.FrontEnd.MsBuild.Serialization; using MsBuildGraphBuilderTool; -using Test.BuildXL.TestUtilities.Xunit; using Test.ProjectGraphBuilder.Utilities; using Xunit; using Xunit.Abstractions; namespace Test.ProjectGraphBuilder { - public class MsBuildGraphProgressTests : TemporaryStorageTestBase + public class MsBuildGraphProgressTests : GraphBuilderToolTestBase { private readonly string m_entryPoint; @@ -69,16 +68,15 @@ public void ProgressIsReported() private bool BuildAndReport(GraphBuilderReporter reporter, out string failure) { string outputFile = Path.Combine(TemporaryDirectory, Guid.NewGuid().ToString()); - var arguments = new MSBuildGraphBuilderArguments( + var arguments = GetStandardBuilderArguments( new[] { m_entryPoint }, outputFile, globalProperties: GlobalProperties.Empty, - mSBuildSearchLocations: new[] {TestDeploymentDir}, entryPointTargets: new string[0], requestedQualifiers: new GlobalProperties[] { GlobalProperties.Empty }, allowProjectsWithoutTargetProtocol: false); - MsBuildGraphBuilder.BuildGraphAndSerializeForTesting(MsBuildAssemblyLoader.Instance, reporter, arguments); + MsBuildGraphBuilder.BuildGraphAndSerializeForTesting(AssemblyLoader, reporter, arguments); var result = SimpleDeserializer.Instance.DeserializeGraph(outputFile); failure = string.Empty; @@ -117,6 +115,5 @@ private Task ConnectToServerPipeAndLogProgress(string pipeName) } ); } - } } \ No newline at end of file diff --git a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphProjectPredictionTests.cs b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphProjectPredictionTests.cs index 95cca274d9..7d3845d228 100644 --- a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphProjectPredictionTests.cs +++ b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildGraphProjectPredictionTests.cs @@ -19,7 +19,7 @@ namespace Test.ProjectGraphBuilder /// /// Makes sure that project predictions are plumbed through and serialized into the project graph. The actual predictions are not tested here. /// - public class MsBuildGraphProjectPredictionTests : TemporaryStorageTestBase + public class MsBuildGraphProjectPredictionTests : GraphBuilderToolTestBase { public MsBuildGraphProjectPredictionTests(ITestOutputHelper output): base(output) { @@ -44,11 +44,10 @@ public void ProjectPredictionsAreSerialized() "); MsBuildGraphBuilder.BuildGraphAndSerialize( - new MSBuildGraphBuilderArguments( + GetStandardBuilderArguments( new[] { entryPoint }, outputFile, globalProperties: GlobalProperties.Empty, - mSBuildSearchLocations: new[] { TestDeploymentDir }, entryPointTargets: new string[0], requestedQualifiers: new GlobalProperties[] { GlobalProperties.Empty }, allowProjectsWithoutTargetProtocol: false)); @@ -72,17 +71,16 @@ public void ProblematicPredictorsAreHandled() using (var reporter = new GraphBuilderReporter(Guid.NewGuid().ToString())) { - var arguments = new MSBuildGraphBuilderArguments( + var arguments = GetStandardBuilderArguments( new[] { entryPoint }, outputFile, globalProperties: GlobalProperties.Empty, - mSBuildSearchLocations: new[] { TestDeploymentDir }, entryPointTargets: new string[0], requestedQualifiers: new GlobalProperties[] { GlobalProperties.Empty }, allowProjectsWithoutTargetProtocol: false); MsBuildGraphBuilder.BuildGraphAndSerializeForTesting( - MsBuildAssemblyLoader.Instance, + AssemblyLoader, reporter, arguments, new IProjectPredictor[] { new ThrowOnPredictionPredictor() }); diff --git a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildPredictionCollectorTests.cs b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildPredictionCollectorTests.cs index 1c21a6e3b2..935c9960bb 100644 --- a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildPredictionCollectorTests.cs +++ b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/MsBuildPredictionCollectorTests.cs @@ -68,7 +68,7 @@ public void AddInputFileHandlesBadPaths() var predictionFailures = new ConcurrentQueue<(string predictorName, string failure)>(); var collector = new MsBuildPredictionCollector(inputFilePredictions, outputFolderPredictions, predictionFailures); - collector.AddInputFile("!@#$%^&*()", TemporaryDirectory, "Mock"); + collector.AddInputFile("!@#$%^&*()\0", TemporaryDirectory, "Mock"); Assert.Equal(0, inputFilePredictions.Count); @@ -76,7 +76,7 @@ public void AddInputFileHandlesBadPaths() Assert.Equal(1, predictionFailures.Count); Assert.Equal("Mock", predictionFailures.Single().predictorName); - Assert.Contains("!@#$%^&*()", predictionFailures.Single().failure); + Assert.Contains("!@#$%^&*()\0", predictionFailures.Single().failure); } [Fact] @@ -160,7 +160,7 @@ public void AddInputDirectoryHandlesBadPaths() var predictionFailures = new ConcurrentQueue<(string predictorName, string failure)>(); var collector = new MsBuildPredictionCollector(inputFilePredictions, outputFolderPredictions, predictionFailures); - collector.AddInputDirectory("!@#$%^&*()", TemporaryDirectory, "Mock"); + collector.AddInputDirectory("!@#$%^&*()\0", TemporaryDirectory, "Mock"); Assert.Equal(0, inputFilePredictions.Count); @@ -168,7 +168,7 @@ public void AddInputDirectoryHandlesBadPaths() Assert.Equal(1, predictionFailures.Count); Assert.Equal("Mock", predictionFailures.Single().predictorName); - Assert.Contains("!@#$%^&*()", predictionFailures.Single().failure); + Assert.Contains("!@#$%^&*()\0", predictionFailures.Single().failure); } [Fact] @@ -222,7 +222,7 @@ public void AddOutputFileHandlesBadPaths() var predictionFailures = new ConcurrentQueue<(string predictorName, string failure)>(); var collector = new MsBuildPredictionCollector(inputFilePredictions, outputFolderPredictions, predictionFailures); - collector.AddOutputFile("!@#$%^&*()", TemporaryDirectory, "Mock"); + collector.AddOutputFile("!@#$%^&*()\0", TemporaryDirectory, "Mock"); Assert.Equal(0, inputFilePredictions.Count); @@ -230,7 +230,7 @@ public void AddOutputFileHandlesBadPaths() Assert.Equal(1, predictionFailures.Count); Assert.Equal("Mock", predictionFailures.Single().predictorName); - Assert.Contains("!@#$%^&*()", predictionFailures.Single().failure); + Assert.Contains("!@#$%^&*()\0", predictionFailures.Single().failure); } [Fact] @@ -282,7 +282,7 @@ public void AddOutputDirectoryHandlesBadPaths() var predictionFailures = new ConcurrentQueue<(string predictorName, string failure)>(); var collector = new MsBuildPredictionCollector(inputFilePredictions, outputFolderPredictions, predictionFailures); - collector.AddOutputDirectory("!@#$%^&*()", TemporaryDirectory, "Mock"); + collector.AddOutputDirectory("!@#$%^&*()\0", TemporaryDirectory, "Mock"); Assert.Equal(0, inputFilePredictions.Count); diff --git a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/Test.Tool.MsBuildGraphBuilder.dsc b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/Test.Tool.MsBuildGraphBuilder.dsc index 4c43e992ef..6395788c55 100644 --- a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/Test.Tool.MsBuildGraphBuilder.dsc +++ b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/Test.Tool.MsBuildGraphBuilder.dsc @@ -6,20 +6,15 @@ import * as MSBuild from "Sdk.Selfhost.MSBuild"; namespace Test.Tool.MsBuildGraphBuilder { - export declare const qualifier: MSBuild.MSBuildQualifier; - - // If the current qualifier is full framework, this tool has to be built with 472 - const msBuildGraphBuilderReference : Managed.Assembly = - importFrom("BuildXL.Tools").MsBuildGraphBuilder.withQualifier(to472(qualifier)).exe; + export declare const qualifier: BuildXLSdk.DefaultQualifier; @@public export const dll = BuildXLSdk.test({ assemblyName: "Test.Tool.ProjectGraphBuilder", sources: globR(d`.`, "*.cs"), - // TODO: QTest testFramework: importFrom("Sdk.Managed.Testing.XUnit").framework, references:[ - msBuildGraphBuilderReference, + importFrom("BuildXL.Tools").MsBuildGraphBuilder.exe, importFrom("Microsoft.Build.Prediction").pkg, importFrom("Newtonsoft.Json").pkg, importFrom("BuildXL.FrontEnd").MsBuild.Serialization.dll, @@ -29,12 +24,8 @@ namespace Test.Tool.MsBuildGraphBuilder { ...MSBuild.msbuildRuntimeContent, ...MSBuild.msbuildReferences, ], - runtimeContentToSkip : [ + runtimeContentToSkip: [ importFrom("System.Threading.Tasks.Dataflow").pkg ] }); - - function to472(aQualifier: (typeof qualifier)) : (typeof qualifier) & {targetFramework: "net472"} { - return Object.merge<(typeof qualifier) & {targetFramework: "net472"}>(aQualifier, {targetFramework: "net472"}); - } } diff --git a/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/Utilities/GraphBuilderToolTestBase.cs b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/Utilities/GraphBuilderToolTestBase.cs new file mode 100644 index 0000000000..078e073268 --- /dev/null +++ b/Public/Src/Tools/UnitTests/MsBuildGraphBuilder/Utilities/GraphBuilderToolTestBase.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using BuildXL.FrontEnd.MsBuild.Serialization; +using MsBuildGraphBuilderTool; +using Test.BuildXL.TestUtilities.Xunit; +using Xunit.Abstractions; + +namespace Test.ProjectGraphBuilder.Utilities +{ + public abstract class GraphBuilderToolTestBase : TemporaryStorageTestBase + { + /// + public GraphBuilderToolTestBase(ITestOutputHelper output) : base(output) {} + +#if FEATURE_CORECLR + /// + protected bool RunningUnderDotNetCore => true; +#else + /// + protected bool RunningUnderDotNetCore => false; +#endif + /// + protected MsBuildAssemblyLoader AssemblyLoader => new MsBuildAssemblyLoader(RunningUnderDotNetCore); + + /// + protected MSBuildGraphBuilderArguments GetStandardBuilderArguments( + IReadOnlyCollection projectsToParse, + string outputFile, + GlobalProperties globalProperties, + IReadOnlyCollection entryPointTargets, + IReadOnlyCollection requestedQualifiers, + bool allowProjectsWithoutTargetProtocol) + { + return new MSBuildGraphBuilderArguments( + projectsToParse, + outputFile, + globalProperties, + mSBuildSearchLocations: new[] {TestDeploymentDir}, + entryPointTargets, + requestedQualifiers, + allowProjectsWithoutTargetProtocol, + RunningUnderDotNetCore); + } + } +} diff --git a/Public/Src/Utilities/Configuration/Resolvers/IMsBuildResolverSettings.cs b/Public/Src/Utilities/Configuration/Resolvers/IMsBuildResolverSettings.cs index 74a19e9df6..2d13a66d88 100644 --- a/Public/Src/Utilities/Configuration/Resolvers/IMsBuildResolverSettings.cs +++ b/Public/Src/Utilities/Configuration/Resolvers/IMsBuildResolverSettings.cs @@ -56,6 +56,26 @@ public interface IMsBuildResolverSettings : IResolverSettings, IUntrackingSettin /// IReadOnlyList MsBuildSearchLocations { get; } + /// + /// Whether to use the full framework or dotnet core version of MSBuild. + /// + /// + /// Selected runtime is used both for build evaluation and execution. + /// Default is full framework. + /// Observe that using the full framework version means that msbuild.exe is expected to be found in msbuildSearchLocations + /// (or PATH if not specified). If using the dotnet core version, the same logic applies but to msbuild.dll + /// + string MsBuildRuntime { get; } + + /// + /// Collection of directories to search for dotnet.exe, when DotNetCore is specified as the msBuildRuntime. + /// + /// + /// If not specified, locations in %PATH% are used. + /// Locations are traversed in specification order. + /// + IReadOnlyList DotNetSearchLocations { get; } + /// /// Optional file paths for the projects or solutions that should be used to start parsing. These are relative /// paths with respect to the root traversal. @@ -193,5 +213,14 @@ public static void ComputeEnvironment(this IMsBuildResolverSettings msBuildResol trackedEnv = trackedList; passthroughEnv = passthroughList; } + + /// + /// Whether MSBuildRuntime is DotNetCore. + /// + /// + /// Keep in sync with Public\Sdk\Public\Prelude\Prelude.Configuration.Resolvers.dsc + /// If not specified, the default is full framework, so this function returns false in that case. + /// + public static bool ShouldRunDotNetCoreMSBuild(this IMsBuildResolverSettings msBuildResolverSettings) => msBuildResolverSettings.MsBuildRuntime == "DotNetCore"; } } diff --git a/Public/Src/Utilities/Configuration/Resolvers/Mutable/MsBuildResolverSettings.cs b/Public/Src/Utilities/Configuration/Resolvers/Mutable/MsBuildResolverSettings.cs index 1eb16a465b..221490b94b 100644 --- a/Public/Src/Utilities/Configuration/Resolvers/Mutable/MsBuildResolverSettings.cs +++ b/Public/Src/Utilities/Configuration/Resolvers/Mutable/MsBuildResolverSettings.cs @@ -45,6 +45,8 @@ public MsBuildResolverSettings() UseLegacyProjectIsolation = resolverSettings.UseLegacyProjectIsolation; DoubleWritePolicy = resolverSettings.DoubleWritePolicy; AllowProjectsToNotSpecifyTargetProtocol = resolverSettings.AllowProjectsToNotSpecifyTargetProtocol; + MsBuildRuntime = resolverSettings.MsBuildRuntime; + DotNetSearchLocations = resolverSettings.DotNetSearchLocations; } /// @@ -109,5 +111,11 @@ public MsBuildResolverSettings() /// public bool? AllowProjectsToNotSpecifyTargetProtocol { get; set; } + + /// + public string MsBuildRuntime { get; set; } + + /// + public IReadOnlyList DotNetSearchLocations { get; set; } } } diff --git a/config.dsc b/config.dsc index bc4a832a86..11a77590b6 100644 --- a/config.dsc +++ b/config.dsc @@ -127,7 +127,6 @@ config({ { id: "System.Data.SQLite.Linq", version: "1.0.102.0" }, { id: "System.Reflection.Metadata", version: "1.6.0" }, { id: "System.Threading.Tasks.Dataflow", version: "4.9.0" }, - { id: "System.Threading.Tasks.Dataflow", version: "4.5.24", alias: "DataflowForMSBuildRuntime"}, // Nuget { id: "NuGet.Commandline", version: "4.7.1" }, @@ -370,7 +369,6 @@ config({ { id: "System.Security.Principal.Windows", version: "4.6.0-preview5.19224.8" }, { id: "System.Security.SecureString", version: "4.3.0" }, { id: "System.Text.Encoding", version: "4.3.0" }, - { id: "System.Text.Encoding.CodePages", version: "4.3.0" }, { id: "System.Text.Encoding.Extensions", version: "4.3.0" }, { id: "System.Text.RegularExpressions", version: "4.3.0" }, { id: "System.Threading", version: "4.3.0" }, @@ -443,6 +441,9 @@ config({ // Extra dependencies to make MSBuild work { id: "Microsoft.VisualStudio.Setup.Configuration.Interop", version: "1.16.30"}, { id: "System.CodeDom", version: "4.4.0"}, + { id: "System.Text.Encoding.CodePages", version: "4.5.1", dependentPackageIdsToSkip: ["System.Runtime.CompilerServices.Unsafe"]}, + { id: "System.Threading.Tasks.Dataflow", version: "4.5.24", alias: "DataflowForMSBuild" }, + { id: "Microsoft.NETCore.App", version: "2.1.0", alias: "Microsoft.NETCore.App.210" }, // Used for MSBuild input/output prediction { id: "Microsoft.Build.Prediction", version: "0.3.0" }, @@ -481,6 +482,10 @@ config({ importFile(f`config.microsoftInternal.dsc`).resolver, + { + kind: "SourceResolver", + modules: [f`Public\Sdk\SelfHost\Libraries\Dotnet-Runtime-External\module.config.dsc`], + }, { kind: "Download", downloads: [ @@ -506,24 +511,42 @@ config({ // DotNet Core Runtime { - moduleName: "DotNet-Runtime.win-x64", + moduleName: "DotNet-Runtime.win-x64.3.0.0-preview5", url: "https://download.visualstudio.microsoft.com/download/pr/9459ede1-e223-40c7-a4c5-2409e789121a/46d4eb6067bda9f412a472f7286ffd94/dotnet-runtime-3.0.0-preview5-27626-15-win-x64.zip", hash: "VSO0:6DBFE7BC9FA24D33A46A3A0732164BD5A4F5984E8FCE091D305FA635CD876AA700", archiveType: "zip", }, { - moduleName: "DotNet-Runtime.osx-x64", + moduleName: "DotNet-Runtime.osx-x64.3.0.0-preview5", url: "https://download.visualstudio.microsoft.com/download/pr/85024962-5dee-4f64-ab29-a903f3749f85/6178bfacc58f4d9a596b5e3facc767ab/dotnet-runtime-3.0.0-preview5-27626-15-osx-x64.tar.gz", hash: "VSO0:C6AB5808D30BFF857263BC467FE8D818F35486763F673F79CA5A758727CEF3A900", archiveType: "tgz", }, { - moduleName: "DotNet-Runtime.linux-x64", + moduleName: "DotNet-Runtime.linux-x64.3.0.0-preview5", url: "https://download.visualstudio.microsoft.com/download/pr/f15ad9ab-7bd2-4ff5-87b6-b1a08f062ea2/6fdd314c16c17ba22934cd0ac6b4d343/dotnet-runtime-3.0.0-preview5-27626-15-linux-x64.tar.gz", - hash: "VSO0:C6AB5808D30BFF857263BC467FE8D818F35486763F673F79CA5A758727CEF3A900", + hash: "VSO0:00F83B929904F647BD8FB22361052BB347A1E5FA9A3A32A67EE1569DE443D92700", + archiveType: "tgz", + }, + // The following are needed for dotnet core MSBuild test deployments + { + moduleName: "DotNet-Runtime.win-x64.2.2.2", + url: "https://download.visualstudio.microsoft.com/download/pr/b10d0a68-b720-48ae-bab8-4ac39bd1b5d3/f32b8b41dff5c1488c2b915a007fc4a6/dotnet-runtime-2.2.2-win-x64.zip", + hash: "VSO0:6BBAE77F9BA0231C90ABD9EA720FF886E8613CE8EF29D8B657AF201E2982829600", + archiveType: "zip", + }, + { + moduleName: "DotNet-Runtime.osx-x64.2.2.2", + url: "https://download.visualstudio.microsoft.com/download/pr/d1f0dfb3-b6bd-42ae-895f-f149bf1d90ca/9b1fb91a9692fc31d6fc83e97caba4cd/dotnet-runtime-2.2.2-osx-x64.tar.gz", + hash: "VSO0:88B2B6E8CEF711E108FDE529E781F555516634CD442B3503B712D22947F0788700", + archiveType: "tgz", + }, + { + moduleName: "DotNet-Runtime.linux-x64.2.2.2", + url: "https://download.visualstudio.microsoft.com/download/pr/97b97652-4f74-4866-b708-2e9b41064459/7c722daf1a80a89aa8c3dec9103c24fc/dotnet-runtime-2.2.2-linux-x64.tar.gz", + hash: "VSO0:6E5172671364C65B06C9940468A62BAF70EE27392CB2CA8B2C8BFE058CCD088300", archiveType: "tgz", }, - // NodeJs { moduleName: "NodeJs.win-x64",