diff --git a/Directory.Build.props b/Directory.Build.props index 53be3bd5c9b01..e681457ecfa18 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -71,6 +71,7 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'AndroidAppBuilder', 'Debug', '$(NetCoreAppToolCurrent)', 'publish')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmAppBuilder', 'Debug', '$(NetCoreAppToolCurrent)', 'publish')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WorkloadBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'RuntimeConfigParser', 'Debug', '$(NetCoreAppToolCurrent)')) @@ -82,6 +83,7 @@ $([MSBuild]::NormalizePath('$(AndroidAppBuilderDir)', 'AndroidAppBuilder.dll')) $([MSBuild]::NormalizePath('$(WasmAppBuilderDir)', 'WasmAppBuilder.dll')) $([MSBuild]::NormalizePath('$(WasmBuildTasksDir)', 'WasmBuildTasks.dll')) + $([MSBuild]::NormalizePath('$(WorkloadBuildTasksDir)', 'WorkloadBuildTasks.dll')) $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) $([MSBuild]::NormalizePath('$(RuntimeConfigParserDir)', 'RuntimeConfigParser.dll')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)')) diff --git a/eng/Versions.props b/eng/Versions.props index 60393072c37ff..608ba5ba260b4 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -159,6 +159,7 @@ 2.0.4 4.12.0 2.14.3 + 6.0.100-preview.7.21326.4 5.0.0-preview-20201009.2 diff --git a/eng/pipelines/runtime-staging.yml b/eng/pipelines/runtime-staging.yml index 666641354bf5a..cf984c7984773 100644 --- a/eng/pipelines/runtime-staging.yml +++ b/eng/pipelines/runtime-staging.yml @@ -389,6 +389,7 @@ jobs: extraHelixArguments: /p:BrowserHost=windows scenarios: - normal + - buildwasmapps condition: >- or( eq(variables['librariesContainsChange'], true), diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index 83d95eae00859..e46aba5219de5 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -7,6 +7,10 @@ true BundleTestAppleApp;BundleTestAndroidApp + + + true @@ -289,5 +293,10 @@ AfterTargets="Build" DependsOnTargets="Publish;$(BundleTestAppTargets);ArchiveTests" /> + + diff --git a/eng/testing/tests.targets b/eng/testing/tests.targets index dd40856e629b1..1e7ec3b173dc3 100644 --- a/eng/testing/tests.targets +++ b/eng/testing/tests.targets @@ -1,13 +1,15 @@ - + RunnerTemplate.cmd RunnerTemplate.sh AppleRunnerTemplate.sh AndroidRunnerTemplate.sh WasmRunnerTemplate.sh WasmRunnerTemplate.cmd + - $(MSBuildThisFileDirectory)$(RunScriptInputName) + + $(MSBuildThisFileDirectory)$(RunScriptInputName) RunTests.sh RunTests.cmd diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index aa4fb305dbd42..cd2fa3e4e131d 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -60,6 +60,11 @@ $(BundleTestWasmAppDependsOn);_BundleAOTTestWasmAppForHelix + + + + + --interpreter + + $(ArtifactsBinDir)dotnet-workload\ + $([MSBuild]::NormalizeDirectory($(SdkPathForWorkloadTesting))) + + $(SdkPathForWorkloadTesting)version-$(SdkVersionForWorkloadTesting).stamp + + diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets index 051d90878db1a..8fef37d7fd866 100644 --- a/src/libraries/Directory.Build.targets +++ b/src/libraries/Directory.Build.targets @@ -7,6 +7,11 @@ $(TestStrongNameKeyId) + + true + $(SdkPathForWorkloadTesting)workload.stamp + + @@ -356,4 +361,87 @@ Text="Analyzers must only target netstandard2.0 since they run in the compiler which targets netstandard2.0. The following files were found to target '%(_AnalyzerPackFile.TargetFramework)': @(_AnalyzerPackFile)" /> + + + + + + + + + + <_DotNetInstallScriptPath Condition="!$([MSBuild]::IsOSPlatform('windows'))">$(DOTNET_INSTALL_DIR)/dotnet-install.sh + <_DotNetInstallScriptPath Condition=" $([MSBuild]::IsOSPlatform('windows'))">$(DOTNET_INSTALL_DIR)/dotnet-install.ps1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_NuGetSourceForWorkloads Include="dotnet6" Value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" /> + + + + + + + + $(SdkPathForWorkloadTesting)packs\Microsoft.NETCore.App.Runtime.AOT.$(NETCoreSdkRuntimeIdentifier).Cross.$(RuntimeIdentifier)\$(PackageVersion) + + + + + + + + <_SdkPropsProperties Condition="!$([MSBuild]::IsOsPlatform('Windows'))" Include="ExeSuffix" Value="" /> + <_SdkPropsProperties Condition="$([MSBuild]::IsOsPlatform('Windows'))" Include="ExeSuffix" Value=".exe" /> + <_SdkPropsProperties Include="TargetRid" Value="browser-wasm" /> + + + + + + + + diff --git a/src/libraries/pretest.proj b/src/libraries/pretest.proj index 3e924e083abd5..fc2fee13d5c33 100644 --- a/src/libraries/pretest.proj +++ b/src/libraries/pretest.proj @@ -22,6 +22,9 @@ + + + - <_ProjectsToBuild Include="$(PerScenarioProjectFile)"> + <_BaseProjectsToBuild Include="$(PerScenarioProjectFile)" Condition="'%(_Scenarios.Identity)' != 'buildwasmapps'"> $(_PropertiesToPass);Scenario=%(_Scenarios.Identity);TestArchiveRuntimeFile=$(TestArchiveRuntimeFile) - %(_ProjectsToBuild.AdditionalProperties);NeedsToBuildWasmAppsOnHelix=$(NeedsToBuildWasmAppsOnHelix) - + %(_BaseProjectsToBuild.AdditionalProperties);NeedsToBuildWasmAppsOnHelix=$(NeedsToBuildWasmAppsOnHelix) + + + + + <_TestValues Include="true;false" /> + + <_OtherProjectsToBuild Include="$(PerScenarioProjectFile)"> + $(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestValues.Identity) + %(_OtherProjectsToBuild.AdditionalProperties);NeedsToBuildWasmAppsOnHelix=$(NeedsToBuildWasmAppsOnHelix) + + + + + <_ProjectsToBuild Include="@(_BaseProjectsToBuild);@(_OtherProjectsToBuild)" /> diff --git a/src/libraries/sendtohelixhelp.proj b/src/libraries/sendtohelixhelp.proj index 12d402fcb7e39..49d45b7c6053a 100644 --- a/src/libraries/sendtohelixhelp.proj +++ b/src/libraries/sendtohelixhelp.proj @@ -7,6 +7,7 @@ true + true $(BUILD_BUILDNUMBER) @@ -53,6 +54,9 @@ $(WaitForWorkItemCompletion) true + + true + true @@ -97,7 +101,7 @@ true - true + true @@ -105,11 +109,21 @@ - - true - true - true - + + + + false + true + + + + + true + true + true + + + - powershell -command "New-SelfSignedCertificate -FriendlyName 'ASP.NET Core HTTPS development certificate' -DnsName @('localhost') -Subject 'CN = localhost' -KeyAlgorithm RSA -KeyLength 2048 -HashAlgorithm sha256 -CertStoreLocation 'Cert:\CurrentUser\My' -TextExtension @('2.5.29.37={text}1.3.6.1.5.5.7.3.1','1.3.6.1.4.1.311.84.1.1={hex}02','2.5.29.19={text}') -KeyUsage DigitalSignature,KeyEncipherment" && - - - $(HelixCommand)call RunTests.cmd - $(HelixCommand) --runtime-path %HELIX_CORRELATION_PAYLOAD% - - $(HelixCommand)./RunTests.sh - $(HelixCommand) --runtime-path "$HELIX_CORRELATION_PAYLOAD" - - @@ -224,6 +228,34 @@ + + @(HelixPreCommand) + $(HelixCommandPrefix) @(HelixCommandPrefixItem -> 'set %(Identity)', ' & ') + $(HelixCommandPrefix) @(HelixCommandPrefixItem, ' ') + true + + + + $(HelixCommandPrefix) + $(HelixCommandPrefix) & + + $(HelixCommand) dotnet dev-certs https && + + + $(HelixCommand) powershell -command "New-SelfSignedCertificate -FriendlyName 'ASP.NET Core HTTPS development certificate' -DnsName @('localhost') -Subject 'CN = localhost' -KeyAlgorithm RSA -KeyLength 2048 -HashAlgorithm sha256 -CertStoreLocation 'Cert:\CurrentUser\My' -TextExtension @('2.5.29.37={text}1.3.6.1.5.5.7.3.1','1.3.6.1.4.1.311.84.1.1={hex}02','2.5.29.19={text}') -KeyUsage DigitalSignature,KeyEncipherment" && + + + $(HelixCommand)call RunTests.cmd + $(HelixCommand) --runtime-path %HELIX_CORRELATION_PAYLOAD% + + $(HelixCommand)./RunTests.sh + $(HelixCommand) --runtime-path "$HELIX_CORRELATION_PAYLOAD" + + @@ -302,7 +334,15 @@ - + + + + + + + + @@ -311,7 +351,7 @@ - + @@ -374,7 +414,7 @@ - + @@ -392,6 +432,7 @@ + diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj index c405468695244..4b137e3ffd698 100644 --- a/src/libraries/tests.proj +++ b/src/libraries/tests.proj @@ -352,9 +352,11 @@ Condition="'$(TestTrimming)' == 'true'" AdditionalProperties="%(AdditionalProperties);SkipTrimmingProjectsRestore=true" /> + + @@ -381,10 +383,11 @@ BuildInParallel="false" /> - + + + BuildInParallel="true" /> diff --git a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Microsoft.NET.Runtime.WebAssembly.Sdk.pkgproj b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Microsoft.NET.Runtime.WebAssembly.Sdk.pkgproj index 874775fe7e47f..3db38ddbbe86a 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Microsoft.NET.Runtime.WebAssembly.Sdk.pkgproj +++ b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Microsoft.NET.Runtime.WebAssembly.Sdk.pkgproj @@ -3,6 +3,7 @@ Provides the tasks+targets, for consumption by wasm based workloads + $(IntermediateOutputPath)AutoImport.props @@ -10,12 +11,23 @@ - + + + + <_AutoImportValues Include="PackageVersion" Value="$(PackageVersion)" /> + + + + + diff --git a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/AutoImport.props b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/AutoImport.props deleted file mode 100644 index f29ea9ae7da30..0000000000000 --- a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/AutoImport.props +++ /dev/null @@ -1,5 +0,0 @@ - - - true - - \ No newline at end of file diff --git a/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/AutoImport.props.in b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/AutoImport.props.in new file mode 100644 index 0000000000000..f925992ce31e4 --- /dev/null +++ b/src/mono/nuget/Microsoft.NET.Runtime.WebAssembly.Sdk/Sdk/AutoImport.props.in @@ -0,0 +1,8 @@ + + + true + true + ${PackageVersion} + $([MSBuild]::NormalizeDirectory($(MSBuildThisFileDirectory), '..', '..', '..')) + + diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Manifest.pkgproj b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Manifest.pkgproj index 0f6c403f4104e..747da02070f2f 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Manifest.pkgproj +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Manifest.pkgproj @@ -13,11 +13,12 @@ $(IntermediateOutputPath)WorkloadManifest.json + $(IntermediateOutputPath)WorkloadManifest.targets - + @@ -37,6 +38,11 @@ TemplateFile="WorkloadManifest.json.in" Properties="@(_WorkloadManifestValues)" OutputPath="$(WorkloadManifestPath)" /> + + diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/WorkloadManifest.json.in b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/WorkloadManifest.json.in index 4c50a08f007c8..436c5b87664b9 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/WorkloadManifest.json.in +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/WorkloadManifest.json.in @@ -296,4 +296,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/WorkloadManifest.targets b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/WorkloadManifest.targets.in similarity index 77% rename from src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/WorkloadManifest.targets rename to src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/WorkloadManifest.targets.in index 2102eabc36ce6..420dc6ccfff14 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/WorkloadManifest.targets +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Manifest/WorkloadManifest.targets.in @@ -1,4 +1,9 @@ + + ${PackageVersion} + true + + true $(WasmNativeWorkload) @@ -43,4 +48,21 @@ + + + + + + + + + + diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index 034287436892b..7364188466631 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -81,7 +81,12 @@ false + + $([MSBuild]::NormalizeDirectory($(WorkloadPacksDir), 'Microsoft.NETCore.App.Runtime.Mono.browser-wasm', '$(RuntimePackInWorkloadVersion)')) + + $([MSBuild]::NormalizeDirectory($(NuGetPackageRoot), 'microsoft.netcore.app.runtime.mono.browser-wasm', '$(BundledNETCoreAppPackageVersion)')) + $([MSBuild]::NormalizeDirectory($(MicrosoftNetCoreAppRuntimePackDir), 'runtimes', 'browser-wasm')) $([MSBuild]::NormalizeDirectory($(MicrosoftNetCoreAppRuntimePackRidDir))) $([MSBuild]::NormalizeDirectory($(MicrosoftNetCoreAppRuntimePackRidDir), 'native')) diff --git a/src/mono/wasm/data/aot-tests/ProxyProjectForAOTOnHelix.proj b/src/mono/wasm/data/aot-tests/ProxyProjectForAOTOnHelix.proj index cdd5e95a4e210..3a7b66fac7389 100644 --- a/src/mono/wasm/data/aot-tests/ProxyProjectForAOTOnHelix.proj +++ b/src/mono/wasm/data/aot-tests/ProxyProjectForAOTOnHelix.proj @@ -22,7 +22,7 @@ - diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index ffc2fb92825ba..4c5983255908c 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -107,7 +107,7 @@ <_EmccVersionRaw>%(_ReversedVersionLines.Identity) - <_EmccVersionRegexPattern>^ *emcc \([^\)]+\) *([^ \(]+) *\(([^\)]+)\)$ + <_EmccVersionRegexPattern>^ *emcc \([^\)]+\) *([0-9\.]+).*\(([^\)]+)\)$ <_EmccVersion>$([System.Text.RegularExpressions.Regex]::Match($(_EmccVersionRaw), $(_EmccVersionRegexPattern)).Groups[1].Value) <_EmccVersionHash>$([System.Text.RegularExpressions.Regex]::Match($(_EmccVersionRaw), $(_EmccVersionRegexPattern)).Groups[2].Value) diff --git a/src/tasks/Common/Utils.cs b/src/tasks/Common/Utils.cs index ea2aba607dd8e..7b25107d56f98 100644 --- a/src/tasks/Common/Utils.cs +++ b/src/tasks/Common/Utils.cs @@ -185,12 +185,12 @@ internal static string CreateTemporaryBatchFile(string command) } #if NETCOREAPP - public static void DirectoryCopy(string sourceDir, string destDir, Func predicate) + public static void DirectoryCopy(string sourceDir, string destDir, Func? predicate=null) { string[] files = Directory.GetFiles(sourceDir, "*", SearchOption.AllDirectories); foreach (string file in files) { - if (!predicate(file)) + if (predicate != null && !predicate(file)) continue; string relativePath = Path.GetRelativePath(sourceDir, file); diff --git a/src/tasks/WorkloadBuildTasks/InstallWorkload.cs b/src/tasks/WorkloadBuildTasks/InstallWorkload.cs new file mode 100644 index 0000000000000..063862273ad05 --- /dev/null +++ b/src/tasks/WorkloadBuildTasks/InstallWorkload.cs @@ -0,0 +1,475 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +#nullable enable + +namespace Microsoft.Workload.Build.Tasks +{ + public class InstallWorkload : Task + { + [Required, NotNull] + public ITaskItem? WorkloadId { get; set; } + + [Required, NotNull] + public ITaskItem? ManifestPackage { get; set; } + + [Required, NotNull] + public ITaskItem? BuiltNuGetsPath { get; set; } + + [Required, NotNull] + public string? OutputDir { get; set; } + + public ITaskItem[]? ExtraNuGetSources { get; set; } + public string? Rid { get; set; } + + private readonly string _tempDir = Path.Combine(Path.GetTempPath(), "install-workload", Path.GetRandomFileName()); + private string? _packsDir; + private const string s_stampFileName = ".installed.stamp"; + + private static string? GetRid() + { + if (OperatingSystem.IsWindows()) + return Environment.Is64BitProcess ? "win-x64": "win-x86"; + else if (OperatingSystem.IsMacOS()) + return "osx-x64"; + else if (OperatingSystem.IsLinux()) + return "linux-x64"; + else + return null; + } + + public override bool Execute() + { + Utils.Logger = Log; + + if (!HasMetadata(ManifestPackage, nameof(ManifestPackage), "VersionBand") || + !HasMetadata(ManifestPackage, nameof(ManifestPackage), "Version") || + !HasMetadata(WorkloadId, nameof(WorkloadId), "Name")) + { + return false; + } + + if (!Directory.Exists(OutputDir)) + { + Log.LogError($"Cannot find OutputDir={OutputDir}"); + return false; + } + + _packsDir = Path.Combine(OutputDir, "packs"); + Rid ??= GetRid(); + if (Rid == null) + { + Log.LogError("Unsupported platform"); + return false; + } + + if (!InstallWorkloadManifest(out ManifestInformation? manifest, out string? manifestNupkgPath)) + return false; + + IEnumerable references = GetPackageReferencesForWorkload(manifest, WorkloadId.ItemSpec); + IEnumerable remaining = LayoutPacksFromBuiltNuGets(references); + if (!remaining.Any()) + return !Log.HasLoggedErrors; + + if (!InstallPacksWithNuGetRestore(remaining)) + return false; + + return !Log.HasLoggedErrors; + } + + private bool InstallWorkloadManifest([NotNullWhen(true)] out ManifestInformation? manifest, [NotNullWhen(true)] out string? manifestNupkgPath) + { + manifest = null; + manifestNupkgPath = null; + + string builtNuGetsFullPath = BuiltNuGetsPath.GetMetadata("FullPath"); + string pkgName = ManifestPackage.ItemSpec; + string pkgVersion = ManifestPackage.GetMetadata("Version"); + + var nupkgFileName = $"{pkgName}.{pkgVersion}.nupkg"; + var nupkgPath = Path.Combine(builtNuGetsFullPath, nupkgFileName); + if (!File.Exists(nupkgPath)) + { + Log.LogError($"Could not find nupkg for the manifest at {nupkgPath}"); + return false; + } + + string baseManifestDir = Path.Combine(OutputDir, "sdk-manifests"); + + string tmpManifestDir = Path.Combine(_tempDir, "manifest"); + ZipFile.ExtractToDirectory(nupkgPath, tmpManifestDir); + + var sourceManifestDirectory = Path.Combine(tmpManifestDir, "data"); + var targetManifestDirectory = Path.Combine(baseManifestDir, ManifestPackage.GetMetadata("VersionBand"), WorkloadId.GetMetadata("Name")); + if (!CopyDirectory(sourceManifestDirectory, targetManifestDirectory)) + return false; + + string jsonPath = Path.Combine(targetManifestDirectory, "WorkloadManifest.json"); + if (!File.Exists(jsonPath)) + { + Log.LogError($"Could not find WorkloadManifest.json at {jsonPath}"); + return false; + } + + manifest = JsonSerializer.Deserialize( + File.ReadAllBytes(jsonPath), + new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip + }); + + if (manifest == null) + { + Log.LogError($"Could not parse manifest from {jsonPath}."); + return false; + } + + manifestNupkgPath = nupkgPath; + return true; + } + + private IEnumerable LayoutPacksFromBuiltNuGets(IEnumerable references) + { + string builtNuGetsFullPath = BuiltNuGetsPath.GetMetadata("FullPath"); + + var allFiles = string.Join($"{Environment.NewLine} ", Directory.EnumerateFiles(builtNuGetsFullPath, "*", new EnumerationOptions { RecurseSubdirectories = true })); + Log.LogMessage(MessageImportance.Low, $"Files in {builtNuGetsFullPath}: {allFiles}"); + + List remaining = new(references.Count()); + foreach (var reference in references) + { + var nupkgFileName = $"{reference.Name}.{reference.Version}.nupkg"; + var nupkgPath = Path.Combine(builtNuGetsFullPath, nupkgFileName); + if (!File.Exists(nupkgPath)) + { + string[] found = Directory.GetFiles(builtNuGetsFullPath, nupkgFileName, new EnumerationOptions + { + MatchCasing = MatchCasing.CaseInsensitive, + RecurseSubdirectories = true + }); + + if (found.Length == 0) + { + remaining.Add(reference); + continue; + } + + nupkgPath = found[0]; + nupkgFileName = Path.GetFileName(nupkgPath); + } + + string installedPackDir = Path.Combine(_packsDir!, reference.Name, reference.Version); + string stampFilePath = Path.Combine(installedPackDir, s_stampFileName); + if (!IsFileNewer(nupkgPath, stampFilePath)) + { + Log.LogMessage(MessageImportance.Normal, $"Skipping {reference.Name}/{reference.Version} as it is already installed in {installedPackDir}.{Environment.NewLine} {nupkgPath} is older than {stampFilePath}"); + continue; + } + + if (Directory.Exists(installedPackDir)) + { + Log.LogMessage(MessageImportance.Normal, $"Deleting {installedPackDir}"); + Directory.Delete(installedPackDir, recursive: true); + } + + Log.LogMessage(MessageImportance.High, $"Extracting {nupkgPath} => {installedPackDir}"); + ZipFile.ExtractToDirectory(nupkgPath, installedPackDir); + + // Add .nupkg.sha512, so it gets picked up when resolving nugets + File.WriteAllText(Path.Combine(installedPackDir, $"{nupkgFileName}.sha512"), string.Empty); + + File.WriteAllText(stampFilePath, string.Empty); + } + + return remaining; + } + + private IEnumerable GetPackageReferencesForWorkload(ManifestInformation manifest, string workloadId) + { + var workload = manifest.Workloads[workloadId]; + var subset = workload.Packs; + if (workload.Extends.Count > 0) + { + subset = new List(subset); + //FIXME: use exceptions! + if (!ProcessWorkload(workload)) + throw new KeyNotFoundException(); + } + + List references = new(); + foreach (var item in manifest.Packs) + { + if (subset != null && !subset.Contains(item.Key)) + { + Log.LogMessage(MessageImportance.Low, $"Ignoring pack {item.Key} as it is not in the workload"); + continue; + } + + var packageName = item.Key; + if (item.Value.AliasTo is Dictionary alias) + { + alias.TryGetValue(Rid!, out packageName); + } + + if (!string.IsNullOrEmpty(packageName) && !packageName.Contains("cross", StringComparison.InvariantCultureIgnoreCase)) + references.Add(new PackageReference(packageName, item.Value.Version)); + } + + return references; + + bool ProcessWorkload(WorkloadInformation workload) + { + if (workload.Extends == null || workload.Extends.Count == 0) + return true; + + foreach (var w in workload.Extends) + { + if (!manifest.Workloads.TryGetValue(w, out WorkloadInformation? depWorkload)) + { + Log.LogError($"Could not find workload {w} needed by {workload.Description} in the manifest"); + return false; + } + //FIXME: + + if (!ProcessWorkload(depWorkload)) + return false; + + subset.AddRange(depWorkload.Packs); + } + + return true; + } + } + + private bool InstallPacksWithNuGetRestore(IEnumerable references) + { + var remaining = SkipInstalledPacks(references); + if (!remaining.Any()) + return true; + + return TryRestorePackages(remaining, out PackageReference[]? restored) && + LayoutPacksFromRestoredNuGets(restored); + + IEnumerable SkipInstalledPacks(IEnumerable candidates) + { + List needed = new List(candidates.Count()); + foreach (PackageReference pr in candidates) + { + var installedPackDir = Path.Combine(_packsDir!, pr.Name, pr.Version); + var packStampFile = Path.Combine(installedPackDir, s_stampFileName); + + if (File.Exists(packStampFile)) + { + Log.LogMessage(MessageImportance.Normal, $"Skipping {pr.Name}/{pr.Version} as it is already installed in {installedPackDir}.{Environment.NewLine} {packStampFile} exists."); + continue; + } + + needed.Add(pr); + } + + return needed; + } + + bool LayoutPacksFromRestoredNuGets(IEnumerable restored) + { + foreach (var pkgRef in restored) + { + if (pkgRef.RestoredPath == null) + { + Log.LogError($"Failed to restore {pkgRef.Name}/{pkgRef.Version}"); + return false; + } + + var source = pkgRef.RestoredPath; + var destDir = Path.Combine(_packsDir!, pkgRef.Name, pkgRef.Version); + if (!CopyDirectory(source, destDir)) + return false; + } + + return true; + } + } + + private bool TryRestorePackages(IEnumerable references, [NotNullWhen(true)]out PackageReference[]? restoredPackages) + { + if (!references.Any()) + { + restoredPackages = Array.Empty(); + return true; + } + + restoredPackages = null; + + var restoreProject = Path.Combine(_tempDir, "restore", "Restore.csproj"); + var restoreProjectDirectory = Directory.CreateDirectory(Path.GetDirectoryName(restoreProject)!); + + File.WriteAllText(Path.Combine(restoreProjectDirectory.FullName, "Directory.Build.props"), ""); + File.WriteAllText(Path.Combine(restoreProjectDirectory.FullName, "Directory.Build.targets"), ""); + + StringBuilder projectFileBuilder = new(); + projectFileBuilder.Append(@" + + + net6.0 + $(NoWarn);NU1213 + + +"); + + foreach (var reference in references) + { + string itemName = "PackageReference"; + projectFileBuilder.AppendLine($"<{itemName} Include=\"{reference.Name}\" Version=\"{reference.Version}\" />"); + } + + projectFileBuilder.Append(@" + + +"); + File.WriteAllText(restoreProject, projectFileBuilder.ToString()); + + if (ExtraNuGetSources?.Length > 0) + { + StringBuilder nugetConfigBuilder = new(); + nugetConfigBuilder.AppendLine($"{Environment.NewLine}"); + + foreach (ITaskItem source in ExtraNuGetSources) + { + string key = source.ItemSpec; + string value = source.GetMetadata("Value"); + if (string.IsNullOrEmpty(value)) + { + Log.LogError($"ExtraNuGetSource {key} is missing Value metadata"); + return false; + } + + nugetConfigBuilder.AppendLine($@""); + } + + nugetConfigBuilder.AppendLine($"{Environment.NewLine}"); + + File.WriteAllText(Path.Combine(restoreProjectDirectory.FullName, "nuget.config"), nugetConfigBuilder.ToString()); + } + + string restoreDir = Path.Combine(_tempDir, "nuget-packages"); + if (Directory.Exists(restoreDir)) + { + Log.LogMessage(MessageImportance.Low, $"Deleting {restoreDir}"); + Directory.Delete(restoreDir, recursive: true); + } + + Log.LogMessage(MessageImportance.High, $"Restoring packages: {string.Join(", ", references.Select(r => $"{r.Name}/{r.Version}"))}"); + + string args = $"restore {restoreProject} /p:RestorePackagesPath={restoreDir}"; + (int exitCode, string output) = Utils.TryRunProcess("dotnet", args, silent: false, debugMessageImportance: MessageImportance.Normal); + if (exitCode != 0) + { + Log.LogError($"Restoring packages returned exit code: {exitCode}. Output:{Environment.NewLine}{output}"); + return false; + } + + restoredPackages = references.Select(reference => + { + var expectedPath = Path.Combine(restoreDir, reference.Name.ToLower(), reference.Version); + if (Directory.Exists(expectedPath)) + { + reference = reference with { RestoredPath = expectedPath }; + File.WriteAllText(Path.Combine(reference.RestoredPath, s_stampFileName), string.Empty); + } + return reference; + }).ToArray(); + + return true; + } + + private bool CopyDirectory(string srcDir, string destDir) + { + try + { + Log.LogMessage(MessageImportance.Low, $"Copying {srcDir} to {destDir}"); + if (Directory.Exists(destDir)) + { + Log.LogMessage(MessageImportance.Normal, $"Deleting {destDir}"); + Directory.Delete(destDir, recursive: true); + } + + Directory.CreateDirectory(destDir); + Utils.DirectoryCopy(srcDir, destDir); + + return true; + } + catch (Exception ex) + { + Log.LogError($"Failed while copying {srcDir} => {destDir}: {ex.Message}"); + if (ex is IOException) + return false; + + throw; + } + } + + private bool HasMetadata(ITaskItem item, string itemName, string metadataName) + { + if (!string.IsNullOrEmpty(item.GetMetadata(metadataName))) + return true; + + Log.LogError($"{itemName} item ({item.ItemSpec}) is missing Name metadata"); + return false; + } + + private static bool IsFileNewer(string sourceFile, string stampFile) + { + if (!File.Exists(sourceFile)) + return true; + + if (!File.Exists(stampFile)) + return true; + + return File.GetLastWriteTimeUtc(sourceFile) > File.GetLastWriteTimeUtc(stampFile); + } + + private record PackageReference(string Name, string Version, string? RestoredPath=null); + + private record ManifestInformation( + object Version, + string Description, + + [property: JsonPropertyName("depends-on")] + IDictionary DependsOn, + IDictionary Workloads, + IDictionary Packs, + object Data + ); + + private record WorkloadInformation( + bool Abstract, + string Kind, + string Description, + + List Packs, + List Extends, + List Platforms + ); + + private record PackVersionInformation( + string Kind, + string Version, + [property: JsonPropertyName("alias-to")] + Dictionary AliasTo + ); + } +} diff --git a/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj b/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj index 328672b45145b..537418ef31dc6 100644 --- a/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj +++ b/src/tasks/WorkloadBuildTasks/WorkloadBuildTasks.csproj @@ -6,6 +6,8 @@ $(NoWarn),CA1050 + +