Skip to content

Commit

Permalink
CI Integration for Cross OS DAC (#39574) (#40120)
Browse files Browse the repository at this point in the history
* Transfer assets from Cross OS DAC builds into a pipeline container
* Preserve additional properties in subsets' projects
* Add packaging projects for Cross OS Dac assets
* Pipeline change to package, sign, and publish the cross OS DAC
  • Loading branch information
hoyosjs committed Jul 30, 2020
1 parent e9bf243 commit 4f703e6
Show file tree
Hide file tree
Showing 12 changed files with 288 additions and 42 deletions.
4 changes: 4 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,10 @@ indent_size = 2
[*.{props,targets,config,nuspec}]
indent_size = 2

# YAML config files
[*.{yml,yaml}]
indent_size = 2

# Shell scripts
[*.sh]
end_of_line = lf
Expand Down
18 changes: 12 additions & 6 deletions eng/Subsets.props
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,14 @@
<!-- CoreClr -->
<SubsetName Include="Clr" Description="The CoreCLR runtime, LinuxDac, CoreLib (+ native), tools and packages." />
<SubsetName Include="Clr.Runtime" Description="The CoreCLR .NET runtime." />
<SubsetName Include="LinuxDac" Condition="$([MSBuild]::IsOsPlatform(Windows))" Description="The cross-OS Windows->libc-based Linux DAC. Skipped on x86." />
<SubsetName Include="AlpineDac" Condition="$([MSBuild]::IsOsPlatform(Windows))" OnDemand="true" Description="The cross-OS Windows->musl-libc-based Linux DAC. Skipped on x86." />
<SubsetName Include="Clr.CoreLib" Description="The managed System.Private.CoreLib library for CoreCLR." />
<SubsetName Include="Clr.NativeCoreLib" Description="Run crossgen on System.Private.CoreLib library for CoreCLR." />
<SubsetName Include="Clr.Tools" Description="Managed tools that support CoreCLR development and testing." />
<SubsetName Include="Clr.Packages" Description="The projects that produce NuGet packages for the CoreCLR runtime, crossgen, and IL tools." />
<SubsetName Include="LinuxDac" Condition="$([MSBuild]::IsOsPlatform(Windows))" Description="The cross-OS Windows->libc-based Linux DAC. Skipped on x86." />
<SubsetName Include="AlpineDac" Condition="$([MSBuild]::IsOsPlatform(Windows))" OnDemand="true" Description="The cross-OS Windows->musl-libc-based Linux DAC. Skipped on x86 and arm." />
<SubsetName Include="CrossDacPack" OnDemand="true"
Description="Packaging of cross OS DAC. Requires all assets needed to be present at a folder specified by $(CrossDacArtifactsDir). See 'Microsoft.CrossOsDiag.Private.CoreCLR.proj' for details." />

<!-- Mono -->
<SubsetName Include="Mono" Description="The Mono runtime and CoreLib." />
Expand Down Expand Up @@ -136,10 +138,14 @@
<ProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" AdditionalProperties="%(AdditionalProperties);CrossDac=linux" Category="clr" />
</ItemGroup>

<ItemGroup Condition="$(_subset.Contains('+alpinedac+')) and $([MSBuild]::IsOsPlatform(Windows)) and '$(TargetArchitecture)' != 'x86'">
<ItemGroup Condition="$(_subset.Contains('+alpinedac+')) and $([MSBuild]::IsOsPlatform(Windows)) and '$(TargetArchitecture)' != 'x86' and '$(TargetArchitecture)' != 'arm'">
<ProjectToBuild Include="$(CoreClrProjectRoot)runtime.proj" AdditionalProperties="%(AdditionalProperties);CrossDac=alpine" Category="clr" />
</ItemGroup>

<ItemGroup Condition="$(_subset.Contains('+crossdacpack+'))">
<ProjectToBuild Include="$(CoreClrProjectRoot)src\.nuget\Microsoft.CrossOsDiag.Private.CoreCLR\Microsoft.CrossOsDiag.Private.CoreCLR.proj" Category="clr" />
</ItemGroup>

<ItemGroup Condition="$(_subset.Contains('+clr.nativecorelib+'))">
<ProjectToBuild Include="$(CoreClrProjectRoot)crossgen-corelib.proj" Category="clr" />
</ItemGroup>
Expand Down Expand Up @@ -256,9 +262,9 @@
<!-- Set default configurations. -->
<ItemGroup>
<ProjectToBuild Update="@(ProjectToBuild)">
<AdditionalProperties Condition="'%(ProjectToBuild.Category)' == 'clr' and '$(CoreCLRConfiguration)' != ''">Configuration=$(CoreCLRConfiguration)</AdditionalProperties>
<AdditionalProperties Condition="'%(ProjectToBuild.Category)' == 'mono' and '$(MonoConfiguration)' != ''">Configuration=$(MonoConfiguration)</AdditionalProperties>
<AdditionalProperties Condition="'%(ProjectToBuild.Category)' == 'libs' and '$(LibrariesConfiguration)' != ''">Configuration=$(LibrariesConfiguration)</AdditionalProperties>
<AdditionalProperties Condition="'%(ProjectToBuild.Category)' == 'clr' and '$(CoreCLRConfiguration)' != ''">%(AdditionalProperties);Configuration=$(CoreCLRConfiguration)</AdditionalProperties>
<AdditionalProperties Condition="'%(ProjectToBuild.Category)' == 'mono' and '$(MonoConfiguration)' != ''">%(AdditionalProperties);Configuration=$(MonoConfiguration)</AdditionalProperties>
<AdditionalProperties Condition="'%(ProjectToBuild.Category)' == 'libs' and '$(LibrariesConfiguration)' != ''">%(AdditionalProperties);Configuration=$(LibrariesConfiguration)</AdditionalProperties>
</ProjectToBuild>
</ItemGroup>

Expand Down
15 changes: 9 additions & 6 deletions eng/pipelines/coreclr/templates/build-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ jobs:
- ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
- template: /eng/pipelines/common/restore-internal-tools.yml

- ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}:
- ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}:
- script: |
du -sh $(Build.SourcesDirectory)/*
df -h
Expand All @@ -147,16 +147,12 @@ jobs:
- script: set __TestIntermediateDir=int&&$(coreClrRepoRootDir)build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(officialBuildIdArg) $(clrInterpreterBuildArg)
displayName: Build CoreCLR Runtime

- ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}:
- ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}:
- script: |
du -sh $(Build.SourcesDirectory)/*
df -h
displayName: Disk Usage after Build
- ${{ if and(eq(parameters.osGroup, 'Windows_NT'), ne(parameters.archType, 'x86')) }}:
- script: set __TestIntermediateDir=int&&$(coreClrRepoRootDir)build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -linuxdac $(officialBuildIdArg)
displayName: Build Cross OS Linux DAC for Windows

# Build CoreCLR Managed Components
- script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.tools+clr.packages $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci
displayName: Build managed product components and packages
Expand Down Expand Up @@ -197,6 +193,13 @@ jobs:
artifactName: $(buildProductArtifactName)
displayName: 'product build'

- ${{ if and(ne(parameters.osGroup, 'OSX'), ne(parameters.archType, 'x86'), ne(parameters.compilerName, 'gcc'), ne(parameters.testGroup, 'clrTools')) }}:
- template: /eng/pipelines/coreclr/templates/crossdac-build.yml
parameters:
archType: ${{ parameters.archType }}
osGroup: ${{ parameters.osGroup }}
osSubgroup: ${{ parameters.osSubgroup }}

- ${{ if and(ne(parameters.compilerName, 'gcc'), ne(parameters.testGroup, ''), ne(parameters.testGroup, 'clrTools')) }}:
# Publish test native components for consumption by test execution.
- ${{ if ne(parameters.isOfficialBuild, true) }}:
Expand Down
53 changes: 53 additions & 0 deletions eng/pipelines/coreclr/templates/crossdac-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
parameters:
archType: ''
osGroup: ''
osSubgroup: ''

steps:
# Always build the crossdac, that way we know in CI/PR if things break to build.
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- ${{ if notin(parameters.archType, 'x86') }}:
- script: set __TestIntermediateDir=int&&$(coreClrRepoRootDir)build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -linuxdac $(officialBuildIdArg)
displayName: Build Cross OS Linux DAC for Windows


# Make the assets available in a single container for the packaging job.
- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- ${{ if notin(parameters.archType, 'x86', 'arm') }}:
- script: set __TestIntermediateDir=int&&$(coreClrRepoRootDir)build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -alpinedac $(officialBuildIdArg)
displayName: Build Cross OS Linux-musl DAC for Windows

- task: CopyFiles@2
displayName: Gather CrossDac Artifacts (Linux)
inputs:
SourceFolder: $(buildLinuxDacRootFolderPath)
Contents: |
**/*
!**/sharedFramework/**/*
TargetFolder: $(buildLinuxDacStagingPath)

- ${{ if ne(parameters.archType, 'arm') }}:
- task: CopyFiles@2
displayName: Gather CrossDac Artifacts (Linux_musl)
inputs:
SourceFolder: $(buildMuslDacRootFolderPath)
Contents: |
**/*
!**/sharedFramework/**/*
TargetFolder: '$(buildMuslDacStagingPath)'

- ${{ if eq(parameters.osGroup, 'Linux') }}:
- task: CopyFiles@2
displayName: Gather runtime for CrossDac
inputs:
SourceFolder: $(coreClrProductRootFolderPath)
Contents: libcoreclr.so
TargetFolder: '$(crossDacArtifactPath)/${{ parameters.osGroup }}${{ parameters.osSubgroup }}.$(archType).$(buildConfigUpper)/$(crossDacHostArch)'

- task: PublishBuildArtifacts@1
displayName: Publish runtime for CrossDac
inputs:
pathtoPublish: $(crossDacArtifactPath)
PublishLocation: Container
artifactName: $(buildCrossDacArtifactName)
74 changes: 74 additions & 0 deletions eng/pipelines/coreclr/templates/crossdac-pack.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
parameters:
archType: ''
buildConfig: ''
container: ''
crossDacPlatforms: {}
isOfficialBuild: false
osGroup: ''
osSubgroup: ''
platform: ''
pool: ''
runtimeVariant: ''
stagedBuild: false
testGroup: ''
timeoutInMinutes: ''
variables: {}

jobs:
- template: xplat-pipeline-job.yml
parameters:
archType: ${{ parameters.archType }}
buildConfig: ${{ parameters.buildConfig }}
container: ${{ parameters.container }}
condition: ${{ parameters.isOfficialBuild }}
helixType: 'build/product/'
osGroup: ${{ parameters.osGroup }}
osSubgroup: ${{ parameters.osSubgroup }}
pool: ${{ parameters.pool }}
runtimeVariant: ${{ parameters.runtimeVariant }}
stagedBuild: ${{ parameters.stagedBuild }}
timeoutInMinutes: ${{ parameters.timeoutInMinutes }}

name: crossdacpack
displayName: CrossDac Packaging

variables:
- name: officialBuildIdArg
value: ''
- name: crossDacArgs
value: ''
- ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
- name: officialBuildIdArg
value: '/p:OfficialBuildId=$(Build.BuildNumber)'
- name: crossDacArgs
value: '/p:CrossDacArtifactsDir=$(crossDacArtifactPath)/$(buildCrossDacArtifactName)'
- ${{ parameters.variables }}

dependsOn:
- ${{ if ne(parameters.crossDacPlatforms, '') }}:
- ${{ each platform in parameters.crossDacPlatforms }}:
- ${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_product_build_${{ platform }}_${{ parameters.buildConfig }}

steps:
- task: DownloadBuildArtifacts@0
displayName: Download CrossDac artifacts
inputs:
artifactName: $(buildCrossDacArtifactName)
downloadPath: $(crossDacArtifactPath)

- script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset crossdacpack -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) $(crossDacArgs) -ci
displayName: Build crossdac packaging

# Save packages using the prepare-signed-artifacts format.
- template: /eng/pipelines/common/upload-unsigned-artifacts-step.yml
parameters:
name: ${{ parameters.platform }}

# Upload to artifacts to be signed
- task: PublishPipelineArtifact@1
displayName: Publish Logs
inputs:
targetPath: $(Build.SourcesDirectory)/artifacts/log
artifactName: 'CrossDacPackagingLogs'
continueOnError: true
condition: always()
27 changes: 26 additions & 1 deletion eng/pipelines/coreclr/templates/xplat-pipeline-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,39 @@ jobs:

- name: binTestsPath
value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr'

# Build product defines what we are trying to build, either coreclr or mono
- name: buildProductArtifactName
value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'

- name: buildProductRootFolderPath
value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)'

- name: buildCrossDacArtifactName
value: CoreCLRCrossDacArtifacts

- name: crossDacArtifactPath
value: $(Build.SourcesDirectory)/artifacts/$(buildCrossDacArtifactName)

- name: buildMuslDacRootFolderPath
value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/alpine.$(archType).$(buildConfigUpper)'

- name: buildMuslDacStagingPath
value: '$(crossDacArtifactPath)/Linux_musl.$(archType).$(buildConfigUpper)'

- name: buildLinuxDacRootFolderPath
value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/Linux.$(archType).$(buildConfigUpper)'

- name: buildLinuxDacStagingPath
value: '$(crossDacArtifactPath)/Linux.$(archType).$(buildConfigUpper)'

- name: crossDacHostArch
value: x64

- ${{ if eq(parameters.archType, 'arm') }}:
- name: crossDacHostArch
value: x86

# We need this because both mono and coreclr build currently depends on CoreClr
- name: coreClrProductArtifactName
value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
Expand Down
17 changes: 17 additions & 0 deletions eng/pipelines/runtime-official.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,23 @@ stages:
jobParameters:
isOfficialBuild: ${{ variables.isOfficialBuild }}

- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/coreclr/templates/crossdac-pack.yml
buildConfig: release
platforms:
- Linux_musl_x64
jobParameters:
isOfficialBuild: ${{ variables.isOfficialBuild }}
crossDacPlatforms:
- Linux_x64
- Linux_arm
- Linux_arm64
- Linux_musl_x64
- Linux_musl_arm64
- Windows_NT_x64
- Windows_NT_arm
- Windows_NT_arm64
#
# Build Mono runtime packs
#
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props))" />
<PropertyGroup>
<CreatePackedPackage>false</CreatePackedPackage>
</PropertyGroup>

<ItemGroup>
<!-- Intentional rather than globbing here, even if very manual, to make sure we publish the Cross OS Dac -->
<NativeBinary Include="$(CrossDacBinRoot)/mscordaccore.dll" />
<NativeBinary Include="$(CrossDacBinRoot)/mscordbi.dll" />
<!-- The infrastructure, and this package is only for indexing. Avoid creating a transport package and just let this one have the symbols itself -->
<File Include="@(NativeBinary -> '%(RootDir)%(Directory)/%(Filename).pdb')">
<TargetPath>runtimes/$(PackageTargetRuntime)/native</TargetPath>
</File>

<NativeBinary Include="$(CrossDacBinRoot)/libcoreclr.so" />
</ItemGroup>

<Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.targets))" />
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<Project>
<Import Sdk="Microsoft.Build.Traversal" Project="Sdk.props" />

<PropertyGroup>
<BuildIdentityPackage>false</BuildIdentityPackage>
<SupportedRids>linux-x64;linux-musl-x64;linux-arm64;linux-musl-arm64;linux-arm;</SupportedRids>
</PropertyGroup>

<Target Name="VerifyCrossDacProperties" BeforeTargets="Build">
<Error Condition="'$(CrossDacArtifactsDir)' == '' or !Exists('$(CrossDacArtifactsDir)')"
Text="Packaging the Cross OS DAC requires property CrossDacArtifactsDir to to a path containing all cross OS DAC assets needed." />
</Target>

<Target Name="FilterSupportedCrossOsDacPackages" BeforeTargets="Build">
<ItemGroup>
<_projectsToBuild Include="@(Project)" Condition="$(SupportedRids.Contains('%(Project.PackageTargetRuntime)'))">
<AdditionalProperties Condition="'%(Project.PackageTargetRuntime)' == 'linux-x64'">%(AdditionalProperties);CrossDacBinRoot=$(CrossDacArtifactsDir)/Linux.x64.$(Configuration)/x64</AdditionalProperties>
<AdditionalProperties Condition="'%(Project.PackageTargetRuntime)' == 'linux-musl-x64'">%(AdditionalProperties);CrossDacBinRoot=$(CrossDacArtifactsDir)/Linux_musl.x64.$(Configuration)/x64</AdditionalProperties>
<AdditionalProperties Condition="'%(Project.PackageTargetRuntime)' == 'linux-arm64'">%(AdditionalProperties);CrossDacBinRoot=$(CrossDacArtifactsDir)/Linux.arm64.$(Configuration)/x64</AdditionalProperties>
<AdditionalProperties Condition="'%(Project.PackageTargetRuntime)' == 'linux-musl-arm64'">%(AdditionalProperties);CrossDacBinRoot=$(CrossDacArtifactsDir)/Linux_musl.arm64.$(Configuration)/x64</AdditionalProperties>
<AdditionalProperties Condition="'%(Project.PackageTargetRuntime)' == 'linux-arm'">%(AdditionalProperties);CrossDacBinRoot=$(CrossDacArtifactsDir)/Linux.arm.$(Configuration)/x86</AdditionalProperties>
</_projectsToBuild>
</ItemGroup>

<ItemGroup>
<ProjectReference Include="@(_projectsToBuild)" />
</ItemGroup>
</Target>

<Import Project="$([MSBuild]::GetPathOfFileAbove(versioning.targets))" />
<Import Sdk="Microsoft.Build.Traversal" Project="Sdk.targets" />

<!-- Ordering matters! Overwriting the Pack target which should just invoke Build. -->
<Target Name="Pack" DependsOnTargets="Build" />
</Project>
30 changes: 1 addition & 29 deletions src/coreclr/src/.nuget/coreclr-packages.proj
Original file line number Diff line number Diff line change
Expand Up @@ -13,36 +13,8 @@
<ProjectReference Include="Microsoft.NETCore.ILDAsm\Microsoft.NETCore.ILDAsm.proj" />
</ItemGroup>

<!-- Generate a version.txt file we include in our packages
The InitializeSourceControlInformationFromSourceControlManager is part of Microsoft.Build.Tasks.Git
and is responsible for setting SourceRevisionId -->
<Target Name="GenerateVersionInfoFileForPackages"
BeforeTargets="Build"
DependsOnTargets="InitializeSourceControlInformationFromSourceControlManager">

<Error Condition="'$(SourceRevisionId)' == ''" Text="SourceRevisionId is not set, which means the SourceLink targets are not included in the build. Those are needed to produce a correct sha for our build outputs." />

<MakeDir Directories="$([System.IO.Path]::GetDirectoryName($(VersionTxtFile)))" />
<WriteLinesToFile
File="$(VersionTxtFile)"
Lines="$(SourceRevisionId)"
Overwrite="true" />
</Target>

<!-- Update the project references with additional properties calculated during the execution phase.
_InitializeAssemblyVersion is provided by Arcade. It sets the AssemblyVersion and FileVersion properties.
We depend on this private Arcade target instead of the SDK-defined GetAssemblyVersion since the packaging
build does not use the .NET SDK -->
<Target Name="UpdateAdditionalProperties"
BeforeTargets="Build"
DependsOnTargets="_InitializeAssemblyVersion">
<ItemGroup>
<!-- Pass the FileVersion calculated by _InitializeAssemblyVersion to referenced projects -->
<ProjectReference Update="@(ProjectReference)"
AdditionalProperties="%(ProjectReference.AdditionalProperties);FileVersion=$(FileVersion)" />
</ItemGroup>
</Target>

<Import Project="versioning.targets" />
<Import Sdk="Microsoft.Build.Traversal" Project="Sdk.targets" />

<!-- Ordering matters! Overwriting the Pack target which should just invoke Build. -->
Expand Down
5 changes: 5 additions & 0 deletions src/coreclr/src/.nuget/descriptions.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@
"Description": "When using NuGet 3.x this package requires at least version {0}.",
"CommonTypes": [ ]
},
{
"Name": "Microsoft.CrossOsDiag.Private.CoreCLR",
"Description": "Private transport package for .NET Core cross OS diagnostic tooling.",
"CommonTypes": [ ]
},
{
"Name": "Microsoft.NETCore.ILAsm",
"Description": "The .NET IL Assembler.",
Expand Down
Loading

0 comments on commit 4f703e6

Please sign in to comment.