diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index cdda8c99dc519..4b06818b1408c 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -237,10 +237,11 @@ jobs: - task: CodeQL3000Init@0 displayName: Initialize CodeQL (manually-injected) - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci $(_archParameter) $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) - displayName: Build product - ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: - continueOnError: ${{ parameters.shouldContinueOnError }} + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: ${{ parameters.buildArgs }} + useContinueOnErrorDuringBuild: ${{ parameters.useContinueOnErrorDuringBuild }} + shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - ${{ if eq(parameters.isManualCodeQLBuild, true) }}: - task: CodeQL3000Finalize@0 diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml new file mode 100644 index 0000000000000..7f38a9fd11840 --- /dev/null +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -0,0 +1,12 @@ +parameters: + buildArgs: '' + useContinueOnErrorDuringBuild: false + shouldContinueOnError: false + archParameter: $(_archParameter) + displayName: Build product + +steps: + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) + displayName: ${{ parameters.displayName }} + ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: + continueOnError: ${{ parameters.shouldContinueOnError }} diff --git a/eng/pipelines/common/templates/runtimes/xplat-job.yml b/eng/pipelines/common/templates/runtimes/xplat-job.yml index f4ac7e8295712..23e74c70e57ac 100644 --- a/eng/pipelines/common/templates/runtimes/xplat-job.yml +++ b/eng/pipelines/common/templates/runtimes/xplat-job.yml @@ -77,15 +77,6 @@ jobs: - name: buildConfig value: ${{ parameters.buildConfig }} - - name: archType - value: ${{ parameters.archType }} - - - name: osGroup - value: ${{ parameters.osGroup }} - - - name: osSubgroup - value: ${{ parameters.osSubgroup }} - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - name: _HelixSource value: official/dotnet/runtime/$(Build.SourceBranch) diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index eb19570aeecac..675a267920186 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -45,6 +45,15 @@ jobs: - name: _BuildConfig value: $(buildConfigUpper) + + - name: archType + value: ${{ parameters.archType }} + + - name: osGroup + value: ${{ parameters.osGroup }} + + - name: osSubgroup + value: ${{ parameters.osSubgroup }} - name: _runSmokeTestsOnlyArg value: '/p:RunSmokeTestsOnly=$(isRunSmokeTestsOnly)' diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index e995e626a5699..ebaeae5e12404 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -231,27 +231,6 @@ jobs: artifactName: $(buildProductArtifactName) displayName: 'product build' - - ${{ if and(in(parameters.osGroup, 'windows', 'linux'), ne(parameters.archType, 'x86')) }}: - - template: /eng/pipelines/coreclr/templates/crossdac-build.yml - parameters: - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - isOfficialBuild: ${{ parameters.signBinaries }} - ${{ if eq(parameters.archType, 'arm') }}: - hostArchType: x86 - ${{ else }}: - hostArchType: x64 - - - ${{ if and(in(parameters.osGroup, 'windows'), eq(parameters.archType, 'x86')) }}: - - template: /eng/pipelines/coreclr/templates/crossdac-build.yml - parameters: - archType: arm - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - isOfficialBuild: ${{ parameters.signBinaries }} - hostArchType: x86 - - ${{ if and(ne(parameters.testGroup, ''), ne(parameters.disableClrTest, true)) }}: # Publish test native components for consumption by test execution. - ${{ if ne(parameters.isOfficialBuild, true) }}: diff --git a/eng/pipelines/coreclr/templates/crossdac-build.yml b/eng/pipelines/coreclr/templates/crossdac-build.yml deleted file mode 100644 index a8accb9d2e3d2..0000000000000 --- a/eng/pipelines/coreclr/templates/crossdac-build.yml +++ /dev/null @@ -1,80 +0,0 @@ -parameters: - archType: '' - isOfficialBuild: false - osGroup: '' - osSubgroup: '' - hostArchType: '' - -steps: - # Always build the crossdac, that way we know in CI/PR if things break to build. - - ${{ if and(eq(parameters.osGroup, 'windows'), notin(parameters.archType, 'x86')) }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) ${{ parameters.archType }} -hostarch ${{ parameters.hostArchType }} -ci -os linux -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" -ninja $(officialBuildIdArg) -component crosscomponents - displayName: Build Cross OS Linux DAC for Windows - - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) ${{ parameters.archType }} -hostarch ${{ parameters.hostArchType }} -ci -os alpine -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" -ninja $(officialBuildIdArg) -component crosscomponents - displayName: Build Cross OS Linux-musl DAC for Windows - - - powershell: | - function CopyAndVerifyCrossOsAssets { - [CmdletBinding()] - param ( - [Parameter(Mandatory)][string]$crossDacDir, - [Parameter(Mandatory)][string]$targetDir - ) - - $crossDacDir = Join-Path $crossDacDir -ChildPath '${{ parameters.hostArchType }}' - - $availableFiles = ls -File $crossDacDir - - Write-Host "Probed for files in ""$crossDacDir"", found:" - $availableFiles | fl - - if (-not ("mscordaccore.dll" -in $availableFiles.Name ` - -and "mscordaccore.pdb" -in $availableFiles.Name ` - -and "mscordbi.dll" -in $availableFiles.Name ` - -and "mscordbi.pdb" -in $availableFiles.Name` - )) - { - Write-Error "Couldn't find one of the expected crossdac files." - } - - New-Item $targetDir -ItemType 'Directory' -Force -ea 0 - $availableFiles | %{ cp $_.FullName $targetDir -v } - } - - $buildMuslDacRootFolderPath = "$(Build.SourcesDirectory)/artifacts/bin/coreclr/alpine.${{ parameters.archType }}.$(buildConfigUpper)" - $buildMuslStagingPath = "$(crossDacArtifactPath)/Linux_musl.${{ parameters.archType }}.$(buildConfigUpper)/${{ parameters.hostArchType }}" - - $buildLinuxDacRootFolderPath = "$(Build.SourcesDirectory)/artifacts/bin/coreclr/Linux.${{ parameters.archType }}.$(buildConfigUpper)" - $buildLinuxDacStagingPath = "$(crossDacArtifactPath)/Linux.${{ parameters.archType }}.$(buildConfigUpper)/${{ parameters.hostArchType }}" - - - CopyAndVerifyCrossOsAssets -CrossDacDir $buildMuslDacRootFolderPath -TargetDir $buildMuslStagingPath - CopyAndVerifyCrossOsAssets -CrossDacDir $buildLinuxDacRootFolderPath -TargetDir $buildLinuxDacStagingPath - - Write-Host "Final directory contents:" - ls -R $(crossDacArtifactPath) - - displayName: Gather CrossDac Artifacts - - - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml - parameters: - basePath: $(crossDacArtifactPath) - isOfficialBuild: ${{ parameters.isOfficialBuild }} - timeoutInMinutes: 30 - - - ${{ if eq(parameters.osGroup, 'linux') }}: - - task: CopyFiles@2 - displayName: Gather runtime for CrossDac - inputs: - SourceFolder: $(coreClrProductRootFolderPath) - Contents: libcoreclr.so - TargetFolder: '$(crossDacArtifactPath)/${{ parameters.osGroup }}${{ parameters.osSubgroup }}.$(archType).$(buildConfigUpper)/${{ parameters.hostArchType }}' - - # Make the assets available in a single container for the packaging job. - - task: PublishBuildArtifacts@1 - displayName: Publish runtime for CrossDac - inputs: - pathtoPublish: $(crossDacArtifactPath) - PublishLocation: Container - artifactName: $(buildCrossDacArtifactName) diff --git a/eng/pipelines/coreclr/templates/crossdac-hostarch.yml b/eng/pipelines/coreclr/templates/crossdac-hostarch.yml new file mode 100644 index 0000000000000..e100217a48151 --- /dev/null +++ b/eng/pipelines/coreclr/templates/crossdac-hostarch.yml @@ -0,0 +1,9 @@ +parameters: + archType: '' + +variables: + - name: crossDacHostArch + value: x64 + - ${{ if eq(parameters.archType, 'arm') }}: + - name: crossDacHostArch + value: x86 \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/crossdac-pack.yml b/eng/pipelines/coreclr/templates/crossdac-pack.yml deleted file mode 100644 index 520eb65fb337e..0000000000000 --- a/eng/pipelines/coreclr/templates/crossdac-pack.yml +++ /dev/null @@ -1,75 +0,0 @@ -parameters: - archType: '' - buildConfig: '' - container: '' - crossDacPlatforms: {} - dependOnEvaluatePaths: false - isOfficialBuild: false - osGroup: '' - osSubgroup: '' - platform: '' - pool: '' - runtimeVariant: '' - testGroup: '' - timeoutInMinutes: '' - variables: {} - -jobs: -- template: xplat-pipeline-job.yml - parameters: - archType: ${{ parameters.archType }} - buildConfig: ${{ parameters.buildConfig }} - container: ${{ parameters.container }} - condition: ${{ parameters.isOfficialBuild }} - helixType: 'build/product/' - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - pool: ${{ parameters.pool }} - runtimeVariant: ${{ parameters.runtimeVariant }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - - name: crossdacpack - displayName: CrossDac Packaging - - variables: - - name: officialBuildIdArg - value: '' - - name: crossDacArgs - value: '/p:CrossDacArtifactsDir=$(crossDacArtifactPath)/$(buildCrossDacArtifactName)' - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - - name: officialBuildIdArg - value: '/p:OfficialBuildId=$(Build.BuildNumber)' - - name: SignType - value: $[ coalesce(variables.OfficialSignType, 'real') ] - - ${{ parameters.variables }} - - dependsOn: - - ${{ if ne(parameters.crossDacPlatforms, '') }}: - - ${{ each platform in parameters.crossDacPlatforms }}: - - ${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_product_build_${{ platform }}_${{ parameters.buildConfig }} - - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download CrossDac artifacts - inputs: - artifactName: $(buildCrossDacArtifactName) - downloadPath: $(crossDacArtifactPath) - checkDownloadedFiles: true - - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset crossdacpack -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) $(crossDacArgs) -ci - displayName: Build crossdac packaging - - # Save packages using the prepare-signed-artifacts format. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: ${{ parameters.platform }} - - # Upload to artifacts to be signed - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'CrossDacPackagingLogs_Attempt$(System.JobAttempt)' - continueOnError: true - condition: always() diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 162f5e9f0ed1c..b6c57be0c7fba 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -69,12 +69,6 @@ jobs: - name: buildProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' - - name: buildCrossDacArtifactName - value: CoreCLRCrossDacArtifacts - - - name: crossDacArtifactPath - value: $(Build.SourcesDirectory)/artifacts/$(buildCrossDacArtifactName) - # We need this because both mono and coreclr build currently depends on CoreClr - name: coreClrProductArtifactName value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index 1c45b437c8013..43f19c69ccacf 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -339,32 +339,9 @@ jobs: df -h displayName: Disk Usage before Build - # Build the default subset non-MacOS platforms - - ${{ if ne(parameters.osGroup, 'osx') }}: - - script: $(BaseJobBuildCommand) - displayName: Build - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} - - # Build corehost, sign and add entitlements to MacOS binaries - - ${{ if eq(parameters.osGroup, 'osx') }}: - - script: $(BaseJobBuildCommand) -subset host.native - displayName: Build CoreHost - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/pipelines/common/macos-sign-with-entitlements.yml - parameters: - filesToSign: - - name: dotnet - path: $(Build.SourcesDirectory)/artifacts/bin/osx-${{ parameters.archType }}.$(_BuildConfig)/corehost - entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist - - name: apphost - path: $(Build.SourcesDirectory)/artifacts/bin/osx-${{ parameters.archType }}.$(_BuildConfig)/corehost - entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist - - - script: $(BaseJobBuildCommand) -subset host.pkg+host.tools+host.tests+packs - displayName: Build and Package - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} + - script: $(BaseJobBuildCommand) + displayName: Build + continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log diff --git a/eng/pipelines/installer/jobs/steps/build-linux-package.yml b/eng/pipelines/installer/jobs/steps/build-linux-package.yml index a5645af3d9ccd..102eab770c278 100644 --- a/eng/pipelines/installer/jobs/steps/build-linux-package.yml +++ b/eng/pipelines/installer/jobs/steps/build-linux-package.yml @@ -4,6 +4,7 @@ parameters: packageStepDescription: null packagingArgs: '' subsetArg: '' + condition: succeeded() steps: ## Run NuGet Authentication for each of the side containers @@ -20,6 +21,7 @@ steps: /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.packageType }} target: ${{ parameters.target }} + condition: ${{ parameters.condition }} # Broken symbolic links break the SBOM processing # We make some symlinks during the installer generation process, # but they aren't always valid on disk afterwards. Some of our tooling, diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 3138f18ebd734..6e6781bb134d5 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -58,47 +58,246 @@ extends: sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os linux -ci /p:SkipLibrariesNativeRuntimePackages=true # - # Build CoreCLR + # Build CoreCLR runtime packs + # Windows x64/arm64 + # Sign diagnostic files after native build # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x64 + - windows_arm64 + jobParameters: + buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + nameSuffix: CoreCLR + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml + parameters: + basePath: $(Build.SourcesDirectory)/artifacts/bin/coreclr + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 30 + # Now that we've signed the diagnostic files, do the rest of the build. + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+libs+host+packs -c $(_BuildConfig) + displayName: Build managed CoreCLR components, all libraries, hosts, and packs + + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + + + # + # Build CoreCLR runtime packs + # Windows x86 + # No NativeAOT as NativeAOT is not supported on x86 + # Sign diagnostic files after native build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x86 + jobParameters: + buildArgs: -s clr.runtime+clr.alljits -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + nameSuffix: CoreCLR + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml + parameters: + basePath: $(Build.SourcesDirectory)/artifacts/bin/coreclr + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 30 + # Now that we've signed the diagnostic files, do the rest of the build. + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.corelib+clr.nativecorelib+clr.tools+clr.packages+libs+host+packs -c $(_BuildConfig) + displayName: Build managed CoreCLR components, all libraries, hosts, and packs + + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + # + # Build CoreCLR runtime packs + # Mac x64/arm64 + # Sign and entitle createdump and corerun after native build. + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - osx_arm64 - osx_x64 + jobParameters: + buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime+host.native -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + nameSuffix: CoreCLR + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 120 + postBuildSteps: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/pipelines/common/macos-sign-with-entitlements.yml + parameters: + filesToSign: + - name: createdump + path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/createdump-entitlements.plist + - name: corerun + path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist + - name: dotnet + path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist + - name: apphost + path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist + + - task: CopyFiles@2 + displayName: 'Copy signed createdump to sharedFramework' + inputs: + contents: createdump + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + targetFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig)/sharedFramework + overWrite: true + + # Now that we've entitled and signed createdump, we can build the rest. + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+libs+host.tools+host.pkg+packs -c $(_BuildConfig) + displayName: Build managed CoreCLR and host components, all libraries, and packs + + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + + # + # Build CoreCLR runtime packs + # Linux and Linux_musl + # CoreCLR runtime for CrossDac packaging + # Create Linux installers + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: - linux_x64 - linux_arm - linux_arm64 - linux_musl_x64 - linux_musl_arm - linux_musl_arm64 - - windows_x86 - - windows_x64 - - windows_arm64 jobParameters: + buildArgs: -s clr.runtime+clr.alljits+clr.corelib+clr.nativecorelib+clr.tools+clr.aot+clr.packages+libs+host+packs -c $(_BuildConfig) + nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} - signBinaries: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 + postBuildSteps: + # Upload libcoreclr.so for CrossDac packaging + - task: CopyFiles@2 + displayName: Gather runtime for CrossDac + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + Contents: libcoreclr.so + TargetFolder: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts/$(osGroup)$(osSubgroup).$(archType).$(_BuildConfig)/$(crossDacHostArch) + - task: PublishBuildArtifacts@1 + displayName: Publish runtime for CrossDac + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts + PublishLocation: Container + artifactName: CoreCLRCrossDacArtifacts + # Create RPMs and DEBs + - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml + parameters: + packageType: deb + target: debpkg + packageStepDescription: Runtime Deps, Runtime, Framework Packs Deb installers + subsetArg: -s packs.installers + packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true + condition: and(succeeded(), eq(variables.osSubgroup, ''), eq(variables.archType, 'x64')) + - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml + parameters: + packageType: rpm + target: rpmpkg + packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers + subsetArg: -s packs.installers + packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true + condition: and(succeeded(), eq(variables.osSubgroup, ''), in(variables.archType, 'x64', 'arm64')) + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/crossdac-hostarch.yml + + # + # Build and Pack CrossDac + # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/crossdac-pack.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - windows_x64 jobParameters: + buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) + nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 - crossDacPlatforms: - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - linux_musl_arm - - linux_musl_arm64 - - windows_x64 - - windows_arm64 + preBuildSteps: + - task: DownloadBuildArtifacts@0 + displayName: Download Runtimes for CrossDac packaging + inputs: + artifactName: $(crossDacArtifactsContainer) + downloadPath: $(crossDacArtifactsBasePath) + checkDownloadedFiles: true + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s linuxdac+alpinedac -c $(_BuildConfig) + archParameter: -arch x64,x86,arm,arm64 + - task: CopyFiles@2 + displayName: Gather CrossDacs + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + Contents: | + ** + !**\sharedFramework\** + TargetFolder: $(crossDacArtifactsPath) + - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml + parameters: + basePath: $(crossDacArtifactsPath) + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 30 + postBuildSteps: + # Save packages using the prepare-signed-artifacts format. + # CrossDac packages are expected to be in the windows_x64 folder. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: windows_x64 + dependsOn: + - build_linux_x64_release_CoreCLR + - build_linux_arm_release_CoreCLR + - build_linux_arm64_release_CoreCLR + - build_linux_musl_x64_release_CoreCLR + - build_linux_musl_arm_release_CoreCLR + - build_linux_musl_arm64_release_CoreCLR + variables: + - name: crossDacArtifactsContainer + value: CoreCLRCrossDacArtifacts + - name: crossDacArtifactsBasePath + value: $(Build.StagingDirectory)/CrossDac + - name: crossDacArtifactsPath + value: $(crossDacArtifactsBasePath)/$(crossDacArtifactsContainer) # # Build NativeAOT runtime packs @@ -122,7 +321,7 @@ extends: - linux_bionic_x64 jobParameters: buildArgs: -s clr.nativeaotlibs+clr.nativeaotruntime+libs+packs -c $(_BuildConfig) /p:BuildNativeAOTRuntimePack=true /p:SkipLibrariesNativeRuntimePackages=true - nameSuffix: AllSubsets_NativeAOT + nameSuffix: NativeAOT isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -165,7 +364,7 @@ extends: # - windows_arm64 jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:BuildMonoAOTCrossCompiler=false - nameSuffix: AllSubsets_Mono + nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -182,7 +381,7 @@ extends: - wasi_wasm jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) - nameSuffix: AllSubsets_Mono + nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -198,7 +397,7 @@ extends: - browser_wasm jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoWasmBuildVariant=multithread /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) - nameSuffix: AllSubsets_Mono_multithread + nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread postBuildSteps: @@ -329,7 +528,7 @@ extends: jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false - nameSuffix: AllSubsets_Mono_LLVMJIT + nameSuffix: Mono_LLVMJIT runtimeVariant: LLVMJIT isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: @@ -343,7 +542,7 @@ extends: jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true - nameSuffix: AllSubsets_Mono_LLVMAOT + nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: @@ -352,49 +551,26 @@ extends: name: MonoRuntimePacks # - # Build libraries using live CoreLib from CoreCLR + # Build libraries AllConfigurations for packages # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: Release platforms: - - osx_arm64 - - osx_x64 - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - linux_musl_arm - - linux_musl_arm64 - - windows_x86 - windows_x64 - - windows_arm64 jobParameters: + buildArgs: -s tools+libs -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + nameSuffix: Libraries_AllConfigurations isOfficialBuild: ${{ variables.isOfficialBuild }} - liveRuntimeBuildConfig: release - # Official builds don't run tests, locally or on Helix - runTests: false - useHelix: false - + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: Libraries_AllConfigurations + timeoutInMinutes: 95 # - # Build libraries AllConfigurations for packages + # Build SourceBuild packages # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - windows_x64 - jobParameters: - framework: allConfigurations - isOfficialBuild: ${{ variables.isOfficialBuild }} - isOfficialAllConfigurations: true - liveRuntimeBuildConfig: release - # Official builds don't run tests, locally or on Helix - runTests: false - useHelix: false - - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml @@ -410,30 +586,6 @@ extends: name: SourceBuildPackages timeoutInMinutes: 95 - # - # Installer Build - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/installer/jobs/build-job.yml - buildConfig: Release - jobParameters: - liveRuntimeBuildConfig: release - liveLibrariesBuildConfig: Release - isOfficialBuild: ${{ variables.isOfficialBuild }} - platforms: - - osx_arm64 - - osx_x64 - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - linux_musl_arm - - linux_musl_arm64 - - windows_x86 - - windows_x64 - - windows_arm64 - # # Build PGO Instrumented CoreCLR Release # @@ -470,24 +622,24 @@ extends: isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 dependsOn: - - Build_android_arm_release_AllSubsets_Mono - - Build_android_arm64_release_AllSubsets_Mono - - Build_android_x86_release_AllSubsets_Mono - - Build_android_x64_release_AllSubsets_Mono - - Build_browser_wasm_Linux_release_AllSubsets_Mono - - Build_wasi_wasm_linux_release_AllSubsets_Mono - - Build_ios_arm64_release_AllSubsets_Mono - - Build_iossimulator_x64_release_AllSubsets_Mono - - Build_iossimulator_arm64_release_AllSubsets_Mono - - Build_maccatalyst_arm64_release_AllSubsets_Mono - - Build_maccatalyst_x64_release_AllSubsets_Mono - - Build_tvos_arm64_release_AllSubsets_Mono - - Build_tvossimulator_arm64_release_AllSubsets_Mono - - Build_tvossimulator_x64_release_AllSubsets_Mono + - Build_android_arm_release_Mono + - Build_android_arm64_release_Mono + - Build_android_x86_release_Mono + - Build_android_x64_release_Mono + - Build_browser_wasm_Linux_release_Mono + - Build_wasi_wasm_linux_release_Mono + - Build_ios_arm64_release_Mono + - Build_iossimulator_x64_release_Mono + - Build_iossimulator_arm64_release_Mono + - Build_maccatalyst_arm64_release_Mono + - Build_maccatalyst_x64_release_Mono + - Build_tvos_arm64_release_Mono + - Build_tvossimulator_arm64_release_Mono + - Build_tvossimulator_x64_release_Mono - Build_windows_x64_release_CrossAOT_Mono - - installer_coreclr__windows_x64_Release_ - - installer_coreclr__windows_x86_Release_ - - installer_coreclr__windows_arm64_Release_ + - Build_windows_x64_release_CoreCLR + - Build_windows_x86_release_CoreCLR + - Build_windows_arm64_release_CoreCLR - ${{ if eq(variables.isOfficialBuild, true) }}: - template: /eng/pipelines/official/stages/publish.yml diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index c83a76993e60d..767b3dc9ba424 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -400,6 +400,27 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_tools_illink.containsChange'], true), eq(variables['isRollingBuild'], true)) + # + # Build CrossDacs + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x64 + variables: + - name: _archParameter + value: -arch x64,x86,arm,arm64 + jobParameters: + buildArgs: -s linuxdac+alpinedac -c Checked,$(_BuildConfig) + nameSuffix: CrossDac + isOfficialBuild: false + timeoutInMinutes: 60 + postBuildSteps: + - publish: $(Build.SourcesDirectory)/artifacts/bin/coreclr + displayName: Publish CrossDacs for diagnostics + artifact: CoreCLRCrossDacArtifacts # Build Mono AOT offset headers once, for consumption elsewhere # Only when mono changed @@ -415,7 +436,7 @@ extends: - ios_arm64 - maccatalyst_x64 jobParameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: false # needed by crossaot condition: >- or( diff --git a/src/coreclr/build-runtime.cmd b/src/coreclr/build-runtime.cmd index ec6887c78d84c..d85e6c95909f0 100644 --- a/src/coreclr/build-runtime.cmd +++ b/src/coreclr/build-runtime.cmd @@ -211,8 +211,13 @@ if NOT "%__BuildType%"=="Release" ( set __PgoOptimize=0 ) -set "__BinDir=%__RootBinDir%\bin\coreclr\%__TargetOS%.%__TargetArch%.%__BuildType%" -set "__IntermediatesDir=%__RootBinDir%\obj\coreclr\%__TargetOS%.%__TargetArch%.%__BuildType%" +set __TargetOSDirName=%__TargetOS% +if "%__TargetOS%"=="alpine" ( + set __TargetOSDirName=linux_musl +) + +set "__BinDir=%__RootBinDir%\bin\coreclr\%__TargetOSDirName%.%__TargetArch%.%__BuildType%" +set "__IntermediatesDir=%__RootBinDir%\obj\coreclr\%__TargetOSDirName%.%__TargetArch%.%__BuildType%" set "__LogsDir=%__RootBinDir%\log\!__BuildType!" set "__MsbuildDebugLogsDir=%__LogsDir%\MsbuildDebugLogs" set "__ArtifactsIntermediatesDir=%__RepoRootDir%\artifacts\obj\coreclr\"