From d0b1a8849bfbc1dde18f201614d9eccaa3db6871 Mon Sep 17 00:00:00 2001 From: Rakesh Singh Date: Wed, 19 Dec 2018 12:08:14 -0800 Subject: [PATCH] Port buildpipeline json to Azure devops Yaml (#4860) * Port buildpipeline json to Azure devops Yaml. * Update per review comments * few more updates to the review comments * Fix PR build issue for Windows leg * NetCorePublic-Int-Pool machine pool doesn't have VS 2015 or VS 2017 installed. Switching to Hosted VS2017 --- DotnetCLIVersion.txt | 2 +- azure-pipelines.yml | 189 ++++++++++++++++++++++++++ config.json | 38 +++++- dependencies.props | 4 +- eng/jobs/finalize-publish.yml | 111 ++++++++++++++++ eng/jobs/linux-build.yml | 227 ++++++++++++++++++++++++++++++++ eng/jobs/osx-build.yml | 59 +++++++++ eng/jobs/validate-signing.yml | 59 +++++++++ eng/jobs/windows-build.yml | 200 ++++++++++++++++++++++++++++ init-tools.cmd | 4 +- init-tools.sh | 4 +- publish/publish.proj | 71 ++++++---- src/corehost/build.cmd | 4 +- src/managed/CommonManaged.props | 6 +- 14 files changed, 939 insertions(+), 39 deletions(-) create mode 100644 azure-pipelines.yml create mode 100644 eng/jobs/finalize-publish.yml create mode 100644 eng/jobs/linux-build.yml create mode 100644 eng/jobs/osx-build.yml create mode 100644 eng/jobs/validate-signing.yml create mode 100644 eng/jobs/windows-build.yml diff --git a/DotnetCLIVersion.txt b/DotnetCLIVersion.txt index 5e908d682f..6f1c03c6a3 100644 --- a/DotnetCLIVersion.txt +++ b/DotnetCLIVersion.txt @@ -1 +1 @@ -2.1.301 +2.1.401 \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000000..b42160ca60 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,189 @@ +trigger: +- master + +variables: + - name: TeamName + value: dotnet-core-acquisition + # Skip Running CI tests + - name: SkipTests + value: false + # Set build as stable to remove build number from package names, used for milestone builds + - name: IsStable + value: false + # Set Official Build Id + - name: OfficialBuildId + value: $(Build.BuildNumber) + # Produce Test build for PR and Public builds + - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + - name: SignType + value: test + # Set variables only for Official build from internal + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - name: SignType + value: real + # Variable groups + - group: DotNet-Blob-Feed + - group: DotNet-Symbol-Server-Pats + # Used for publishing individual leg assets to azure blobstorage + - group: DotNet-DotNetCli-Storage + # Used for publishing to myget + - group: DotNet-MyGet-Publish + # Used for dotnet/versions update + - group: DotNet-Versions-Publish + + # BlobFeed update + - name: _PublishBlobFeedUrl + value: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + - name: _BlobFeedArgs + value: /p:PackagesUrl=$(_PublishBlobFeedUrl) + /p:SymbolPackagesUrl=$(_PublishBlobFeedUrl) + /p:TransportFeedAccessToken=$(dotnetfeed-storage-access-key-1) + + # Symbol Server update + - name: _SymbolServerPath + value: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection + - name: _SymbolServerArgs + value: /p:SymbolServerPath=$(_SymbolServerPath) + /p:SymbolServerPAT=$(microsoft-symbol-server-pat) + /p:SymbolExpirationInDays=365 + + # ******** Official values ***** + - name: _CommonPublishArgs + value: /p:AzureAccountName=dotnetcli + /p:ContainerName=dotnet + /p:AzureAccessToken=$(dotnetcli-storage-key) + /p:ChecksumAzureAccountName=dotnetclichecksums + /p:ChecksumContainerName=dotnet + /p:ChecksumAzureAccessToken=$(dotnetclichecksums-storage-key) + # Used for publishing to dotnet myget account + - name: MyGetApiKey + value: $(dotnet-myget-org-api-key ) + - name: MyGetFeedUrl + value: https://dotnet.myget.org/F/dotnet-core/api/v2/package + - name: MyGetSymbolsFeedUrl + value: https://dotnet.myget.org/F/dotnet-core/symbols/api/v2/package + - name: _NugetFeedArgs + value: /p:NuGetFeedUrl=$(MyGetFeedUrl) + /p:NuGetSymbolsFeedUrl=$(MyGetSymbolsFeedUrl) + /p:NuGetApiKey=$(MyGetApiKey) + + # ******* End Dev testing Configuration ******** +jobs: + ################################################################################ + # Build Linux legs + ################################################################################ +- template: /eng/jobs/linux-build.yml + parameters: + displayName: Build_Linux_x64_glibc + dockerImage: microsoft/dotnet-buildtools-prereqs:centos-7-d485f41-20173404063424 + packageDistroListDeb: [debian.8,debian.9,ubuntu.16.04,ubuntu.18.04] + packageDistroListRpm: [centos.7,fedora.27,opensuse.42,oraclelinux.7,sles.12] + portableBuild: true + targetArchitecture: x64 + +- template: /eng/jobs/linux-build.yml + parameters: + crossBuild: true + displayName: Build_Linux_Arm + dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180323032140 + additionalRunArgs: -e ROOTFS_DIR=/crossrootfs/arm + portableBuild: true + skipTests: true + targetArchitecture: arm + +- template: /eng/jobs/linux-build.yml + parameters: + crossBuild: true + displayName: Build_Linux_Arm64 + dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180316023254 + additionalRunArgs: -e ROOTFS_DIR=/crossrootfs/arm64 + portableBuild: true + skipTests: true + targetArchitecture: arm64 + +- template: /eng/jobs/linux-build.yml + parameters: + additionalMSBuildArgs: /p:OutputRid=linux-musl-x64 + displayName: Build_Linux_x64_Alpine36 + dockerImage: microsoft/dotnet-buildtools-prereqs:alpine-3.6-3148f11-20171119021156 + portableBuild: false + targetArchitecture: x64 + +- template: /eng/jobs/linux-build.yml + parameters: + additionalMSBuildArgs: /p:OutputRid=rhel.6-x64 + displayName: Build_Linux_x64_Rhel6 + dockerImage: microsoft/dotnet-buildtools-prereqs:centos-6-376e1a3-20174311014331 + portableBuild: false + targetArchitecture: x64 + +# - template: /eng/jobs/linux-build.yml +# parameters: +# additionalMSBuildArgs: /p:OutputRid=rhel.6-x64 +# displayName: Build_Linux_x64_FreeBSD +# dockerImage: microsoft/dotnet-buildtools-prereqs:centos-6-376e1a3-20174311014331 +# osGroup: FreeBSD +# portableBuild: true +# skipTests: true +# targetArchitecture: x64 + + ################################################################################ + # Build Windows legs + ################################################################################ + +# Windows x64 +- template: /eng/jobs/windows-build.yml + parameters: + buildFullPlatformManifest: true + displayName: Build_Windows_x64 + publishRidAgnosticPackages: true + targetArchitecture: x64 + +# Windows x86 +- template: /eng/jobs/windows-build.yml + parameters: + displayName: Build_Windows_x86 + targetArchitecture: x86 + +# Windows Arm64 +- template: /eng/jobs/windows-build.yml + parameters: + displayName: Build_Windows_Arm64 + skipTests: true + targetArchitecture: arm64 + +# Windows Arm +- template: /eng/jobs/windows-build.yml + parameters: + displayName: Build_Windows_Arm + skipTests: true + targetArchitecture: arm + + ################################################################################ + # Build Build_OSX leg + ################################################################################ +- template: /eng/jobs/osx-build.yml + +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + ################################################################################ + # Signing validation + ################################################################################ + - template: /eng/jobs/validate-signing.yml + + ################################################################################ + # Finalize build publish + # publish nuget packages to blobfeed, symbolserver, and myget + ################################################################################ + - template: /eng/jobs/finalize-publish.yml + parameters: + _PublishType: nopublishtype + + ################################################################################ + # Publish to B.A.R + ################################################################################ + - template: /eng/common/templates/job/publish-build-assets.yml + parameters: + pool: + name: dotnet-internal-temp + dependsOn: + - Finalize_Publish \ No newline at end of file diff --git a/config.json b/config.json index e7d7a69834..c24940b9f9 100644 --- a/config.json +++ b/config.json @@ -196,12 +196,42 @@ "Project": "src\\corehost\\build.proj" } }, + "packaging": { + "description": "Build packaging project.", + "settings": { + "Project": "src\\pkg\\packaging\\dir.proj" + } + }, + "pkg": { + "description": "Build nuget packaging project.", + "settings": { + "Project": "src\\pkg\\dir.proj" + } + }, + "publishing": { + "description": "Build publishing project.", + "settings": { + "Project": "publish\\publish.proj" + } + }, + "sharedfx": { + "description": "Build Shared Framework project.", + "settings": { + "Project": "src\\sharedframework\\sharedframework.proj" + } + }, "signing-validation": { "description": "Build signing validation project.", "settings": { "Project": "signing\\signingvalidation.proj" } }, + "signing": { + "description": "Build signing project.", + "settings": { + "Project": "signing\\sign.proj" + } + }, "GenerateVersion": { "description": "Generates the version header for native binaries.", "settings": { @@ -345,12 +375,12 @@ "msbuild": { "osSpecific":{ "windows": { - "defaultParameters": "/nologo /verbosity:minimal /clp:Summary /maxcpucount /nodeReuse:false", - "path": "Tools/msbuild.cmd" + "defaultParameters": "msbuild /nologo /verbosity:minimal /clp:Summary /maxcpucount /nodeReuse:false", + "path": "Tools/dotnetcli/dotnet" }, "unix":{ - "defaultParameters": "/nologo /verbosity:minimal /clp:Summary /maxcpucount /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log", - "path": "Tools/msbuild.sh" + "defaultParameters": "msbuild /nologo /verbosity:minimal /clp:Summary /maxcpucount /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log", + "path": "Tools/dotnetcli/dotnet" } }, "valueTypes": { diff --git a/dependencies.props b/dependencies.props index 1976c921bd..80f15174d2 100644 --- a/dependencies.props +++ b/dependencies.props @@ -45,13 +45,13 @@ Microsoft.DotNet.Build.Tasks.Feed - 3.0.0-preview1-03430-01 + 2.2.0-beta.18578.9 Microsoft.SymbolUploader.Build.Task - 1.0.0-beta-63412-03 + 1.0.0-beta-63604-05 diff --git a/eng/jobs/finalize-publish.yml b/eng/jobs/finalize-publish.yml new file mode 100644 index 0000000000..0d6d0b2bff --- /dev/null +++ b/eng/jobs/finalize-publish.yml @@ -0,0 +1,111 @@ +parameters: + _PublishType: {} +jobs: + - job: Finalize_Publish + displayName: Finalize_Publish + # Run only if all build legs succeeded + condition: and(succeeded('Build_Linux_x64_glibc'), + succeeded('Build_Linux_Arm64'), + succeeded('Build_Linux_Arm'), + succeeded('Build_Linux_x64_Alpine36'), + succeeded('Build_Linux_x64_Rhel6'), + succeeded('Build_OSX'), + succeeded('Build_Windows_x64'), + succeeded('Build_Windows_x86'), + succeeded('Build_Windows_Arm'), + succeeded('Build_Windows_Arm64'), + ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) + # Run after all dependent legs are executed + dependsOn: + - Build_Linux_x64_glibc + - Build_Linux_Arm64 + - Build_Linux_Arm + - Build_Linux_x64_Alpine36 + # - Build_Linux_x64_FreeBSD + - Build_Linux_x64_Rhel6 + - Build_Windows_x64 + - Build_Windows_x86 + - Build_Windows_Arm + - Build_Windows_Arm64 + - Build_OSX + pool: + # For public or PR jobs, use the hosted pool. For internal jobs use the internal pool. + # Will eventually change this to two BYOC pools. + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: Hosted VS2017 + ${{ if ne(variables['System.TeamProject'], 'public') }}: + name: dotnet-internal-temp + variables: + _PublishType: ${{ parameters._PublishType}} + steps: + + # Initialize tooling + - script: build.cmd + -- /t:BuildTraversalBuildDependencies + displayName: Initialize tooling + condition: succeeded() + + - task: MicroBuildSigningPlugin@2 + displayName: Install MicroBuild plugin for Signing + inputs: + signType: $(SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + continueOnError: false + condition: and(succeeded(), in(variables['SignType'], 'real', 'test')) + + - task: MSBuild@1 + displayName: Publish (no PublishType) + inputs: + solution: $(Build.SourcesDirectory)\publish\publish.proj + platform: x64 + configuration: Release + msbuildVersion: 15.0 + msbuildArchitecture: x64 + msbuildArguments: '/p:Finalize=true + /p:OfficialPublish=true + /p:PublishType=nopublishtype + /p:SignType=$(SignType) + /p:StabilizePackageVersion=$(IsStable) + /p:TargetArchitecture=x64 + $(_BlobFeedArgs) + $(_CommonPublishArgs) + $(_NugetFeedArgs) + $(_SymbolServerArgs) + /bl:$(Build.SourcesDirectory)\finalizepublish.binlog' + condition: and(succeeded(), eq(variables._PublishType, 'nopublishtype')) + + - task: CopyFiles@2 + displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + *.log + *.binlog + TargetFolder: '$(Build.StagingDirectory)\BuildLogs' + continueOnError: true + condition: succeededOrFailed() + + - task: CopyFiles@2 + displayName: Copy Manifest file + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/AssetManifest' + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + continueOnError: true + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact BuildLogs + inputs: + PathtoPublish: '$(Build.StagingDirectory)\BuildLogs' + ArtifactName: Finalize_Publish + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: true + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/jobs/linux-build.yml b/eng/jobs/linux-build.yml new file mode 100644 index 0000000000..600886f137 --- /dev/null +++ b/eng/jobs/linux-build.yml @@ -0,0 +1,227 @@ +parameters: + additionalMSBuildArgs: '' + additionalRunArgs: '' + crossBuild: false + displayName: null + dockerImage: null + osGroup: Linux + packageDistroListDeb: null + packageDistroListRpm: null + skipTests: $(SkipTests) + strategy: + matrix: + debug: + _BuildConfig: Debug + _PublishType: none + release: + _BuildConfig: Release + _PublishType: blob + targetArchitecture: x64 +jobs: + - job: ${{ parameters.displayName }} + displayName: ${{ parameters.displayName }} + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: dnceng-linux-external-temp + ${{ if ne(variables['System.TeamProject'], 'public') }}: + name: dnceng-linux-internal-temp + strategy: ${{ parameters.strategy }} + variables: + + ${{ if ne(parameters.displayName,'Build_Linux_x64_FreeBSD')}}: + RunArguments: docker run --privileged --rm + -v "$(Build.SourcesDirectory):/root/coresetup" + -w="/root/coresetup" + ${{ parameters.additionalRunArgs }} + ${{ parameters.dockerImage }} + + ${{ if eq(parameters.displayName,'Build_Linux_x64_FreeBSD')}}: + RunArguments: export DotNetBootstrapCliTarPath=/dotnet-sdk-freebsd-x64.tar && + + BuildArguments: -OfficialBuildId=$(OfficialBuildId) + -ConfigurationGroup=$(_BuildConfig) + -CrossBuild=${{ parameters.crossBuild }} + -PortableBuild=${{ parameters.portableBuild }} + -strip-symbols + -SkipTests=${{ parameters.skipTests }} + -TargetArchitecture=${{ parameters.targetArchitecture }} + -- /p:StabilizePackageVersion=$(IsStable) + ${{ parameters.additionalMSBuildArgs }} + + PublishArguments: /p:PublishType=$(_PublishType) + /p:ConfigurationGroup=$(_BuildConfig) + /p:OSGroup=${{ parameters.osGroup }} + /p:PortableBuild=${{ parameters.portableBuild }} + /p:StabilizePackageVersion=$(IsStable) + /p:TargetArchitecture=${{ parameters.targetArchitecture }} + ${{ parameters.additionalMSBuildArgs }} + + CommonMSBuildArgs: /bl + /p:ConfigurationGroup=$(_BuildConfig) + /p:OfficialBuildId=$(OfficialBuildId) + /p:OSGroup=${{ parameters.osGroup }} + /p:PortableBuild=false + /p:StabilizePackageVersion=$(IsStable) + /p:TargetArchitecture=${{ parameters.targetArchitecture }} + + # Packaging related variables + # Debian + CommonDebianRunCommand: run + -v $(Build.SourcesDirectory):/root/coresetup + -v $(Build.StagingDirectory)/sharedFrameworkPublish/:/root/sharedFrameworkPublish/ + -w=/root/coresetup + microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-debpkg-e5cf912-20175003025046 + /root/coresetup/Tools/msbuild.sh + DebianPackagingCommand: $(CommonDebianRunCommand) + /root/coresetup/src/pkg/packaging/dir.proj + /p:UsePrebuiltPortableBinariesForInstallers=true + /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ + $(CommonMSBuildArgs) + + # RPM + CommonRpmRunCommand: run + -v $(Build.SourcesDirectory):/root/coresetup + -v $(Build.StagingDirectory)/sharedFrameworkPublish/:/root/sharedFrameworkPublish/ + -w=/root/coresetup + microsoft/dotnet-buildtools-prereqs:rhel-7-rpmpkg-c982313-20174116044113 + /root/coresetup/Tools/msbuild.sh + RpmPackagingCommand: $(CommonRpmRunCommand) + /root/coresetup/src/pkg/packaging/dir.proj + /p:UsePrebuiltPortableBinariesForInstallers=true + /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ + $(CommonMSBuildArgs) + steps: + + # Build binary and nuget packages + - script: $(RunArguments) + ./build.sh + $(BuildArguments) + displayName: Build + workingDirectory: '$(Build.SourcesDirectory)' + + # Publish only if internal and not PR + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(RunArguments) + ./Tools/msbuild.sh + ./publish/publish.proj + $(PublishArguments) + $(_CommonPublishArgs) + displayName: Publish + workingDirectory: '$(Build.SourcesDirectory)' + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + + # Only for glibc leg, here we produce RPMs and Debs + - ${{ if eq(parameters.displayName,'Build_Linux_x64_glibc')}}: + - task: CopyFiles@2 + displayName: 'Copy built Portable linux-x64 binaries to staging directory' + inputs: + SourceFolder: '$(Build.SourcesDirectory)/bin/obj/linux-x64.$(_BuildConfig)/sharedFrameworkPublish' + TargetFolder: '$(Build.StagingDirectory)/sharedFrameworkPublish' + - script: docker + $(CommonDebianRunCommand) + /root/coresetup/build.proj + /t:BuildTraversalBuildDependencies + $(CommonMSBuildArgs) + displayName: Build traversal build dependencies - Ubuntu 14.04 + workingDirectory: '$(Build.SourcesDirectory)' + + # Debian packaging + - script: docker + $(DebianPackagingCommand) + displayName: 'Package Runtime packages and Runtime Dep - Ubuntu 14.04' + workingDirectory: '$(Build.SourcesDirectory)' + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: docker + $(CommonDebianRunCommand) + /root/coresetup/publish/publish.proj + /p:PublishType=$(_PublishType) + /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ + $(CommonMSBuildArgs) + $(_CommonPublishArgs) + displayName: 'Publish Runtime and Runtime Dep - Ubuntu 14.04' + workingDirectory: '$(Build.SourcesDirectory)' + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + + # Package and publish for each debian distros + - ${{ each debdistrorid in parameters.packageDistroListDeb }}: + - script: docker + $(DebianPackagingCommand) /p:BuildRuntimeDebs=false + /p:OutputRid=${{ debdistrorid }}-${{ parameters.targetArchitecture }} + displayName: 'Package Runtime Dep - ${{ debdistrorid }}' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: docker + $(CommonDebianRunCommand) + /root/coresetup/publish/publish.proj + /p:PublishType=$(_PublishType) + /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ + $(CommonMSBuildArgs) + $(_CommonPublishArgs) + /p:OutputRid=${{ debdistrorid }}-${{ parameters.targetArchitecture }} + displayName: 'Publish Runtime Dep - ${{ debdistrorid }}' + workingDirectory: '$(Build.SourcesDirectory)' + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + + # RPM Packaging + - script: docker + $(CommonRpmRunCommand) + /root/coresetup/build.proj + /t:BuildTraversalBuildDependencies + $(CommonMSBuildArgs) + displayName: Build traversal build dependencies - Rhel7 + workingDirectory: '$(Build.SourcesDirectory)' + - script: docker + $(RpmPackagingCommand) + displayName: Package Runtime Dep - Rhel7 + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: docker + $(CommonRpmRunCommand) + /root/coresetup/publish/publish.proj + /p:PublishType=$(_PublishType) + /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ + $(CommonMSBuildArgs) + $(_CommonPublishArgs) + displayName: Publish Runtime Dep - Rhel7 + workingDirectory: '$(Build.SourcesDirectory)' + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + + # Package and publish for each rpm distros + - ${{ each rpmdistrorid in parameters.packageDistroListRpm }}: + - script: docker + $(RpmPackagingCommand) /p:BuildRuntimeRpms=false + /p:OutputRid=${{ rpmdistrorid }}-${{ parameters.targetArchitecture }} + displayName: 'Package Runtime Dep - ${{ rpmdistrorid }}' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: docker + $(CommonRpmRunCommand) + /root/coresetup/publish/publish.proj + /p:PublishType=$(_PublishType) + /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ + $(CommonMSBuildArgs) + $(_CommonPublishArgs) + /p:OutputRid=${{ rpmdistrorid }}-${{ parameters.targetArchitecture }} + displayName: 'Publish Runtime Dep -${{ rpmdistrorid }}' + workingDirectory: '$(Build.SourcesDirectory)' + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + - task: CopyFiles@2 + displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + *.log + *.binlog + TargetFolder: '$(Build.StagingDirectory)\BuildLogs' + continueOnError: true + condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact BuildLogs + inputs: + PathtoPublish: '$(Build.StagingDirectory)\BuildLogs' + ArtifactName: ${{ parameters.displayName }}-$(_BuildConfig) + continueOnError: true + condition: succeededOrFailed() + + # Force clean up machine in case any docker images are left behind + - script: docker system prune -f + displayName: Run Docker clean up + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/jobs/osx-build.yml b/eng/jobs/osx-build.yml new file mode 100644 index 0000000000..8c1e506ef0 --- /dev/null +++ b/eng/jobs/osx-build.yml @@ -0,0 +1,59 @@ +jobs: +- job: Build_OSX + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: Hosted macOS + ${{ if ne(variables['System.TeamProject'], 'public') }}: + name: Hosted Mac internal + strategy: + matrix: + debug: + _BuildConfig: Debug + _PublishType: none + release: + _BuildConfig: Release + _PublishType: blob + steps: + - script: $(Build.SourcesDirectory)/build.sh + -OfficialBuildId=$(OfficialBuildId) + -ConfigurationGroup=$(_BuildConfig) + -PortableBuild=true + -strip-symbols + -SkipTests=$(SkipTests) + -- /p:StabilizePackageVersion=$(IsStable) + /bl:$(Build.SourcesDirectory)\build.binlog + displayName: Build + condition: succeeded() + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Build.SourcesDirectory)/Tools/msbuild.sh + $(Build.SourcesDirectory)/publish/publish.proj + $(_CommonPublishArgs) + /p:PublishType=$(_PublishType) + /p:TargetArchitecture=x64 + /p:ConfigurationGroup=$(_BuildConfig) + /p:PortableBuild=true + /p:OSGroup=OSX + /p:StabilizePackageVersion=$(IsStable) + /bl:$(Build.SourcesDirectory)\publish.binlog + displayName: Publish + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + + - task: CopyFiles@2 + displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + *.log + *.binlog + TargetFolder: '$(Build.StagingDirectory)\BuildLogs' + continueOnError: true + condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact BuildLogs + inputs: + PathtoPublish: '$(Build.StagingDirectory)\BuildLogs' + ArtifactName: $(Agent.Os)_$(Agent.JobName) + continueOnError: true + condition: succeededOrFailed() + \ No newline at end of file diff --git a/eng/jobs/validate-signing.yml b/eng/jobs/validate-signing.yml new file mode 100644 index 0000000000..9ded95719e --- /dev/null +++ b/eng/jobs/validate-signing.yml @@ -0,0 +1,59 @@ +jobs: +- job: Signing_Validation + # Run only if all build legs succeeded + condition: and(succeeded('Build_Linux_x64_glibc'), + succeeded('Build_Linux_Arm64'), + succeeded('Build_Linux_Arm'), + succeeded('Build_Linux_x64_Alpine36'), + succeeded('Build_Linux_x64_Rhel6'), + succeeded('Build_Windows_x64'), + succeeded('Build_Windows_x86'), + succeeded('Build_Windows_Arm'), + succeeded('Build_Windows_Arm64'), + succeeded('Build_OSX'), + ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) + # Run after all dependent legs are executed + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: Hosted VS2017 + ${{ if ne(variables['System.TeamProject'], 'public') }}: + name: dotnet-internal-temp + dependsOn: + - Build_Linux_x64_glibc + - Build_Linux_Arm64 + - Build_Linux_Arm + - Build_Linux_x64_Alpine36 + # - Build_Linux_x64_FreeBSD + - Build_Linux_x64_Rhel6 + - Build_Windows_x64 + - Build_Windows_x86 + - Build_Windows_Arm + - Build_Windows_Arm64 + - Build_OSX + steps: + + - script: build.cmd + -signing-validation + -- '$(_CommonPublishArgs) + /p:OfficialBuildId=$(OfficialBuildId) + /flp:v=detailed + /p:SignType=$(SignType) + /p:StabilizePackageVersion=$(IsStable)' + + - task: CopyFiles@2 + displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + *.log + Bin\SigningValidation\Logs\** + TargetFolder: '$(Build.StagingDirectory)\BuildLogs' + continueOnError: true + condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact BuildLogs + inputs: + PathtoPublish: '$(Build.StagingDirectory)\BuildLogs' + ArtifactName: Signing_Validation + continueOnError: true + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/jobs/windows-build.yml b/eng/jobs/windows-build.yml new file mode 100644 index 0000000000..e2251dbee5 --- /dev/null +++ b/eng/jobs/windows-build.yml @@ -0,0 +1,200 @@ +parameters: + additionalMSBuildArguments: '' + buildFullPlatformManifest: false + displayName: '' + publishRidAgnosticPackages: false + skipTests: $(SkipTests) + targetArchitecture: null + +jobs: + - job: ${{ parameters.displayName }} + displayName: ${{ parameters.displayName }} + pool: + # For public or PR jobs, use the hosted pool. For internal jobs use the internal pool. + # Will eventually change this to two BYOC pools. + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: Hosted VS2017 + ${{ if ne(variables['System.TeamProject'], 'public') }}: + name: dotnet-internal-temp + strategy: + matrix: + debug: + _BuildConfig: Debug + _PublishType: none + release: + _BuildConfig: Release + _PublishType: blob + variables: + CommonMSBuildArgs: "/p:ConfigurationGroup=$(_BuildConfig) + /p:TargetArchitecture=${{ parameters.targetArchitecture }} + /p:PortableBuild=true + /p:SkipTests=${{ parameters.skipTests }} + /p:StabilizePackageVersion=$(IsStable)" + MsbuildSigningArguments: /p:CertificateId=400 /v:detailed /p:SignType=$(SignType) + steps: + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildSigningPlugin@2 + displayName: Install MicroBuild plugin for Signing + inputs: + signType: $(SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + continueOnError: false + condition: and(succeeded(), in(variables['SignType'], 'real', 'test')) + + # Initialize building + - script: $(Build.SourcesDirectory)\build.cmd + -- $(CommonMSBuildArgs) + '/t:BuildTraversalBuildDependencies + /bl:$(Build.SourcesDirectory)\init.binlog' + displayName: Build traversal build dependencies + condition: succeeded() + + # Build binaries + - script: build.cmd + --src-builds + -- $(CommonMSBuildArgs) + /p:BuildAppx=false + displayName: Build binaries + condition: succeeded() + + # Sign Binaries, signing requires VS MSBuild + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MSBuild@1 + displayName: Sign Binaries + inputs: + solution: '$(Build.SourcesDirectory)\signing\sign.proj' + msbuildVersion: 15.0 + msbuildArchitecture: x64 + msbuildArguments: '/t:SignBinaries + $(CommonMSBuildArgs) + $(MsbuildSigningArguments) + /bl:$(Build.SourcesDirectory)\singbinaries.binlog' + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + + # Build Nuget package + - script: build.cmd -pkg + -- '$(CommonMSBuildArgs) + /p:BuildFullPlatformManifest=${{ parameters.buildFullPlatformManifest }} + /flp:v=detailed;LogFile=$(Build.SourcesDirectory)\packages.binlog' + displayName: Build Nuget package + condition: succeeded() + + # Build sharedframework layout + - script: build.cmd -sharedfx + -- '$(CommonMSBuildArgs) + /bl:$(Build.SourcesDirectory)\sharedframework.binlog' + displayName: Build sharedframework layout + condition: succeeded() + + - ${{ if and(ne(parameters.displayName,'Build_Windows_x86'), ne(parameters.displayName,'Build_Windows_x64')) }}: + - script: build.cmd -packaging + -- '$(CommonMSBuildArgs) + /bl:$(Build.SourcesDirectory)\packaging.binlog' + displayName: Create Package + condition: succeeded() + + # Following steps followed in sequence only for Windows x86 and Windows x64 + # Create installers + # Sign MSIs + # Create bundles + # Extract Engine + # Sign Engine + # Re-attach Bundle + # Sign Bundle + - ${{ if or(eq(parameters.displayName,'Build_Windows_x86'), eq(parameters.displayName,'Build_Windows_x64')) }}: + - script: build.cmd -packaging + -- '/t:BuildInstallers + $(CommonMSBuildArgs) + /bl:$(Build.SourcesDirectory)\packaging.binlog' + displayName: Create installers + condition: succeeded() + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MSBuild@1 + displayName: Sign MSI and cab files + inputs: + solution: '$(Build.SourcesDirectory)\signing\sign.proj' + msbuildVersion: 15.0 + msbuildArchitecture: x64 + msbuildArguments: '/t:SignMsiAndCab + $(CommonMSBuildArgs) + $(MsbuildSigningArguments)' + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + + - script: build.cmd -packaging + -- '/t:BuildCombinedInstallers + $(CommonMSBuildArgs) + /bl:$(Build.SourcesDirectory)\createbundles.binlog' + displayName: Create bundles + condition: succeeded() + + - script: build.cmd -packaging + -- '/t:ExtractEngineBundle + $(CommonMSBuildArgs) + /bl:$(Build.SourcesDirectory)\extractengine.binlog' + displayName: Extract Engine from Bundle + condition: succeeded() + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MSBuild@1 + displayName: Sign Engine + inputs: + solution: '$(Build.SourcesDirectory)\signing\sign.proj' + msbuildVersion: 15.0 + msbuildArchitecture: x64 + msbuildArguments: '/t:SignEngine + $(CommonMSBuildArgs) + $(MsbuildSigningArguments) + /bl:$(Build.SourcesDirectory)\signengine.binlog' + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + + - script: build.cmd -packaging + -- '/t:ReattachEngineToBundle + $(CommonMSBuildArgs) + /bl:$(Build.SourcesDirectory)\reattachengine.binlog' + displayName: Re-attach Engine + condition: succeeded() + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MSBuild@1 + displayName: Sign Bundle + inputs: + solution: '$(Build.SourcesDirectory)\signing\sign.proj' + msbuildVersion: 15.0 + msbuildArchitecture: x64 + msbuildArguments: '/t:SignBundle + $(CommonMSBuildArgs) + $(MsbuildSigningArguments) + /bl:$(Build.SourcesDirectory)\signbundle.binlog' + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + # Publish all build assets + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: build.cmd -publishing + -- '$(CommonMSBuildArgs) + $(_CommonPublishArgs) + /p:PublishType=$(_PublishType) + /p:PublishRidAgnosticPackages=${{ parameters.publishRidAgnosticPackages }} + /p:BuildFullPlatformManifest=${{ parameters.buildFullPlatformManifest }} + /bl:$(Build.SourcesDirectory)\publish.binlog' + displayName: Publish build assets + condition: and(succeeded(), eq(variables._BuildConfig, 'Release')) + + - task: CopyFiles@2 + displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + *.log + *.binlog + TargetFolder: '$(Build.StagingDirectory)\BuildLogs' + continueOnError: true + condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact BuildLogs + inputs: + PathtoPublish: '$(Build.StagingDirectory)\BuildLogs' + ArtifactName: ${{ parameters.displayName }}-$(_BuildConfig) + continueOnError: true + condition: succeededOrFailed() \ No newline at end of file diff --git a/init-tools.cmd b/init-tools.cmd index 7c5fbfa122..286488fa11 100644 --- a/init-tools.cmd +++ b/init-tools.cmd @@ -63,8 +63,8 @@ if NOT exist "%DOTNET_LOCAL_PATH%" ( if exist "%BUILD_TOOLS_PATH%" goto :afterbuildtoolsrestore echo Restoring BuildTools version %BUILDTOOLS_VERSION%... -echo Running: "%DOTNET_CMD%" restore "%INIT_TOOLS_RESTORE_PROJECT%" --no-cache --packages "%PACKAGES_DIR%" --source "%BUILDTOOLS_SOURCE%" /p:BuildToolsPackageVersion=%BUILDTOOLS_VERSION% /p:ToolsDir=%TOOLRUNTIME_DIR% >> "%INIT_TOOLS_LOG%" -call "%DOTNET_CMD%" restore "%INIT_TOOLS_RESTORE_PROJECT%" --no-cache --packages "%PACKAGES_DIR%" --source "%BUILDTOOLS_SOURCE%" /p:BuildToolsPackageVersion=%BUILDTOOLS_VERSION% /p:ToolsDir=%TOOLRUNTIME_DIR% >> "%INIT_TOOLS_LOG%" +echo Running: "%DOTNET_CMD%" restore "%INIT_TOOLS_RESTORE_PROJECT%" --no-cache --packages "%PACKAGES_DIR%" --source "%BUILDTOOLS_SOURCE%" --source https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /p:BuildToolsPackageVersion=%BUILDTOOLS_VERSION% /p:ToolsDir=%TOOLRUNTIME_DIR% >> "%INIT_TOOLS_LOG%" +call "%DOTNET_CMD%" restore "%INIT_TOOLS_RESTORE_PROJECT%" --no-cache --packages "%PACKAGES_DIR%" --source "%BUILDTOOLS_SOURCE%" --source https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /p:BuildToolsPackageVersion=%BUILDTOOLS_VERSION% /p:ToolsDir=%TOOLRUNTIME_DIR% >> "%INIT_TOOLS_LOG%" if NOT exist "%BUILD_TOOLS_PATH%\init-tools.cmd" ( echo ERROR: Could not restore build tools correctly. 1>&2 goto :error diff --git a/init-tools.sh b/init-tools.sh index ec82371aba..5e6abb3ca7 100755 --- a/init-tools.sh +++ b/init-tools.sh @@ -149,8 +149,8 @@ fi if [ ! -e "$__BUILD_TOOLS_PATH" ]; then echo "Restoring BuildTools version $__BUILD_TOOLS_PACKAGE_VERSION..." - echo "Running: $__DOTNET_CMD restore \"$__INIT_TOOLS_RESTORE_PROJECT\" --no-cache --packages $__PACKAGES_DIR --source $__BUILDTOOLS_SOURCE /p:BuildToolsPackageVersion=$__BUILD_TOOLS_PACKAGE_VERSION /p:ToolsDir=$__TOOLRUNTIME_DIR" >> "$__init_tools_log" - "$__DOTNET_CMD" restore "$__INIT_TOOLS_RESTORE_PROJECT" --no-cache --packages "$__PACKAGES_DIR" --source "$__BUILDTOOLS_SOURCE" /p:BuildToolsPackageVersion=$__BUILD_TOOLS_PACKAGE_VERSION /p:ToolsDir="$__TOOLRUNTIME_DIR" >> "$__init_tools_log" + echo "Running: $__DOTNET_CMD restore \"$__INIT_TOOLS_RESTORE_PROJECT\" --no-cache --packages $__PACKAGES_DIR --source $__BUILDTOOLS_SOURCE --source https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /p:BuildToolsPackageVersion=$__BUILD_TOOLS_PACKAGE_VERSION /p:ToolsDir=$__TOOLRUNTIME_DIR" >> "$__init_tools_log" + "$__DOTNET_CMD" restore "$__INIT_TOOLS_RESTORE_PROJECT" --no-cache --packages "$__PACKAGES_DIR" --source "$__BUILDTOOLS_SOURCE" --source https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json /p:BuildToolsPackageVersion=$__BUILD_TOOLS_PACKAGE_VERSION /p:ToolsDir="$__TOOLRUNTIME_DIR" >> "$__init_tools_log" if [ ! -e "$__BUILD_TOOLS_PATH/init-tools.sh" ]; then echo "ERROR: Could not restore build tools correctly." 1>&2 display_error_message diff --git a/publish/publish.proj b/publish/publish.proj index 461b0446c9..c20635d8b9 100644 --- a/publish/publish.proj +++ b/publish/publish.proj @@ -12,7 +12,7 @@ - + PublishToAzure; @@ -32,7 +32,7 @@ + DependsOnTargets="PublishCoreHostPackages;SetupPublishSymbols;PublishSymbols;FinalizeBuildInAzure" /> @@ -207,7 +207,7 @@ - <_CoreHostPackages Include="%(_BlobList.Identity)" Condition="'%(_BlobList.Extension)' == '.nupkg'" /> + <_CoreHostPackages Include="%(_BlobList.Identity)" /> $(BinDir)PackageDownload/ @@ -223,18 +223,21 @@ BlobNamePrefix="Runtime/$(SharedFrameworkNugetVersion)/" DownloadDirectory="$(DownloadDirectory)" /> - <_DownloadedPackages Include="@(_CoreHostPackages->'$(DownloadDirectory)%(Filename)%(Extension)')" /> + <_DownloadedAssets Include="@(_CoreHostPackages->'$(DownloadDirectory)%(Filename)%(Extension)')" /> + <_DownloadedPackages Include="%(_DownloadedAssets.Identity)" + Condition="$([System.String]::new('%(_DownloadedAssets.Identity)').EndsWith('.nupkg'))" /> <_DownloadedSymbolsPackages Include="%(_DownloadedPackages.Identity)" Condition="$([System.String]::new('%(_DownloadedPackages.Identity)').EndsWith('.symbols.nupkg'))" /> <_DownloadedStandardPackages Include="@(_DownloadedPackages)" - Exclude="@(_DownloadedSymbolsPackages)" /> + Exclude="@(_DownloadedSymbolsPackages)" + Condition="$([System.String]::new('%(_DownloadedPackages.Identity)').EndsWith('.nupkg'))" /> 1 - true + false @@ -262,7 +265,7 @@ @@ -270,32 +273,54 @@ $(PackagesUrl) + $(BUILD_SOURCESDIRECTORY)/artifacts/log/AssetManifest/Manifest.xml - - - - - - - - - $(SymbolPackagesUrl) - - - + Overwrite="$(OverwriteOnPublish)" + ManifestBuildData="Location=$(ExpectedFeedUrl)" + ManifestRepoUri="$(BUILD_REPOSITORY_URI)" + ManifestBranch="$(BUILD_SOURCEBRANCH)" + ManifestBuildId="$(BUILD_BUILDNUMBER)" + ManifestCommit="$(BUILD_SOURCEVERSION)" + AssetManifestPath="$(AssetManifestFilePath)" + /> + + + + + + $(PackagesUrl) + $(BUILD_SOURCESDIRECTORY)/artifacts/log/AssetManifest/Manifest_Installers.xml + + + + + + assets/core-setup/$(InstallersRelativePath)%(Filename)%(Extension) + + + + diff --git a/src/corehost/build.cmd b/src/corehost/build.cmd index dc4a628816..5191188742 100644 --- a/src/corehost/build.cmd +++ b/src/corehost/build.cmd @@ -141,8 +141,8 @@ cd %__rootDir% SET __NativeBuildArgs=/t:rebuild if /i "%__IncrementalNativeBuild%" == "1" SET __NativeBuildArgs= -echo %__rootDir%\run.cmd build-native -- "%__IntermediatesDir%\ALL_BUILD.vcxproj" %__NativeBuildArgs% /p:Configuration=%CMAKE_BUILD_TYPE% %__msbuildArgs% -call %__rootDir%\run.cmd build-native -- "%__IntermediatesDir%\ALL_BUILD.vcxproj" %__NativeBuildArgs% /p:Configuration=%CMAKE_BUILD_TYPE% %__msbuildArgs% +echo msbuild "%__IntermediatesDir%\ALL_BUILD.vcxproj" %__NativeBuildArgs% /p:Configuration=%CMAKE_BUILD_TYPE% %__msbuildArgs% +call msbuild "%__IntermediatesDir%\ALL_BUILD.vcxproj" %__NativeBuildArgs% /p:Configuration=%CMAKE_BUILD_TYPE% %__msbuildArgs% IF ERRORLEVEL 1 ( goto :Failure ) diff --git a/src/managed/CommonManaged.props b/src/managed/CommonManaged.props index a442d51900..a6894f86bb 100644 --- a/src/managed/CommonManaged.props +++ b/src/managed/CommonManaged.props @@ -10,7 +10,7 @@ true https://github.com/dotnet/core-setup/blob/master/LICENSE.TXT https://dot.net - $(RepoRoot)LICENSE.TXT + $(RepoRoot)LICENSE.TXT $(RepoRoot)THIRD-PARTY-NOTICES.TXT @@ -36,8 +36,8 @@ - +