build-job.yml

parameters:
  archType: ''
  buildConfig: ''
  # compilerName specifies the compiler to use to do the builds. This can either be 'gcc' or left
  # unset to use the default (clang on Linux/Mac, Visual C++ on Windows).
  compilerName: ''
  condition: true
  container: ''
  crossBuild: false
  dependOnEvaluatePaths: false
  disableClrTest: false
  isOfficialBuild: false
  osGroup: ''
  osSubgroup: ''
  platform: ''
  pool: ''
  runtimeVariant: ''
  signBinaries: false
  testGroup: ''
  timeoutInMinutes: ''
  variables: {}
  pgoType: ''

### Product build
jobs:
- template: xplat-pipeline-job.yml
  parameters:
    buildConfig: ${{ parameters.buildConfig }}
    archType: ${{ parameters.archType }}
    osGroup: ${{ parameters.osGroup }}
    osSubgroup: ${{ parameters.osSubgroup }}
    runtimeVariant: ${{ parameters.runtimeVariant }}
    testGroup: ${{ parameters.testGroup }}
    helixType: 'build/product/'
    enableMicrobuild: true
    pool: ${{ parameters.pool }}
    condition: ${{ parameters.condition }}
    dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
    disableClrTest: ${{ parameters.disableClrTest }}
    pgoType: ${{ parameters.pgoType }}

    # Compute job name from template parameters
    ${{ if and(ne(parameters.testGroup, 'clrTools'), eq(parameters.compilerName, 'gcc')) }}:
      name: ${{ format('coreclr_{0}_product_build_{1}{1}_{3}_{4}', parameters.compilerName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
      displayName: ${{ format('GCC Product Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
    ${{ if and(ne(parameters.testGroup, 'clrTools'), ne(parameters.compilerName, 'gcc')) }}:
      name: ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}{5}',
        parameters.runtimeVariant,
        parameters.osGroup,
        parameters.osSubgroup,
        parameters.archType,
        parameters.buildConfig,
        parameters.pgoType) }}
      displayName: ${{ format('CoreCLR {0} Product Build {1}{2} {3} {4} {5}',
        parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup,
        parameters.archType,
        parameters.buildConfig,
        parameters.pgoType) }}
    ${{ if eq(parameters.testGroup, 'clrTools') }}:
      name: ${{ format('coreclr_{0}_tools_unittests_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
      displayName: ${{ format('CoreCLR {0} Tools Unit Tests {1}{2} {3} {4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}

    # Run all steps in the container.
    # Note that the containers are defined in platform-matrix.yml
    container: ${{ parameters.container }}

    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}

    crossBuild: ${{ parameters.crossBuild }}

    gatherAssetManifests: true
    variables:
    - name: osGroup
      value: ${{ parameters.osGroup }}
    - name: osSubgroup
      value: ${{ parameters.osSubgroup }}
    - name: compilerArg
      value: ''
    - name: publishLogsArtifactPrefix
      value: 'BuildLogs_CoreCLR'
    - ${{ if eq(parameters.testGroup, 'clrTools') }}:
      - name: publishLogsArtifactPrefix
        value: 'BuildLogs_CoreCLR_ToolsUnitTests'
    - ${{ if eq(parameters.compilerName, 'gcc') }}:
      - name: compilerArg
        value: '-gcc'
      - name: publishLogsArtifactPrefix
        value: 'BuildLogs_CoreCLR_GCC'
    - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc')) }}:
      - name: compilerArg
        value: '-clang9'
      # We need to use the stable version available on Alpine Linux
      - ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, '_musl')) }}:
        - name: compilerArg
          value: ''
      # AppleClang has different version scheme, so we let compiler introspection pick up the available clang from PATH
      - ${{ if eq(parameters.osGroup, 'osx') }}:
        - name: compilerArg
          value: ''
    - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
      # Variables used to publish packages to blob feed
      - name: dotnetfeedUrl
        value: https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json
      - name: dotnetfeedPAT
        value: $(dotnetfeed-storage-access-key-1)
      # Variables used by arcade to gather asset manifests
      - name: _DotNetPublishToBlobFeed
        value: true
    - name: officialBuildIdArg
      value: ''
    - ${{ if eq(parameters.isOfficialBuild, true) }}:
      - name: officialBuildIdArg
        value: '/p:OfficialBuildId=$(Build.BuildNumber)'
    - name: enforcePgoArg
      value: ''
    # The EnforcePGO script is only supported on Windows and is not supported on arm or arm64.
    - ${{ if and(eq(parameters.buildConfig, 'Release'), and(eq(parameters.osGroup, 'windows'), not(or(eq(parameters.archType, 'arm64'), eq(parameters.archType, 'arm')))), ne(parameters.pgoType, 'pgo')) }}:
      - name: enforcePgoArg
        value: '-enforcepgo'

    - name: clrInterpreterBuildArg
      value: ''
    - ${{ if eq(parameters.testGroup, 'clrinterpreter') }}:
      - name: clrInterpreterBuildArg
        value: '-cmakeargs "-DFEATURE_INTERPRETER=1"'

    - name: clrRuntimeComponentsBuildArg
      value: ''
    - ${{ if ne(parameters.testGroup, 'innerloop') }}:
      - name: clrRuntimeComponentsBuildArg
        value: '-component runtime -component alljits -component paltests -component nativeaot -component spmi '
    - ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.archType, 'x86')) }}:
      - name: clrRuntimeComponentsBuildArg
        value: '-component runtime -component jit -component iltools -component spmi '

    - name: pgoInstrumentArg
      value: ''
    - ${{ if eq(parameters.pgoType, 'PGO' )}}:
      - name: pgoInstrumentArg
        value: '-pgoinstrument '

    - name: SignType
      value: $[ coalesce(variables.OfficialSignType, 'real') ]

    - name: clrRuntimePortableBuildArg
      value: ''
    - ${{ if eq(parameters.osGroup, 'tizen') }}:
      - name: clrRuntimePortableBuildArg
        value: '-portablebuild=false'

    - name: nativeSymbols
      value: ''
    - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
      - name: nativeSymbols
        value: '--keepnativesymbols'

    - ${{ parameters.variables }}

    steps:

    # Install native dependencies
    # Linux builds use docker images with dependencies preinstalled,
    # and FreeBSD builds use a build agent with dependencies
    # preinstalled, so we only need this step for OSX and Windows.
    - ${{ if eq(parameters.osGroup, 'osx') }}:
      - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup)
        displayName: Install native dependencies

    # Install internal tools on official builds
    # Since our internal tools are behind an authenticated feed,
    # we need to use the DotNetCli AzDO task to restore from the feed using a service connection.
    # We can't do this from within the build, so we need to do this as a separate step.
    - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
      - template: /eng/pipelines/common/restore-internal-tools.yml

    # Install MicroBuild for signing the DAC and DBI
    - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.signBinaries, true), eq(parameters.osGroup, 'windows')) }}:
      - task: MicroBuildSigningPlugin@2
        displayName: Install MicroBuild plugin for Signing
        inputs:
          signType: $(SignType)
          zipSources: false
          feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
        continueOnError: false
        condition: and(succeeded(), in(variables['SignType'], 'real', 'test'))

    - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}:
      - script: |
          du -sh $(Build.SourcesDirectory)/*
          df -h
        displayName: Disk Usage before Build

    # Build/Generate native prerequisites
    - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.nativeprereqs $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci /bl:$(Build.SourcesDirectory)artifacts/log/$(buildConfig)/CoreCLRNativePrereqs.binlog
      displayName: Build and generate native prerequisites

    # Build CoreCLR Runtime
    - ${{ if ne(parameters.osGroup, 'windows') }}:
      - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(clrRuntimePortableBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols)
        displayName: Build CoreCLR Runtime
    - ${{ if eq(parameters.osGroup, 'windows') }}:
      - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg)
        displayName: Build CoreCLR Runtime

    - ${{ if ne(parameters.archType, 'x64') }}:
      - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 $(osArg) -ci $(compilerArg) -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) $(clrRuntimePortableBuildArg)
        displayName: Build CoreCLR Cross-Arch Tools (Tools that run on x64 targeting x86)

    - ${{ if and(eq(parameters.osGroup, 'windows'), eq(parameters.archType, 'arm')) }}:
      - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x86 $(osArg) -ci $(compilerArg) -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) $(clrRuntimePortableBuildArg)
        displayName: Build CoreCLR Cross-Arch Tools (Tools that run on x86 targeting arm)

    - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}:
      - script: |
          du -sh $(Build.SourcesDirectory)/*
          df -h
        displayName: Disk Usage after Build

    # Build CoreCLR Managed Components
    - ${{ if or(ne(parameters.osGroup, 'linux'), ne(parameters.archType, 'x86')) }}:
      - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+clr.paltestlist $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci
        displayName: Build managed product components and packages
    - ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.archType, 'x86')) }}:
      - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci
        displayName: Build managed product components and packages

    # Run CoreCLR Tools unit tests
    - ${{ if eq(parameters.testGroup, 'clrTools') }}:
      - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset libs $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci
        displayName: Build libs
      - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.toolstests $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci -test
        displayName: Run CoreCLR Tools unit tests

    # Build native test components
    - ${{ if and(ne(parameters.isOfficialBuild, true), ne(parameters.disableClrTest, true)) }}:
      - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(osArg) $(priorityArg) $(compilerArg)
        displayName: Build native test components

    # Build libs.native, host.native and mono with gcc
    - ${{ if eq(parameters.compilerName, 'gcc') }}:
      - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) libs.native+host.native+mono $(compilerArg) $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci
        displayName: Build libs.native+host.native+mono

    # Sign and add entitlements to these MacOS binaries
    - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
      - ${{ if eq(parameters.osGroup, 'osx') }}:

        - template: /eng/pipelines/common/macos-sign-with-entitlements.yml
          parameters:
            filesToSign:
            - name: createdump
              path: $(buildProductRootFolderPath)
              entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/createdump-entitlements.plist
            - name: corerun
              path: $(buildProductRootFolderPath)
              entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist

        - task: CopyFiles@2
          displayName: 'Copy signed createdump to sharedFramework'
          inputs:
            contents: createdump
            sourceFolder: $(buildProductRootFolderPath)
            targetFolder: $(buildProductRootFolderPath)/sharedFramework
            overWrite: true

    - ${{ if and(eq(parameters.osGroup, 'windows'), eq(parameters.signBinaries, true)) }}:
      - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml
        parameters:
          basePath: $(buildProductRootFolderPath)
          isOfficialBuild: ${{ parameters.signBinaries }}
          timeoutInMinutes: 30

    # Builds using gcc are not tested, and clrTools unitests do not publish the build artifacts
    - ${{ if and(ne(parameters.compilerName, 'gcc'), ne(parameters.testGroup, 'clrTools'), ne(parameters.disableClrTest, true)) }}:
      # Publish product output directory for consumption by tests.
      - template: /eng/pipelines/common/upload-artifact-step.yml
        parameters:
          rootFolder: $(buildProductRootFolderPath)
          includeRootFolder: false
          archiveType: $(archiveType)
          tarCompression: $(tarCompression)
          archiveExtension: $(archiveExtension)
          artifactName: $(buildProductArtifactName)
          displayName: 'product build'

    - ${{ if and(in(parameters.osGroup, 'windows', 'linux'), ne(parameters.archType, 'x86'), ne(parameters.compilerName, 'gcc'), ne(parameters.testGroup, 'clrTools'), eq(parameters.pgoType, '')) }}:
        - template: /eng/pipelines/coreclr/templates/crossdac-build.yml
          parameters:
            archType: ${{ parameters.archType }}
            osGroup: ${{ parameters.osGroup }}
            osSubgroup: ${{ parameters.osSubgroup }}
            isOfficialBuild: ${{ parameters.signBinaries }}
            ${{ if eq(parameters.archType, 'arm') }}:
              hostArchType: x86
            ${{ else }}:
              hostArchType: x64

    - ${{ if and(ne(parameters.compilerName, 'gcc'), ne(parameters.testGroup, ''), ne(parameters.testGroup, 'clrTools'), ne(parameters.disableClrTest, true)) }}:
      # Publish test native components for consumption by test execution.
      - ${{ if and(ne(parameters.isOfficialBuild, true), eq(parameters.pgoType, '')) }}:
        - template: /eng/pipelines/common/upload-artifact-step.yml
          parameters:
            rootFolder: $(nativeTestArtifactRootFolderPath)
            includeRootFolder: false
            archiveType: $(archiveType)
            tarCompression: $(tarCompression)
            archiveExtension: $(archiveExtension)
            artifactName: $(nativeTestArtifactName)
            displayName: 'native test components'

    # Get key vault secrets for publishing
    - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
      - task: AzureKeyVault@1
        inputs:
          azureSubscription: 'DotNet-Engineering-Services_KeyVault'
          KeyVaultName: EngKeyVault
          SecretsFilter: 'dotnetfeed-storage-access-key-1,microsoft-symbol-server-pat,symweb-symbol-server-pat'

    # Save packages using the prepare-signed-artifacts format.
    - ${{ if and(eq(parameters.isOfficialBuild, true), ne(parameters.testGroup, 'clrTools'), eq(parameters.pgoType, '')) }}:
      - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
        parameters:
          name: ${{ parameters.platform }}

    # Publish unit tests results if executing unit tests
    - ${{ if eq(parameters.testGroup, 'clrTools') }}:
      - task: PublishTestResults@2
        inputs:
          testResultsFormat: 'xUnit'
          testResultsFiles: '*.xml'
          testRunTitle: CoreCLR-Tools-Unittests-$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)
          searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
        continueOnError: true
        condition: always()

    # Publish Logs
    - task: PublishPipelineArtifact@1
      displayName: Publish Logs
      inputs:
        targetPath: $(Build.SourcesDirectory)/artifacts/log
        artifactName: '$(publishLogsArtifactPrefix)${{ parameters.pgoType }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
      continueOnError: true
      condition: always()