Update eng/common to dotnet/docker-tools@6fabb8f (#284)

* upgrade docker-tools to 6fabb8f * Move 1ES PT image definition inside internal project conditional * Use WS2016 image with support for 1ES Pipeline Templates
microsoft · Mar 27, 2024 · 74d0cc3 · 74d0cc3
1 parent 0130565
commit 74d0cc3
Show file tree

Hide file tree

Showing 19 changed files with 112 additions and 106 deletions.
diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml
@@ -26,6 +26,8 @@ jobs:
     imageBuilderDockerRunExtraOptions: $(build.imageBuilderDockerRunExtraOptions)
     versionsRepoPath: versions
     sbomDirectory: $(Build.ArtifactStagingDirectory)/sbom
+    imageInfoHostDir: $(Build.ArtifactStagingDirectory)/imageInfo
+    imageInfoContainerDir: $(artifactsPath)/imageInfo
     ${{ if eq(parameters.noCache, false) }}:
       versionsBasePath: $(versionsRepoPath)/
       pipelineDisabledCache: false
@@ -76,6 +78,8 @@ jobs:
       echo "##vso[task.setvariable variable=baseContainerRepoPath]$baseContainerRepoPath"
     displayName: Set Base Container Repo Path
   - template: /eng/common/templates/jobs/${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}@self
+    parameters:
+      cleanupDocker: true
   - ${{ parameters.customInitSteps }}
   - template: /eng/common/templates/steps/set-image-info-path-var.yml@self
     parameters:
@@ -90,7 +94,8 @@ jobs:
       # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since
       # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates
       # the environment variable for us.
-      $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(artifactsPath)/$(legName)-image-info.json"
+      New-Item -Path $(imageInfoHostDir) -ItemType Directory -Force
+      $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json"
       if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
         $imageBuilderBuildArgs = "$imageBuilderBuildArgs --registry-override $(acr.server) --repo-prefix $(stagingRepoPrefix) --source-repo-prefix $(mirrorRepoPrefix) --push --registry-creds ""$(acr.server)=$(acr.userName);$(acr.password)"""
       }
@@ -123,7 +128,7 @@ jobs:
     displayName: Build Images
   - template: /eng/common/templates/steps/publish-artifact.yml@self
     parameters:
-      path: $(Build.ArtifactStagingDirectory)/$(legName)-image-info.json
+      path: $(imageInfoHostDir)
       artifactName: $(legName)-image-info-$(System.JobAttempt)
       displayName: Publish Image Info File Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -181,15 +186,14 @@ jobs:
             -PackageVersion '$(Build.BuildNumber)' `
             -ManifestDirPath $sbomChildDir `
             -DockerImagesToScan $_ `
-            -Verbosity Information 
+            -Verbosity Information
         }
       displayName: Generate SBOMs
       condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))
   - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
     - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self
       parameters:
         condition: ne(variables.testScriptPath, '')
-  - template: /eng/common/templates/jobs/${{ format('../steps/cleanup-docker-{0}.yml', parameters.dockerClientOS) }}@self
   - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
     - template: /eng/common/templates/steps/publish-artifact.yml@self
       parameters:

diff --git a/eng/common/templates/jobs/cg-build-projects.yml b/eng/common/templates/jobs/cg-build-projects.yml
@@ -0,0 +1,17 @@
+# This job builds all projects in the repository. It is intended to be used for CG purposes.
+# The 1ES CG step does not scan artifacts that are built within Dockerfiles therefore they
+# need to be built outside of Dockerfiles.
+jobs:
+- job: BuildProjects
+  displayName: Build Projects
+  pool:
+    name: NetCore1ESPool-Internal
+    image: 1es-ubuntu-2204
+    os: linux
+  steps:
+  - powershell: >
+      ./eng/common/Install-DotNetSdk.ps1 /usr/share/.dotnet
+    displayName: Run Dotnet Install Script
+  - script: >
+      find . -name '*.csproj' | grep $(cgBuildGrepArgs) | xargs -n 1 /usr/share/.dotnet/dotnet build
+    displayName: Build Projects
diff --git a/eng/common/templates/jobs/cg-detection.yml b/eng/common/templates/jobs/cg-detection.yml
diff --git a/eng/common/templates/jobs/copy-base-images.yml b/eng/common/templates/jobs/copy-base-images.yml
@@ -16,4 +16,3 @@ jobs:
       additionalOptions: ${{ parameters.additionalOptions }}
       publicProjectName: ${{ parameters.publicProjectName }}
       continueOnError: true
-  - template: /eng/common/templates/steps/cleanup-docker-linux.yml@self
diff --git a/eng/common/templates/jobs/generate-matrix.yml b/eng/common/templates/jobs/generate-matrix.yml
@@ -38,4 +38,3 @@ jobs:
       $(additionalGenerateBuildMatrixOptions)
     displayName: Generate ${{ parameters.matrixType }} Matrix
     name: matrix
-  - template: /eng/common/templates/steps/cleanup-docker-linux.yml@self
diff --git a/eng/common/templates/jobs/post-build.yml b/eng/common/templates/jobs/post-build.yml
@@ -8,32 +8,35 @@ jobs:
   pool: ${{ parameters.pool }}
   variables:
     imageInfosSubDir: "/image-infos"
-    sbomSubDir: "/sbom"
+    imageInfosHostDir: "$(Build.ArtifactStagingDirectory)$(imageInfosSubDir)"
+    imageInfosContainerDir: "$(artifactsPath)$(imageInfosSubDir)"
+    imageInfosOutputSubDir: "/output"
+    sbomOutputDir: "$(Build.ArtifactStagingDirectory)/sbom"
   steps:
   - template: /eng/common/templates/steps/init-docker-linux.yml@self
   - template: /eng/common/templates/steps/download-build-artifact.yml@self
     parameters:
       targetPath: $(Build.ArtifactStagingDirectory)
   - powershell: |
       # Move all image-info artifacts to their own directory
-      New-Item -ItemType Directory -Path $(Build.ArtifactStagingDirectory)$(imageInfosSubDir)
+      New-Item -ItemType Directory -Path $(imageInfosHostDir)
       Get-ChildItem -Directory -Filter "*-image-info-*" $(Build.ArtifactStagingDirectory) |
-        Move-Item -Verbose -Destination $(Build.ArtifactStagingDirectory)$(imageInfosSubDir)
+        Move-Item -Verbose -Destination $(imageInfosHostDir)
     displayName: Collect Image Info Files
   - powershell: |
       # Move the contents of all the SBOM artifact directories to a single location
-      New-Item -ItemType Directory -Path $(Build.ArtifactStagingDirectory)$(sbomSubDir)
+      New-Item -ItemType Directory -Path $(sbomOutputDir)
       Get-ChildItem -Directory -Filter "*-sboms" $(Build.ArtifactStagingDirectory) |
         ForEach-Object {
-          Get-ChildItem $_ -Directory | Move-Item -Force -Verbose -Destination $(Build.ArtifactStagingDirectory)$(sbomSubDir)
+          Get-ChildItem $_ -Directory | Move-Item -Force -Verbose -Destination $(sbomOutputDir)
         }
     displayName: Consolidate SBOMs to Single Directory
   - powershell: |
       # Deletes the artifacts from all the unsuccessful jobs
-      Get-ChildItem $(Build.ArtifactStagingDirectory)$(imageInfosSubDir) -Directory |
+      Get-ChildItem $(imageInfosHostDir) -Directory |
           ForEach-Object {
               [pscustomobject]@{
-                  # Parse the artifact name to separate the base of the name from the job attempt number 
+                  # Parse the artifact name to separate the base of the name from the job attempt number
                   BaseName = $_.Name.Substring(0, $_.Name.LastIndexOf('-'));
                   JobAttempt = $_.Name.Substring($_.Name.LastIndexOf('-') + 1)
                   FullName = $_.FullName
@@ -48,23 +51,24 @@ jobs:
                   Remove-Item -Recurse -Force
           }
     displayName: Prune Publish Artifacts
-  - script: >
-      $(runImageBuilderCmd) mergeImageInfo
-      --manifest $(manifest)
-      $(artifactsPath)$(imageInfosSubDir)
-      $(artifactsPath)$(imageInfosSubDir)/image-info.json
-      $(manifestVariables)
+  - powershell: |
+      New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force
+      $(runImageBuilderCmd) mergeImageInfo `
+        --manifest $(manifest) `
+        $(imageInfosContainerDir) `
+        $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json `
+        $(manifestVariables)
     displayName: Merge Image Info Files
   - template: /eng/common/templates/steps/publish-artifact.yml@self
     parameters:
-      path: $(Build.ArtifactStagingDirectory)$(sbomSubDir)
+      path: $(sbomOutputDir)
       artifactName: sboms
       displayName: Publish SBOM Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
   - template: /eng/common/templates/steps/publish-artifact.yml@self
     parameters:
-      path: $(Build.ArtifactStagingDirectory)$(imageInfosSubDir)/image-info.json
+      path: $(imageInfosHostDir)$(imageInfosOutputSubDir)
       artifactName: image-info
       displayName: Publish Image Info File Artifact
       internalProjectName: ${{ parameters.internalProjectName }}

diff --git a/eng/common/templates/jobs/publish.yml b/eng/common/templates/jobs/publish.yml
@@ -21,6 +21,12 @@ jobs:
       value: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ]
     ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}:
       value: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ]
+  - name: imageInfoHostDir
+    value: $(Build.ArtifactStagingDirectory)/imageInfo
+  - name: imageInfoContainerDir
+    value: $(artifactsPath)/imageInfo
+  - name: sourceBuildIdOutputDir
+    value: $(Build.ArtifactStagingDirectory)/sourceBuildId
   - ${{ parameters.customPublishVariables }}
   steps:
   - template: /eng/common/templates/steps/retain-build.yml@self
@@ -35,17 +41,19 @@ jobs:
       internalProjectName: ${{ parameters.internalProjectName }}
   - template: /eng/common/templates/steps/download-build-artifact.yml@self
     parameters:
-      targetPath: $(Build.ArtifactStagingDirectory)
+      targetPath: $(imageInfoHostDir)
       artifactName: image-info
   - template: /eng/common/templates/steps/set-image-info-path-var.yml@self
     parameters:
       publicSourceBranch: $(publicSourceBranch)
   - template: /eng/common/templates/steps/set-dry-run.yml@self
-  - script: echo $(sourceBuildId) > $(Build.ArtifactStagingDirectory)/source-build-id.txt
+  - powershell: |
+      New-Item -ItemType Directory -Path $(sourceBuildIdOutputDir)
+      Set-Content -Path $(sourceBuildIdOutputDir)/source-build-id.txt -Value $(sourceBuildId)
     displayName: Write Source Build ID to File
   - template: /eng/common/templates/steps/publish-artifact.yml@self
     parameters:
-      path: $(Build.ArtifactStagingDirectory)/source-build-id.txt
+      path: $(sourceBuildIdOutputDir)
       artifactName: source-build-id
       displayName: Publish Source Build ID Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -54,7 +62,7 @@ jobs:
     displayName: Set Publish Variables
   - script: >
       $(runImageBuilderCmd) trimUnchangedPlatforms
-      '$(artifactsPath)/image-info.json'
+      '$(imageInfoContainerDir)/image-info.json'
     displayName: Trim Unchanged Images
   - script: >
       $(runImageBuilderCmd) copyAcrImages
@@ -67,14 +75,14 @@ jobs:
       --os-type '*'
       --architecture '*'
       --repo-prefix '$(publishRepoPrefix)'
-      --image-info '$(artifactsPath)/image-info.json'
+      --image-info '$(imageInfoContainerDir)/image-info.json'
       $(dryRunArg)
       $(imageBuilder.pathArgs)
       $(imageBuilder.commonCmdArgs)
     displayName: Copy Images
   - script: >
       $(runImageBuilderCmd) publishManifest
-      '$(artifactsPath)/image-info.json'
+      '$(imageInfoContainerDir)/image-info.json'
       --repo-prefix '$(publishRepoPrefix)'
       --registry-creds '$(acr.server)=$(acr.userName);$(acr.password)'
       --os-type '*'
@@ -85,7 +93,7 @@ jobs:
     displayName: Publish Manifest
   - template: /eng/common/templates/steps/publish-artifact.yml@self
     parameters:
-      path: $(Build.ArtifactStagingDirectory)/image-info.json
+      path: $(imageInfoHostDir)
       artifactName: image-info-final-$(System.JobAttempt)
       displayName: Publish Image Info File Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -102,7 +110,7 @@ jobs:
       condition: and(succeeded(), eq(variables['publishReadme'], 'true'))
   - script: >
       $(runImageBuilderCmd) publishImageInfo
-      '$(artifactsPath)/image-info.json'
+      '$(imageInfoContainerDir)/image-info.json'
       '$(gitHubVersionsRepoInfo.userName)'
       '$(gitHubVersionsRepoInfo.email)'
       '$(gitHubVersionsRepoInfo.accessToken)'
@@ -116,7 +124,7 @@ jobs:
     displayName: Publish Image Info
   - script: >
       $(runImageBuilderCmd) ingestKustoImageInfo
-      '$(artifactsPath)/image-info.json'
+      '$(imageInfoContainerDir)/image-info.json'
       '$(kusto.cluster)'
       '$(kusto.database)'
       '$(kusto.imageTable)'
@@ -134,7 +142,7 @@ jobs:
       $(runImageBuilderCmd) postPublishNotification
       '$(publishNotificationRepoName)'
       '$(branchName)'
-      '$(artifactsPath)/image-info.json'
+      '$(imageInfoContainerDir)/image-info.json'
       $(Build.BuildId)
       '$(System.AccessToken)'
       '$(azdoOrgName)'
@@ -154,4 +162,3 @@ jobs:
       $(imageBuilder.commonCmdArgs)
     displayName: Post Publish Notification
     condition: and(always(), eq(variables['publishNotificationsEnabled'], 'true'))
-  - template: /eng/common/templates/steps/cleanup-docker-linux.yml@self
diff --git a/eng/common/templates/jobs/wait-for-ingestion.yml b/eng/common/templates/jobs/wait-for-ingestion.yml
@@ -24,4 +24,3 @@ jobs:
     parameters:
       commitDigest: $(readmeCommitDigest)
       condition: and(succeeded(), ne(variables['readmeCommitDigest'], ''))
-  - template: ../steps/cleanup-docker-linux.yml
diff --git a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml
@@ -56,7 +56,7 @@ stages:
 
     internalVersionsRepoRef: InternalVersionsRepo
     publicVersionsRepoRef: PublicVersionsRepo
-    
+
     ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
       customPublishVariables:
       - group: DotNet-AllOrgs-Darc-Pats
@@ -71,30 +71,33 @@ stages:
         name: NetCore1ESPool-Internal
         image: 1es-ubuntu-2204
         os: linux
-    
+
     # Linux Arm64
     linuxArm64Pool:
-      image: Mariner-2-Docker-ARM64
       os: linux
       hostArchitecture: Arm64
       ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
         name: Docker-Linux-Arm-Public
       ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
+        image: Mariner-2-Docker-ARM64
         name: Docker-Linux-Arm-Internal
 
     # Linux Arm32
     linuxArm32Pool:
-      image: Mariner-2-Docker-ARM64
       os: linux
       hostArchitecture: Arm64
       ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
         name: Docker-Linux-Arm-Public
       ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
+        image: Mariner-2-Docker-ARM64
         name: Docker-Linux-Arm-Internal
 
     # Windows Server 2016
-    # TODO: Update this to 1ES-compatible pool
-    windows2016Pool: Docker-2016-${{ variables['System.TeamProject'] }}
+    windows2016Pool:
+      os: windows
+      name: Docker-2016-${{ variables['System.TeamProject'] }}
+      ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
+        image: Server2016-NESDockerBuilds-PT
 
     # Windows Server 2019 (1809)
     ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:

diff --git a/eng/common/templates/steps/copy-base-images.yml b/eng/common/templates/steps/copy-base-images.yml
@@ -2,7 +2,7 @@ parameters:
   additionalOptions: null
   publicProjectName: null
   continueOnError: false
-  
+
 steps:
 - ${{ if or(eq(variables['System.TeamProject'], parameters.publicProjectName), eq(variables['Build.Reason'], 'PullRequest')) }}:
   - template: /eng/common/templates/steps/set-dry-run.yml@self

diff --git a/eng/common/templates/steps/init-docker-linux.yml b/eng/common/templates/steps/init-docker-linux.yml
@@ -1,7 +1,7 @@
 parameters:
   setupImageBuilder: true
   setupTestRunner: false
-  cleanupDocker: true
+  cleanupDocker: false
   condition: true
 
 steps:
@@ -16,7 +16,7 @@ steps:
   # Cleanup Docker Resources
   ################################################################################
 - ${{ if eq(parameters.cleanupDocker, 'true') }}:
-  - template: cleanup-docker-linux.yml
+  - template: /eng/common/templates/steps/cleanup-docker-linux.yml@self
     parameters:
       condition: ${{ parameters.condition }}
 

diff --git a/eng/common/templates/steps/set-dry-run.yml b/eng/common/templates/steps/set-dry-run.yml
@@ -1,12 +1,11 @@
 steps:
 - powershell: |
-      # Use dry-run option for certain publish operations if this is not a production build
-      $dryRunArg=""
-      if (-not "$(officialRepoPrefixes)".Split(',').Contains("$(publishRepoPrefix)") `
-          -or "$(System.TeamProject)" -eq "$(publicProjectName)")
-      {
-        $dryRunArg=" --dry-run"
-      }
-      
-      echo "##vso[task.setvariable variable=dryRunArg]$dryRunArg"
+    # Use dry-run option for certain publish operations if this is not a production build
+    $dryRunArg=""
+    if (-not "$(officialRepoPrefixes)".Split(',').Contains("$(publishRepoPrefix)") `
+        -or "$(System.TeamProject)" -eq "$(publicProjectName)")
+    {
+      $dryRunArg=" --dry-run"
+    }
+    echo "##vso[task.setvariable variable=dryRunArg]$dryRunArg"
   displayName: Set dry-run arg for non-prod