Fix duplicate artifact error on PR pipeline job re-runs

eng/pipelines/pr/sqlclient-pr-pipeline.yml

@@ -109,6 +109,7 @@ stages:
       buildConfiguration: Debug
       buildSuffix: pr
       stageName: ${{ variables.stageNamePack }}
+      packArtifactBaseName: ${{ variables.packArtifactBaseName }}
 
   # Stage 1b: Generate secrets
   - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml@self
@@ -142,6 +143,7 @@ stages:
   # Stage 3: Collect code coverage
   - template: /eng/pipelines/pr/stages/collect-coverage-stage.yml@self
     parameters:
+      coverageArtifactBaseName: ${{ variables.coverageArtifactBaseName }}
       dependsOn:
         - ${{ each platform in parameters.platforms }}:
             - "test_${{ platform.displayName }}"

eng/pipelines/pr/stages/collect-coverage-stage.yml

@@ -11,6 +11,11 @@ parameters:
     type: object
     default: []
 
+  # Base name of the artifact to publish the merged coverage report to. The job attempt number is
+  # appended to this to prevent collision across re-runs of the job.
+  - name: coverageArtifactBaseName
+    type: string
+
 stages:
   - stage: collect_code_coverage
     displayName: "Collect code coverage"
@@ -61,11 +66,14 @@ stages:
 
           # Part 2) Publish merged results to the pipeline results/artifacts
 
-          # Publish the merged coverage file as a pipeline artifact
+          # Publish the merged coverage file as a pipeline artifact.
+          #
+          # The artifact name includes the job attempt number so that re-running a failed job does
+          # not collide with the artifact published by a previous attempt.
           - task: PublishPipelineArtifact@1
             displayName: Publish coverage artifact
             inputs:
-              artifact: merged_coverage
+              artifact: ${{ parameters.coverageArtifactBaseName }}_attempt$(System.JobAttempt)
               targetPath: "${{ variables.workingDir }}/merge"
 
           # Publish the merged coverage file as coverage results so they can be viewed in ADO UI.

eng/pipelines/pr/stages/pack-stage.yml

@@ -37,6 +37,11 @@ parameters:
   - name: stageName
     type: string
 
+  # Base name of the artifact to publish the build output to. The job attempt number is appended
+  # to this to prevent collision across re-runs of the job.
+  - name: packArtifactBaseName
+    type: string
+
 stages:
   - stage: ${{ parameters.stageName }}
     displayName: Build and Pack Projects
@@ -121,9 +126,15 @@ stages:
             displayName: Output Build Output Tree
             condition: succeededOrFailed()
 
-          # Upload the build output as the artifact of the job
+          # Upload the build output as the artifact of the job.
+          #
+          # The artifact name includes the job attempt number because pipeline artifact names must
+          # be unique within a build. Re-running a failed job would otherwise attempt to publish
+          # an artifact that already exists and fail with:
+          #   "Artifact build_and_pack_projects already exists for build <id>."
           - publish: $(BUILD_OUTPUT)
-            artifact: build_and_pack_projects
+            artifact: ${{ parameters.packArtifactBaseName }}_attempt$(System.JobAttempt)
+            displayName: Publish Build Output
             condition: succeededOrFailed()
 
 

eng/pipelines/pr/steps/publish-test-results-step.yml

@@ -31,7 +31,7 @@ parameters:
     type: string
 
   # Absolute path to the test results folder. This folder will be published to the artifact defined
-  # by ${{ testResultsArtifactBaseName }}_$(System.JobId)
+  # by ${{ parameters.testResultsArtifactBaseName }}_$(System.JobId)_attempt$(System.JobAttempt)
   - name: testResultsPath
     type: string
 
@@ -52,11 +52,15 @@ steps:
       testRunTitle: "${{ parameters.platformDisplayName }}_${{ parameters.testDisplayName }}"
     condition: succeededOrFailed()
 
-  # Publish the test results as artifacts for the pipeline
+  # Publish the test results as artifacts for the pipeline.
+  #
+  # The artifact name includes the job attempt number because System.JobId is stable across
+  # attempts of the same job. Without it, re-running a failed test job would fail with
+  # "Artifact ... already exists for build <id>."
   - task: PublishPipelineArtifact@1
     displayName: 'Publish Test Artifacts'
     inputs:
-      artifact: ${{ parameters.testResultsArtifactBaseName }}_$(System.JobId)
+      artifact: ${{ parameters.testResultsArtifactBaseName }}_$(System.JobId)_attempt$(System.JobAttempt)
       targetPath: ${{ parameters.testResultsPath }}
     condition: succeededOrFailed()
 

eng/pipelines/pr/variables/pr-variables.yml

@@ -35,6 +35,24 @@ variables:
   - name: stageNameSecrets
     value: "secrets_stage"
 
+  # Base names of the artifacts published by this pipeline.
+  #
+  # Pipeline artifact names must be unique within a build, and neither the job ID nor the stage
+  # name changes when a job is re-run. Every publish step therefore appends
+  # $(System.JobAttempt) (and, where relevant, $(System.JobId)) to these base names so that
+  # re-running a failed job does not collide with the artifact published by a previous attempt.
+  #
+  # These are defined here, rather than inline at the publish step, so that producers and any
+  # future consumers of the artifacts stay in sync through a single definition.
+
+  # Base name of the build output artifact published by the pack stage.
+  - name: packArtifactBaseName
+    value: "build_and_pack_projects"
+
+  # Base name of the merged code coverage artifact published by the coverage stage.
+  - name: coverageArtifactBaseName
+    value: "merged_coverage"
+
   # Base name of the test result artifacts. Job ID should be appended to this to prevent collision.
   - name: testResultsArtifactBaseName
     value: "test_results"