From 5c2af7f884759d586174fd6626d54667300c1e4d Mon Sep 17 00:00:00 2001 From: Doug Bunting <6431421+dougbu@users.noreply.github.com> Date: Wed, 2 Nov 2022 09:15:30 -0700 Subject: [PATCH] [release/1.1] Add CodeQL3000 to microsoft-reverse-proxy-official (#1917) * [release/1.1] Add CodeQL3000 to microsoft-reverse-proxy-official - backport of #1914 - add a new weekly schedule to keep database and tools up to database - add top-level parameter enabling CodeQL3000 in manual builds - add CodeQL3000 variables - add CodeQL3000 tasks - not auto-injected because that happens only in builds of the default branch - add tag indicating CodeQL3000 tasks did useful work - ignore no-op task executions - add a tsaoptions.json file - cribbed values from eng/sdl-tsa-vars.config file also: - backport #1843 - backport #1848 - switch to `-Svc` pools since this is a servicing branch nits: - disable auto-injection of build steps we or Arcade do explicitly - remove irrelevant variables in test jobs * !fixup! Take more of #1848 --- .config/tsaoptions.json | 12 ++ azure-pipelines.yml | 135 ++++++++++++------ eng/common/templates/job/source-build.yml | 4 +- .../templates/job/source-index-stage1.yml | 4 +- 4 files changed, 104 insertions(+), 51 deletions(-) create mode 100644 .config/tsaoptions.json diff --git a/.config/tsaoptions.json b/.config/tsaoptions.json new file mode 100644 index 000000000..02096468f --- /dev/null +++ b/.config/tsaoptions.json @@ -0,0 +1,12 @@ +{ + "areaPath": "DevDiv\\ASP.NET Core\\YARP", + "codebaseName": "ReverseProxy", + "instanceUrl": "https://devdiv.visualstudio.com/", + "iterationPath": "DevDiv", + "notificationAliases": [ + "dotnetrp@microsoft.com" + ], + "projectName": "DEVDIV", + "repositoryName": "ReverseProxy", + "template": "TFSDEVDIV" +} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 34d7ff7ed..ea991747d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -2,6 +2,27 @@ # See https://docs.microsoft.com/azure/devops/pipelines/yaml-schema for details # +schedules: +- cron: 0 9 * * 1 + displayName: "Run CodeQL3000 weekly, Monday at 2:00 AM PDT" + branches: + include: + - release/1.0 + - release/1.1 + - release/latest + - main + always: true + +parameters: + # Parameter below is ignored in public builds. + # + # Choose whether to run the CodeQL3000 tasks. + # Manual builds align w/ official builds unless this parameter is true. + - name: runCodeQL3000 + default: false + displayName: Run CodeQL3000 tasks + type: boolean + variables: - name: _TeamName value: AspNetCore @@ -11,9 +32,17 @@ variables: value: true - name: _BuildConfig value: Release + - name: runCodeQL3000 + value: ${{ and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.Reason'], 'Schedule'), and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.runCodeQL3000, 'true')))) }} + # Rely on task Arcade injects, not auto-injected build step. + - name: skipComponentGovernanceDetection + value: true + # Rely on explicit tasks, not auto-injected build steps. CodeQL3000 not enabled by default in any case. + - name: Codeql.SkipTaskAutoInjection + value: true - # used for post-build phases, internal builds only - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # used for post-build phases, internal builds only + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.runCodeQL3000, 'true')) }}: - group: YARP-SDLValidation-Params # CI and PR triggers @@ -37,10 +66,10 @@ stages: jobs: - template: /eng/common/templates/jobs/jobs.yml parameters: - enableMicrobuild: true + enableMicrobuild: ${{ ne(variables.runCodeQL3000, 'true') }} enablePublishBuildArtifacts: true - enablePublishTestResults: true - enablePublishBuildAssets: true + enablePublishTestResults: ${{ ne(variables.runCodeQL3000, 'true') }} + enablePublishBuildAssets: ${{ ne(variables.runCodeQL3000, 'true') }} enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} enableTelemetry: true mergeTestResults: true @@ -48,16 +77,35 @@ stages: - job: Windows pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore1ESPool-Public - demands: ImageOverride -equals Build.Windows.10.Amd64.VS2019.Pre.Open + name: NetCore-Svc-Public + demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Windows.10.Amd64.VS2019.Pre + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2019.amd64 + ${{ if eq(variables.runCodeQL3000, 'true') }}: + # Component governance and SBOM creation are not needed here. Disable what Arcade would inject. + disableComponentGovernance: true + enableSbom: false + # CodeQL3000 extends build duration. + timeoutInMinutes: 180 variables: - - + - ${{ if eq(variables.runCodeQL3000, 'true') }}: + - _OfficialBuildArgs: /p:Test=false /p:Sign=false /p:Pack=false /p:Publish=false /p:UseSharedCompilation=false + - _SignType: none + # Security analysis is included in normal runs. Disable its auto-injection. + - skipNugetSecurityAnalysis: true + # Do not let CodeQL3000 Extension gate scan frequency. + - Codeql.Cadence: 0 + # Enable CodeQL3000 unconditionally so it may be run on any branch. + - Codeql.Enabled: true + # Ignore test and infrastructure code. + - Codeql.SourceRoot: src + # CodeQL3000 needs this plumbed along as a variable to enable TSA. + - Codeql.TSAEnabled: ${{ eq(variables['Build.Reason'], 'Schedule') }} + # Default expects tsaoptions.json under SourceRoot. + - Codeql.TSAOptionsPath: '$(Build.SourcesDirectory)/.config/tsaoptions.json' # Only enable publishing in official builds. - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.runCodeQL3000, 'true')) }}: # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT - group: Publish-Build-Assets - name: _OfficialBuildArgs @@ -77,25 +125,35 @@ stages: steps: - checkout: self clean: true + - ${{ if eq(variables.runCodeQL3000, 'true') }}: + - task: CodeQL3000Init@0 + displayName: CodeQL Initialize + - script: "echo ##vso[build.addbuildtag]CodeQL3000" + displayName: 'Set CI CodeQL3000 tag' + condition: ne(variables.CODEQL_DIST,'') - script: eng\common\cibuild.cmd -configuration $(_BuildConfig) -prepareMachine $(_OfficialBuildArgs) displayName: Build and Publish - - task: PublishBuildArtifacts@1 - displayName: Upload TestResults - condition: always() - continueOnError: true - inputs: - pathtoPublish: artifacts/TestResults/$(_BuildConfig)/ - artifactName: $(Agent.Os)_$(Agent.JobName) TestResults - artifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Upload package artifacts - condition: and(succeeded(), eq(variables['system.pullrequest.isfork'], false), eq(variables['_BuildConfig'], 'Release')) - inputs: - pathtoPublish: artifacts/packages/ - artifactName: artifacts - artifactType: Container - parallel: true + - ${{ if eq(variables.runCodeQL3000, 'true') }}: + - task: CodeQL3000Finalize@0 + displayName: CodeQL Finalize + - ${{ else }}: + - task: PublishBuildArtifacts@1 + displayName: Upload TestResults + condition: always() + continueOnError: true + inputs: + pathtoPublish: artifacts/TestResults/$(_BuildConfig)/ + artifactName: $(Agent.Os)_$(Agent.JobName) TestResults + artifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Upload package artifacts + condition: and(succeeded(), eq(variables['system.pullrequest.isfork'], false), eq(variables['_BuildConfig'], 'Release')) + inputs: + pathtoPublish: artifacts/packages/ + artifactName: artifacts + artifactType: Container + parallel: true - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - job: Ubuntu @@ -104,21 +162,12 @@ stages: variables: - name: _SignType value: none - - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: _OfficialBuildArgs - value: -p:OfficialBuildId=$(Build.BuildNumber) - # else - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - name: _OfficialBuildArgs - value: '' steps: - checkout: self clean: true - script: eng/common/cibuild.sh --configuration $(_BuildConfig) --prepareMachine - $(_InternalRuntimeDownloadArgs) displayName: Build - task: PublishBuildArtifacts@1 displayName: Upload TestResults @@ -137,20 +186,12 @@ stages: variables: - name: _SignType value: none - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: _OfficialBuildArgs - value: -p:OfficialBuildId=$(Build.BuildNumber) - # else - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - name: _OfficialBuildArgs - value: '' steps: - checkout: self clean: true - script: eng/common/cibuild.sh --configuration $(_BuildConfig) --prepareMachine - $(_InternalRuntimeDownloadArgs) displayName: Build - task: PublishBuildArtifacts@1 displayName: Upload TestResults @@ -162,7 +203,7 @@ stages: artifactType: Container parallel: true -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.runCodeQL3000, 'true')) }}: - template: eng\common\templates\post-build\post-build.yml parameters: publishingInfraVersion: 3 diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 5cd5325d7..b6137f44a 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -46,10 +46,10 @@ jobs: # source-build builds run in Docker, including the default managed platform. pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore1ESPool-Public + name: NetCore-Svc-Public demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal + name: NetCore1ESPool-Svc-Internal demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 4af724eb1..3b20d7884 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -24,10 +24,10 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore1ESPool-Public + name: NetCore-Svc-Public demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal + name: NetCore1ESPool-Svc-Internal demands: ImageOverride -equals Build.Server.Amd64.VS2019 steps: - ${{ each preStep in parameters.preSteps }}: