[microsoft/release-branch.go1.19] Cherry-pick: Move CodeQL to validation pipeline, keep patch check in Actions

.github/workflows/test.yml

@@ -0,0 +1,28 @@
+# Copyright (c) Microsoft Corporation.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This job tests that the patches apply cleanly, and nothing else. The goal is to test this as
+# quickly as possible. This job is a good signal for devs: GitHub Actions is quick to get an agent,
+# so when this job fails, it's easy to see, and clear to the dev that the rest of the PR's jobs
+# aren't going to succeed and can be ignored.
+#
+# Ideally, failure of this job would block the tests from running, because it would be a waste of
+# time to hit the patch failure N times. However, the actual tests run in AzDO, so we can't
+# reasonably cancel them from here (GitHub Actions).
+
+name: "Test"
+
+on:
+  pull_request:
+    branches: [ microsoft/* ]
+
+jobs:
+  check_patches:
+    name: Patches Apply Cleanly
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - run: pwsh eng/run.ps1 submodule-refresh -shallow

eng/pipeline/rolling-internal-validation-pipeline.yml

@@ -2,14 +2,14 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
-# This pipeline runs after each internal rolling build of Go and validates compliance.
-#
-# This pipeline template runs automated SDL validation with internal-only tooling. It uses a job
-# template from dotnet/arcade that runs the Guardian suite of tools and reports the results to TSA
-# (Trust Services Automation).
+# This pipeline runs after each internal rolling build of Go and validates compliance. It runs the
+# automated CodeQL scan and SDL validation with internal-only tooling. It uses a job template from
+# dotnet/arcade that runs the Guardian suite of tools and reports the results to TSA (Trust Services
+# Automation).
 #
 # For more information, see:
-# https://microsoft.sharepoint.com/teams/managedlanguages/_layouts/OneNote.aspx?id=%2Fteams%2Fmanagedlanguages%2Ffiles%2FTeam%20Notebook%2FGoLang%20Team&wd=target%28Main.one%7C62B655D4-14E7-41D6-A063-0869C28D63FC%2FSDL%20Tools%7C3908F727-3751-4ACC-8C71-6CEB2DF277B4%2F%29
+# CodeQL: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/codeql-semmle
+# SDL: https://microsoft.sharepoint.com/teams/managedlanguages/_layouts/OneNote.aspx?id=%2Fteams%2Fmanagedlanguages%2Ffiles%2FTeam%20Notebook%2FGoLang%20Team&wd=target%28Main.one%7C62B655D4-14E7-41D6-A063-0869C28D63FC%2FSDL%20Tools%7C3908F727-3751-4ACC-8C71-6CEB2DF277B4%2F%29
 
 trigger: none
 pr: none
@@ -34,7 +34,14 @@ resources:
             - microsoft/dev.boringcrypto.go*
 
 stages:
+  - template: stages/shorthand-builders-to-builders.yml
+    parameters:
+      jobsTemplate: builders-to-stages.yml
+      shorthandBuilders:
+        - { os: linux, arch: amd64, config: codeql }
+
   - stage: SDLValidate
+    dependsOn: []
     variables:
       # TSA variables.
       - group: go-sdl-validation

eng/pipeline/stages/run-stage.yml

@@ -27,6 +27,10 @@ stages:
           # https://github.com/microsoft/go/issues/568
           timeoutInMinutes: 180
 
+        ${{ if eq(parameters.builder.config, 'codeql') }}:
+          # Allow CodeQL to take a while. https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/configuring-codeql3000-ado-pipelines#other-issues
+          timeoutInMinutes: 360
+
         ${{ if eq(parameters.builder.os, 'windows') }}:
           pool:
             name: ${{ parameters.dncengPool.name }}
@@ -38,12 +42,14 @@ stages:
             pool:
               name: ${{ parameters.dncengPool.name }}
               demands: ${{ parameters.dncengPool.demands.linux }}
-            # The image used for the container this job runs in. The tests run in this container, so it
-            # should match what we support as closely as possible.
-            ${{ if not(parameters.builder.distro) }}:
-              container: golangpublicimages.azurecr.io/go-infra-images/prereqs:cbl-mariner-1.0.20211027-20211201-0cccc22
-            ${{ if eq(parameters.builder.distro, 'ubuntu') }}:
-              container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20211022152710-047508b
+            # The image used for the container this job runs in. The tests run in this container, so
+            # it should match what we support as closely as possible. Don't use a container for the
+            # CodeQL scan build to avoid interfering with the LD_PRELOAD hook.
+            ${{ if ne(parameters.builder.config, 'codeql') }}:
+              ${{ if eq(parameters.builder.distro, 'ubuntu') }}:
+                container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20211022152710-047508b
+              ${{ else }}:
+                container: golangpublicimages.azurecr.io/go-infra-images/prereqs:cbl-mariner-1.0.20211027-20211201-0cccc22
           ${{ if eq(parameters.builder.hostArch, 'arm64') }}:
             pool:
               name: Docker-Linux-Arm-Internal
@@ -52,6 +58,16 @@ stages:
 
         variables:
           - group: go-cmdscan-rules
+          - ${{ if eq(parameters.builder.config, 'codeql') }}:
+            # Enable CodeQL scan and configure options.
+            # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/configuring-codeql3000-ado-pipelines#additional-options
+            - name: Codeql.Enabled
+              value: true
+            - name: Codeql.Language
+              value: go,cpp
+            # Always scan. This way we don't miss out on a release branch build, for example.
+            - name: Codeql.Cadence
+              value: 0
 
         steps:
           - ${{ if eq(parameters.builder.os, 'linux') }}:
@@ -100,6 +116,12 @@ stages:
             # Set Git authorship info for 'cmd/go TestScript/build_buildvcs_auto'.
             - template: ../steps/set-bot-git-author-task.yml
 
+          # Manually init (and finalize, later) the CodeQL3000 extension so that it will run on any
+          # branch, not just the default.
+          # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/configuring-codeql3000-ado-pipelines#how-does-this-extension-work
+          - ${{ if eq(parameters.builder.config, 'codeql') }}:
+            - task: CodeQL3000Init@0
+
           - pwsh: |
               # Apply the patches as staged changes, so the HEAD commit is the same as upstream.
               eng/run.ps1 submodule-refresh
@@ -125,6 +147,13 @@ stages:
               displayName: Pipeline publish
               condition: succeededOrFailed()
 
+          # CodeQL plugs into the compiler to find the code. Just build.
+          - ${{ elseif eq(parameters.builder.config, 'codeql' ) }}:
+            - pwsh: |
+                eng/run.ps1 cmdscan -envprefix GO_CMDSCAN_RULE_ -- `
+                  pwsh eng/run.ps1 build
+              displayName: Build
+
           # Use run-builder for any configuration that includes tests. run-builder uses the "gotestsum"
           # module to convert test results to a JUnit file that Azure DevOps can understand.
           - ${{ else }}:
@@ -167,3 +196,6 @@ stages:
               displayName: Update file ownership from root to build agent account
               continueOnError: true
               condition: succeededOrFailed()
+
+          - ${{ if eq(parameters.builder.config, 'codeql') }}:
+            - task: CodeQL3000Finalize@0