[1.17] Merge from 1.16

eng/_core/archive/archive.go

@@ -16,12 +16,49 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
 )
 
+// CreateFromSource runs a Git command to generate an archive file from the Go source code at
+// "source". If output is "", the archive is produced in the build directory inside the
+// "eng/artifacts/bin" directory. A checksum file is also produced.
+func CreateFromSource(source string, output string) error {
+	fmt.Printf("---- Creating Go source archive (tarball) from '%v'...\n", source)
+
+	if output == "" {
+		output = filepath.Join(getBinDir(source), fmt.Sprintf("go.%v.src.tar.gz", getBuildID()))
+	}
+
+	// Ensure the target directory exists.
+	archiveDir := filepath.Dir(output)
+	if err := os.MkdirAll(archiveDir, os.ModeDir|os.ModePerm); err != nil {
+		return err
+	}
+
+	// Use "^{tree}" to avoid Git including a global extended pax header. The commit it would list
+	// is a temporary commit, and would only be confusing. See https://git-scm.com/docs/git-archive.
+	cmd := exec.Command("git", "archive", "-o", output, "--prefix=go/", "HEAD^{tree}")
+	cmd.Dir = source
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	fmt.Printf("---- Running command: %v\n", cmd.Args)
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	fmt.Printf("---- Creating checksum file...\n")
+	if err := writeSHA256ChecksumFile(output); err != nil {
+		return err
+	}
+
+	fmt.Printf("---- Pack complete.\n")
+	return nil
+}
+
 // CreateFromBuild walks the Go build directory at "source" and produces an archive with the path
 // "output". If output is "", CreateFromBuild produces a file in the build directory inside the
 // "eng/artifacts/bin" directory. The output directory is created if it doesn't exist. This function
@@ -34,21 +71,14 @@ func CreateFromBuild(source string, output string) error {
 	fmt.Printf("---- Creating Go archive (zip/tarball) from '%v'...\n", source)
 
 	if output == "" {
-		// If BUILD_BUILDNUMBER is defined (e.g. a CI build), use it. For local builds, use "dev".
-		archiveVersion := os.Getenv("BUILD_BUILDNUMBER")
-		if archiveVersion == "" {
-			archiveVersion = "dev"
-		}
-
+		archiveVersion := getBuildID()
 		archiveExtension := ".tar.gz"
 		if runtime.GOOS == "windows" {
 			archiveExtension = ".zip"
 		}
 
 		archiveName := fmt.Sprintf("go.%v.%v-%v%v", archiveVersion, runtime.GOOS, runtime.GOARCH, archiveExtension)
-		binDir := filepath.Join(source, "..", "eng", "artifacts", "bin")
-
-		output = filepath.Join(binDir, archiveName)
+		output = filepath.Join(getBinDir(source), archiveName)
 	}
 
 	// Ensure the target directory exists.
@@ -186,12 +216,27 @@ func CreateFromBuild(source string, output string) error {
 	}
 
 	fmt.Printf("---- Creating checksum file...\n")
-	writeSHA256ChecksumFile(output)
+	if err := writeSHA256ChecksumFile(output); err != nil {
+		return err
+	}
 
 	fmt.Printf("---- Pack complete.\n")
 	return nil
 }
 
+// getBuildID returns BUILD_BUILDNUMBER if defined (e.g. a CI build). Otherwise, "dev".
+func getBuildID() string {
+	archiveVersion := os.Getenv("BUILD_BUILDNUMBER")
+	if archiveVersion == "" {
+		return "dev"
+	}
+	return archiveVersion
+}
+
+func getBinDir(source string) string {
+	return filepath.Join(source, "..", "eng", "artifacts", "bin")
+}
+
 // getArchivePathRuntime takes a path like "go1.7.linux-amd64.tar.gz" and extension like ".tar.gz",
 // and returns the os (linux) and arch (amd64). The "path" extension may have multiple '.'
 // characters in it, so "ext" must be passed in explicitly or else the match would be ambiguous.

eng/_core/cmd/pack-source/pack-source.go

@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/microsoft/go/_core/archive"
+)
+
+const description = `
+This command creates a source archive of the Go submodule at HEAD.
+`
+
+func main() {
+	output := flag.String("o", "", "The path of the archive file to create, including extension. Default: a tar.gz file including build number in 'eng/artifacts/bin'.")
+	help := flag.Bool("h", false, "Print this help message.")
+
+	flag.Usage = func() {
+		fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n")
+		flag.PrintDefaults()
+		fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description)
+	}
+
+	flag.Parse()
+	if *help {
+		flag.Usage()
+		return
+	}
+
+	repoRootDir, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+
+	goRootDir := filepath.Join(repoRootDir, "go")
+
+	if err := archive.CreateFromSource(goRootDir, *output); err != nil {
+		panic(err)
+	}
+}

eng/_core/cmd/submodule-refresh/submodule-refresh.go

@@ -19,6 +19,7 @@ applies patches to the stage by default, or optionally as commits.
 `
 
 var commits = flag.Bool("commits", false, "Apply the patches as commits.")
+var skipPatch = flag.Bool("skip-patch", false, "Skip applying patches.")
 var origin = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.")
 var shallow = flag.Bool("shallow", false, "Clone the submodule with depth 1.")
 var fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.")
@@ -57,6 +58,10 @@ func refresh(rootDir string) error {
 		return err
 	}
 
+	if *skipPatch {
+		return nil
+	}
+
 	mode := patch.ApplyModeIndex
 	if *commits {
 		mode = patch.ApplyModeCommits

eng/_util/go.mod

@@ -7,6 +7,6 @@ module github.com/microsoft/go/_util
 go 1.16
 
 require (
-	github.com/microsoft/go-infra v0.0.0-20211206195537-520f2621bcf4
+	github.com/microsoft/go-infra v0.0.0-20220208232830-51df6c5b8843
 	gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d
 )

eng/_util/go.sum

@@ -15,8 +15,8 @@ github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/microsoft/go-infra v0.0.0-20211206195537-520f2621bcf4 h1:goYnnCGbN9HBwd0CtdiokVAlq5rTQkqq66K43kvGnhc=
-github.com/microsoft/go-infra v0.0.0-20211206195537-520f2621bcf4/go.mod h1:3IVGTm7qFJldQHximiWLg2kYfmugjZMGNHnvUo5Mo5M=
+github.com/microsoft/go-infra v0.0.0-20220208232830-51df6c5b8843 h1:IAk1GrsBP2l3sdWnaARrLALS9m6fVkGNgefpWRS0x2c=
+github.com/microsoft/go-infra v0.0.0-20220208232830-51df6c5b8843/go.mod h1:3IVGTm7qFJldQHximiWLg2kYfmugjZMGNHnvUo5Mo5M=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

eng/pipeline/jobs/builders-to-jobs.yml

@@ -10,12 +10,15 @@ parameters:
   # If true, include a signing job that depends on all 'buildandpack' builder jobs finishing. This
   # lets us start the lengthy tasks of signing and testing in parallel.
   sign: false
+  # If true, generate source archive tarballs, and sign them if signing is enabled.
+  createSourceArchive: false
 
 jobs:
   - ${{ each builder in parameters.builders }}:
     - template: run-job.yml
       parameters:
         builder: ${{ builder }}
+        createSourceArchive: ${{ parameters.createSourceArchive }}
 
   - ${{ if eq(parameters.sign, true) }}:
     - template: sign-job.yml

eng/pipeline/jobs/go-builder-matrix-jobs.yml

@@ -9,13 +9,15 @@ parameters:
   innerloop: false
   outerloop: false
   sign: false
+  createSourceArchive: false
 
 jobs:
   - template: shorthand-builders-to-builders.yml
     parameters:
       jobsTemplate: builders-to-jobs.yml
       jobsParameters:
         sign: ${{ parameters.sign }}
+        createSourceArchive: ${{ parameters.createSourceArchive }}
       shorthandBuilders:
         - ${{ if eq(parameters.innerloop, true) }}:
           - { os: linux, arch: amd64, config: buildandpack }

eng/pipeline/jobs/run-job.yml

@@ -7,6 +7,7 @@
 parameters:
   # { id, os, arch, config, distro? }
   builder: {}
+  createSourceArchive: false
 
 jobs:
   - job: ${{ parameters.builder.id }}
@@ -41,34 +42,30 @@ jobs:
             Write-Host "##vso[task.setvariable variable=GO_MAKE_MAX_RETRY_ATTEMPTS]5"
           displayName: Increase 'make' retry attempts
 
-
       # Initialize stage 0 toolset ahead of time so we can track timing data separately from the
       # build operations. When we call this script again later, it won't download Go again.
       - pwsh: |
           . eng/utilities.ps1
           Get-Stage0GoRoot
         displayName: Init stage 0 Go toolset
 
+      - template: ../steps/init-submodule-task.yml
+
+      # Create the source archive on one job only. The os choice is arbitrary.
+      - ${{ if and(eq(parameters.createSourceArchive, true), eq(parameters.builder.config, 'buildandpack'), eq(parameters.builder.os, 'linux')) }}:
+        - pwsh: |
+            git config --global user.name "microsoft-golang-bot"
+            git config --global user.email "microsoft-golang-bot@users.noreply.github.com"
+
+            # Turn the patches into commits, so HEAD includes the changes.
+            eng/run.ps1 submodule-refresh -commits
+            eng/run.ps1 pack-source
+          displayName: Archive submodule source
+
       - pwsh: |
-          function fetch_submodule() {
-            eng/run.ps1 submodule-refresh -shallow @args
-          }
-
-          if ("$env:FETCH_BEARER_TOKEN") {
-            fetch_submodule `
-              -origin 'https://dnceng@dev.azure.com/dnceng/internal/_git/microsoft-go-mirror' `
-              -fetch-bearer-token $env:FETCH_BEARER_TOKEN
-          } else {
-            fetch_submodule
-          }
-        # If non-public, use access token to fetch from repo. If public, don't use the access token,
-        # because anonymous auth is fine.
-        ${{ if ne(variables['System.TeamProject'], 'public') }}:
-          env:
-            FETCH_BEARER_TOKEN: $(System.AccessToken)
-          displayName: Set up submodule from internal mirror
-        ${{ if eq(variables['System.TeamProject'], 'public') }}:
-          displayName: Set up submodule
+          # Apply the patches as staged changes, so the HEAD commit is the same as upstream.
+          eng/run.ps1 submodule-refresh
+        displayName: Apply patches
 
       # Use build script directly for "buildandpack". If we used run-builder, we would need to
       # download its external module dependencies.

eng/pipeline/rolling-internal-pipeline.yml

@@ -20,6 +20,7 @@ stages:
         parameters:
           innerloop: true
           sign: true
+          createSourceArchive: true
 
   - stage: Publish
     dependsOn: Build
@@ -37,6 +38,7 @@ stages:
         steps:
           - template: steps/checkout-unix-task.yml
           - template: steps/init-pwsh-task.yml
+          - template: steps/init-submodule-task.yml
 
           - pwsh: |
               function TrimStart($s, $prefix) {
@@ -68,7 +70,7 @@ stages:
           - pwsh: |
               eng/run.ps1 createbuildassetjson `
                 -artifacts-dir '$(Pipeline.Workspace)/Binaries Signed/' `
-                -source-dir '$(Build.SourcesDirectory)' `
+                -source-dir '$(Build.SourcesDirectory)/go' `
                 -destination-url '$(blobDestinationUrl)' `
                 -branch '$(PublishBranchAlias)' `
                 -o '$(Pipeline.Workspace)/Binaries Signed/assets.json'

eng/pipeline/steps/init-submodule-task.yml

@@ -0,0 +1,26 @@
+# Copyright (c) Microsoft Corporation.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# Shallow checkout sources on Windows
+steps:
+  - pwsh: |
+      function fetch_submodule() {
+        eng/run.ps1 submodule-refresh -shallow -skip-patch @args
+      }
+
+      if ("$env:FETCH_BEARER_TOKEN") {
+        fetch_submodule `
+          -origin 'https://dnceng@dev.azure.com/dnceng/internal/_git/microsoft-go-mirror' `
+          -fetch-bearer-token $env:FETCH_BEARER_TOKEN
+      } else {
+        fetch_submodule
+      }
+    # If non-public, use access token to fetch from repo. If public, don't use the access token,
+    # because anonymous auth is fine.
+    ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      env:
+        FETCH_BEARER_TOKEN: $(System.AccessToken)
+      displayName: Set up submodule from internal mirror
+    ${{ if eq(variables['System.TeamProject'], 'public') }}:
+      displayName: Set up submodule

patches/0001-cmd-dist-add-JSON-output-support-for-some-tests.patch

@@ -1,4 +1,4 @@
-From be418f20052ddea746451bafe21fdbf2bb1c915d Mon Sep 17 00:00:00 2001
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
 From: Davis Goodin <dagood@microsoft.com>
 Date: Tue, 4 Jan 2022 11:23:27 -0600
 Subject: [PATCH] cmd/dist: add JSON output support for some tests
@@ -17,7 +17,7 @@ in JSON format, and the ordinary logs are still important.
  1 file changed, 10 insertions(+), 1 deletion(-)
 
 diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go
-index 50a2e5936c..0436c63063 100644
+index f40fa926df..f76ecbe28c 100644
 --- a/src/cmd/dist/test.go
 +++ b/src/cmd/dist/test.go
 @@ -29,6 +29,7 @@ func cmdtest() {
@@ -36,7 +36,7 @@ index 50a2e5936c..0436c63063 100644
  	rebuild     bool
  	failed      bool
  	keepGoing   bool
-@@ -294,9 +296,13 @@ func short() string {
+@@ -286,9 +288,13 @@ func short() string {
  // Callers should use goTest and then pass flags overriding these
  // defaults as later arguments in the command line.
  func (t *tester) goTest() []string {
@@ -51,7 +51,7 @@ index 50a2e5936c..0436c63063 100644
  }
 
  func (t *tester) tags() string {
-@@ -379,6 +385,9 @@ func (t *tester) registerStdTest(pkg string, useG3 bool) {
+@@ -371,6 +377,9 @@ func (t *tester) registerStdTest(pkg string, useG3 bool) {
  				t.timeout(timeoutSec),
  				"-gcflags=all=" + gcflags,
  			}