diff --git a/eng/_core/archive/archive.go b/eng/_core/archive/archive.go index 9ac017d77ba..213c874e5ed 100644 --- a/eng/_core/archive/archive.go +++ b/eng/_core/archive/archive.go @@ -16,12 +16,49 @@ import ( "io" "io/fs" "os" + "os/exec" "path/filepath" "runtime" "strings" "time" ) +// CreateFromSource runs a Git command to generate an archive file from the Go source code at +// "source". If output is "", the archive is produced in the build directory inside the +// "eng/artifacts/bin" directory. A checksum file is also produced. +func CreateFromSource(source string, output string) error { + fmt.Printf("---- Creating Go source archive (tarball) from '%v'...\n", source) + + if output == "" { + output = filepath.Join(getBinDir(source), fmt.Sprintf("go.%v.src.tar.gz", getBuildID())) + } + + // Ensure the target directory exists. + archiveDir := filepath.Dir(output) + if err := os.MkdirAll(archiveDir, os.ModeDir|os.ModePerm); err != nil { + return err + } + + // Use "^{tree}" to avoid Git including a global extended pax header. The commit it would list + // is a temporary commit, and would only be confusing. See https://git-scm.com/docs/git-archive. + cmd := exec.Command("git", "archive", "-o", output, "--prefix=go/", "HEAD^{tree}") + cmd.Dir = source + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + fmt.Printf("---- Running command: %v\n", cmd.Args) + if err := cmd.Run(); err != nil { + return err + } + + fmt.Printf("---- Creating checksum file...\n") + if err := writeSHA256ChecksumFile(output); err != nil { + return err + } + + fmt.Printf("---- Pack complete.\n") + return nil +} + // CreateFromBuild walks the Go build directory at "source" and produces an archive with the path // "output". If output is "", CreateFromBuild produces a file in the build directory inside the // "eng/artifacts/bin" directory. The output directory is created if it doesn't exist. This function @@ -34,21 +71,14 @@ func CreateFromBuild(source string, output string) error { fmt.Printf("---- Creating Go archive (zip/tarball) from '%v'...\n", source) if output == "" { - // If BUILD_BUILDNUMBER is defined (e.g. a CI build), use it. For local builds, use "dev". - archiveVersion := os.Getenv("BUILD_BUILDNUMBER") - if archiveVersion == "" { - archiveVersion = "dev" - } - + archiveVersion := getBuildID() archiveExtension := ".tar.gz" if runtime.GOOS == "windows" { archiveExtension = ".zip" } archiveName := fmt.Sprintf("go.%v.%v-%v%v", archiveVersion, runtime.GOOS, runtime.GOARCH, archiveExtension) - binDir := filepath.Join(source, "..", "eng", "artifacts", "bin") - - output = filepath.Join(binDir, archiveName) + output = filepath.Join(getBinDir(source), archiveName) } // Ensure the target directory exists. @@ -186,12 +216,27 @@ func CreateFromBuild(source string, output string) error { } fmt.Printf("---- Creating checksum file...\n") - writeSHA256ChecksumFile(output) + if err := writeSHA256ChecksumFile(output); err != nil { + return err + } fmt.Printf("---- Pack complete.\n") return nil } +// getBuildID returns BUILD_BUILDNUMBER if defined (e.g. a CI build). Otherwise, "dev". +func getBuildID() string { + archiveVersion := os.Getenv("BUILD_BUILDNUMBER") + if archiveVersion == "" { + return "dev" + } + return archiveVersion +} + +func getBinDir(source string) string { + return filepath.Join(source, "..", "eng", "artifacts", "bin") +} + // getArchivePathRuntime takes a path like "go1.7.linux-amd64.tar.gz" and extension like ".tar.gz", // and returns the os (linux) and arch (amd64). The "path" extension may have multiple '.' // characters in it, so "ext" must be passed in explicitly or else the match would be ambiguous. diff --git a/eng/_core/cmd/pack-source/pack-source.go b/eng/_core/cmd/pack-source/pack-source.go new file mode 100644 index 00000000000..62e7f294113 --- /dev/null +++ b/eng/_core/cmd/pack-source/pack-source.go @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/microsoft/go/_core/archive" +) + +const description = ` +This command creates a source archive of the Go submodule at HEAD. +` + +func main() { + output := flag.String("o", "", "The path of the archive file to create, including extension. Default: a tar.gz file including build number in 'eng/artifacts/bin'.") + help := flag.Bool("h", false, "Print this help message.") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + + flag.Parse() + if *help { + flag.Usage() + return + } + + repoRootDir, err := os.Getwd() + if err != nil { + panic(err) + } + + goRootDir := filepath.Join(repoRootDir, "go") + + if err := archive.CreateFromSource(goRootDir, *output); err != nil { + panic(err) + } +} diff --git a/eng/_core/cmd/submodule-refresh/submodule-refresh.go b/eng/_core/cmd/submodule-refresh/submodule-refresh.go index ca582702585..9566fb11511 100644 --- a/eng/_core/cmd/submodule-refresh/submodule-refresh.go +++ b/eng/_core/cmd/submodule-refresh/submodule-refresh.go @@ -19,6 +19,7 @@ applies patches to the stage by default, or optionally as commits. ` var commits = flag.Bool("commits", false, "Apply the patches as commits.") +var skipPatch = flag.Bool("skip-patch", false, "Skip applying patches.") var origin = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.") var shallow = flag.Bool("shallow", false, "Clone the submodule with depth 1.") var fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.") @@ -57,6 +58,10 @@ func refresh(rootDir string) error { return err } + if *skipPatch { + return nil + } + mode := patch.ApplyModeIndex if *commits { mode = patch.ApplyModeCommits diff --git a/eng/_util/go.mod b/eng/_util/go.mod index e821571fcfb..dbb261edf7c 100644 --- a/eng/_util/go.mod +++ b/eng/_util/go.mod @@ -7,6 +7,6 @@ module github.com/microsoft/go/_util go 1.16 require ( - github.com/microsoft/go-infra v0.0.0-20211206195537-520f2621bcf4 + github.com/microsoft/go-infra v0.0.0-20220208232830-51df6c5b8843 gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d ) diff --git a/eng/_util/go.sum b/eng/_util/go.sum index 1b46c2b6f8b..06f8bd92ba4 100644 --- a/eng/_util/go.sum +++ b/eng/_util/go.sum @@ -15,8 +15,8 @@ github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/microsoft/go-infra v0.0.0-20211206195537-520f2621bcf4 h1:goYnnCGbN9HBwd0CtdiokVAlq5rTQkqq66K43kvGnhc= -github.com/microsoft/go-infra v0.0.0-20211206195537-520f2621bcf4/go.mod h1:3IVGTm7qFJldQHximiWLg2kYfmugjZMGNHnvUo5Mo5M= +github.com/microsoft/go-infra v0.0.0-20220208232830-51df6c5b8843 h1:IAk1GrsBP2l3sdWnaARrLALS9m6fVkGNgefpWRS0x2c= +github.com/microsoft/go-infra v0.0.0-20220208232830-51df6c5b8843/go.mod h1:3IVGTm7qFJldQHximiWLg2kYfmugjZMGNHnvUo5Mo5M= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/eng/pipeline/jobs/builders-to-jobs.yml b/eng/pipeline/jobs/builders-to-jobs.yml index fb28e1517a0..05d7a751e9a 100644 --- a/eng/pipeline/jobs/builders-to-jobs.yml +++ b/eng/pipeline/jobs/builders-to-jobs.yml @@ -10,12 +10,15 @@ parameters: # If true, include a signing job that depends on all 'buildandpack' builder jobs finishing. This # lets us start the lengthy tasks of signing and testing in parallel. sign: false + # If true, generate source archive tarballs, and sign them if signing is enabled. + createSourceArchive: false jobs: - ${{ each builder in parameters.builders }}: - template: run-job.yml parameters: builder: ${{ builder }} + createSourceArchive: ${{ parameters.createSourceArchive }} - ${{ if eq(parameters.sign, true) }}: - template: sign-job.yml diff --git a/eng/pipeline/jobs/go-builder-matrix-jobs.yml b/eng/pipeline/jobs/go-builder-matrix-jobs.yml index 375d5e49f87..588287276bf 100644 --- a/eng/pipeline/jobs/go-builder-matrix-jobs.yml +++ b/eng/pipeline/jobs/go-builder-matrix-jobs.yml @@ -9,6 +9,7 @@ parameters: innerloop: false outerloop: false sign: false + createSourceArchive: false jobs: - template: shorthand-builders-to-builders.yml @@ -16,6 +17,7 @@ jobs: jobsTemplate: builders-to-jobs.yml jobsParameters: sign: ${{ parameters.sign }} + createSourceArchive: ${{ parameters.createSourceArchive }} shorthandBuilders: - ${{ if eq(parameters.innerloop, true) }}: - { os: linux, arch: amd64, config: buildandpack } diff --git a/eng/pipeline/jobs/run-job.yml b/eng/pipeline/jobs/run-job.yml index 89797a2b8c5..cd6546aaabb 100644 --- a/eng/pipeline/jobs/run-job.yml +++ b/eng/pipeline/jobs/run-job.yml @@ -7,6 +7,7 @@ parameters: # { id, os, arch, config, distro? } builder: {} + createSourceArchive: false jobs: - job: ${{ parameters.builder.id }} @@ -41,7 +42,6 @@ jobs: Write-Host "##vso[task.setvariable variable=GO_MAKE_MAX_RETRY_ATTEMPTS]5" displayName: Increase 'make' retry attempts - # Initialize stage 0 toolset ahead of time so we can track timing data separately from the # build operations. When we call this script again later, it won't download Go again. - pwsh: | @@ -49,26 +49,23 @@ jobs: Get-Stage0GoRoot displayName: Init stage 0 Go toolset + - template: ../steps/init-submodule-task.yml + + # Create the source archive on one job only. The os choice is arbitrary. + - ${{ if and(eq(parameters.createSourceArchive, true), eq(parameters.builder.config, 'buildandpack'), eq(parameters.builder.os, 'linux')) }}: + - pwsh: | + git config --global user.name "microsoft-golang-bot" + git config --global user.email "microsoft-golang-bot@users.noreply.github.com" + + # Turn the patches into commits, so HEAD includes the changes. + eng/run.ps1 submodule-refresh -commits + eng/run.ps1 pack-source + displayName: Archive submodule source + - pwsh: | - function fetch_submodule() { - eng/run.ps1 submodule-refresh -shallow @args - } - - if ("$env:FETCH_BEARER_TOKEN") { - fetch_submodule ` - -origin 'https://dnceng@dev.azure.com/dnceng/internal/_git/microsoft-go-mirror' ` - -fetch-bearer-token $env:FETCH_BEARER_TOKEN - } else { - fetch_submodule - } - # If non-public, use access token to fetch from repo. If public, don't use the access token, - # because anonymous auth is fine. - ${{ if ne(variables['System.TeamProject'], 'public') }}: - env: - FETCH_BEARER_TOKEN: $(System.AccessToken) - displayName: Set up submodule from internal mirror - ${{ if eq(variables['System.TeamProject'], 'public') }}: - displayName: Set up submodule + # Apply the patches as staged changes, so the HEAD commit is the same as upstream. + eng/run.ps1 submodule-refresh + displayName: Apply patches # Use build script directly for "buildandpack". If we used run-builder, we would need to # download its external module dependencies. diff --git a/eng/pipeline/rolling-internal-pipeline.yml b/eng/pipeline/rolling-internal-pipeline.yml index f459b740ecf..1ec8425471e 100644 --- a/eng/pipeline/rolling-internal-pipeline.yml +++ b/eng/pipeline/rolling-internal-pipeline.yml @@ -20,6 +20,7 @@ stages: parameters: innerloop: true sign: true + createSourceArchive: true - stage: Publish dependsOn: Build @@ -37,6 +38,7 @@ stages: steps: - template: steps/checkout-unix-task.yml - template: steps/init-pwsh-task.yml + - template: steps/init-submodule-task.yml - pwsh: | function TrimStart($s, $prefix) { @@ -68,7 +70,7 @@ stages: - pwsh: | eng/run.ps1 createbuildassetjson ` -artifacts-dir '$(Pipeline.Workspace)/Binaries Signed/' ` - -source-dir '$(Build.SourcesDirectory)' ` + -source-dir '$(Build.SourcesDirectory)/go' ` -destination-url '$(blobDestinationUrl)' ` -branch '$(PublishBranchAlias)' ` -o '$(Pipeline.Workspace)/Binaries Signed/assets.json' diff --git a/eng/pipeline/steps/init-submodule-task.yml b/eng/pipeline/steps/init-submodule-task.yml new file mode 100644 index 00000000000..8e370d3b8ab --- /dev/null +++ b/eng/pipeline/steps/init-submodule-task.yml @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Shallow checkout sources on Windows +steps: + - pwsh: | + function fetch_submodule() { + eng/run.ps1 submodule-refresh -shallow -skip-patch @args + } + + if ("$env:FETCH_BEARER_TOKEN") { + fetch_submodule ` + -origin 'https://dnceng@dev.azure.com/dnceng/internal/_git/microsoft-go-mirror' ` + -fetch-bearer-token $env:FETCH_BEARER_TOKEN + } else { + fetch_submodule + } + # If non-public, use access token to fetch from repo. If public, don't use the access token, + # because anonymous auth is fine. + ${{ if ne(variables['System.TeamProject'], 'public') }}: + env: + FETCH_BEARER_TOKEN: $(System.AccessToken) + displayName: Set up submodule from internal mirror + ${{ if eq(variables['System.TeamProject'], 'public') }}: + displayName: Set up submodule diff --git a/patches/0001-cmd-dist-add-JSON-output-support-for-some-tests.patch b/patches/0001-cmd-dist-add-JSON-output-support-for-some-tests.patch index a9bcff182ae..d2fb5d83ff4 100644 --- a/patches/0001-cmd-dist-add-JSON-output-support-for-some-tests.patch +++ b/patches/0001-cmd-dist-add-JSON-output-support-for-some-tests.patch @@ -1,4 +1,4 @@ -From be418f20052ddea746451bafe21fdbf2bb1c915d Mon Sep 17 00:00:00 2001 +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 4 Jan 2022 11:23:27 -0600 Subject: [PATCH] cmd/dist: add JSON output support for some tests @@ -17,7 +17,7 @@ in JSON format, and the ordinary logs are still important. 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go -index 50a2e5936c..0436c63063 100644 +index f40fa926df..f76ecbe28c 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -29,6 +29,7 @@ func cmdtest() { @@ -36,7 +36,7 @@ index 50a2e5936c..0436c63063 100644 rebuild bool failed bool keepGoing bool -@@ -294,9 +296,13 @@ func short() string { +@@ -286,9 +288,13 @@ func short() string { // Callers should use goTest and then pass flags overriding these // defaults as later arguments in the command line. func (t *tester) goTest() []string { @@ -51,7 +51,7 @@ index 50a2e5936c..0436c63063 100644 } func (t *tester) tags() string { -@@ -379,6 +385,9 @@ func (t *tester) registerStdTest(pkg string, useG3 bool) { +@@ -371,6 +377,9 @@ func (t *tester) registerStdTest(pkg string, useG3 bool) { t.timeout(timeoutSec), "-gcflags=all=" + gcflags, }