diff --git a/Makefile.buildx b/Makefile.buildx index 9b99031954..957d04b698 100644 --- a/Makefile.buildx +++ b/Makefile.buildx @@ -37,6 +37,9 @@ WORMHOLE_IMG ?= quay.io/gravitational/wormhole:0.3.3 SELINUX_VERSION := 6.0.0 SELINUX_REPO := git@github.com:a-palchikov/selinux.git SELINUX_BRANCH := dmitri/buildx +#TODO(dima): update once the https://github.com/gravitational/selinux/pull/16 has been merged +#SELINUX_REPO := git@github.com:gravitational/selinux.git +#SELINUX_BRANCH := distro/centos_rhel/7 # Set to non-empty value to trigger streaming builder progress CI ?= diff --git a/build.go b/build.go index e86295960e..6b7823d46b 100644 --- a/build.go +++ b/build.go @@ -19,8 +19,11 @@ limitations under the License. package main import ( - // mage:import + //mage:import "github.com/gravitational/gravity/mage" + + //mage:import + _ "github.com/gravitational/magnet/common" ) // Default specifies the default build target diff --git a/go.mod b/go.mod index d44bc06103..3b86908a5b 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/gravitational/go-vhost v0.0.0-20171024163855-94d0c42e3263 github.com/gravitational/kingpin v2.1.11-0.20160205192003-785686550a08+incompatible // indirect github.com/gravitational/license v0.0.0-20171013193735-f3111b1818ce - github.com/gravitational/magnet v0.2.6 + github.com/gravitational/magnet v0.2.7-0.20210609203954-0a59f53f530e github.com/gravitational/oxy v0.0.0-20180629203109-e4a7e35311e6 // indirect github.com/gravitational/rigging v0.0.0-20210330165731-4602e892394d github.com/gravitational/roundtrip v1.0.0 @@ -146,14 +146,13 @@ replace ( github.com/google/uuid => github.com/google/uuid v1.1.0 github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.0.0-20181207171349-d3bcea3cf97e github.com/gorilla/mux => github.com/gorilla/mux v1.7.0 - github.com/gravitational/magnet => github.com/a-palchikov/magnet v0.2.7-0.20210520113349-2a5f8ce635ab github.com/imdario/mergo => github.com/imdario/mergo v0.0.0-20160517064435-50d4dbd4eb0e github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305 github.com/json-iterator/go => github.com/json-iterator/go v1.1.5 github.com/julienschmidt/httprouter => github.com/julienschmidt/httprouter v1.1.0 github.com/kr/pty => github.com/kr/pty v1.0.0 github.com/kylelemons/godebug => github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb - github.com/magefile/mage => github.com/a-palchikov/mage v1.9.1-0.20210510142914-0614fcadda0b + github.com/magefile/mage => github.com/knisbet/mage v1.9.1-0.20210609142646-749a704341ac github.com/mattn/go-isatty => github.com/mattn/go-isatty v0.0.3 github.com/mattn/go-runewidth => github.com/mattn/go-runewidth v0.0.2-0.20161012013512-737072b4e32b github.com/miekg/dns => github.com/miekg/dns v1.0.4 diff --git a/go.sum b/go.sum index 4f723d0000..b354763fb0 100644 --- a/go.sum +++ b/go.sum @@ -95,10 +95,6 @@ github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:H github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/a-palchikov/mage v1.9.1-0.20210510142914-0614fcadda0b h1:4zD3qgA1LWkxwXN2p8U6set5w5ZIAWWlDy78CTb2FIg= -github.com/a-palchikov/mage v1.9.1-0.20210510142914-0614fcadda0b/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/a-palchikov/magnet v0.2.7-0.20210520113349-2a5f8ce635ab h1:8ZTjoU8XFETBN/LHo3ELgdRPLeMzKdoiQ+VrqoseR1I= -github.com/a-palchikov/magnet v0.2.7-0.20210520113349-2a5f8ce635ab/go.mod h1:I168G8xokpoH32modDL8a0Ht2MWQyU+DCfcc6q+bVIQ= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= @@ -551,6 +547,8 @@ github.com/gravitational/log v0.0.0-20200127200505-fdffa14162b0 h1:gxT3g2wRixRJ2 github.com/gravitational/log v0.0.0-20200127200505-fdffa14162b0/go.mod h1:zJJqcfDBGf9G5qdBzs0Ai3FDwIvmvzl16nlClqcJyy4= github.com/gravitational/logrus v1.4.3 h1:gdyMN/zLd4R/kATybKzG1I6gjrbCqNJOl1XiixdT5e4= github.com/gravitational/logrus v1.4.3/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= +github.com/gravitational/magnet v0.2.7-0.20210609203954-0a59f53f530e h1:yDDoKxGE9kpId7555BP1IgjikZFLX2h/6yA3MomGMDw= +github.com/gravitational/magnet v0.2.7-0.20210609203954-0a59f53f530e/go.mod h1:1hC1lZ075A/syd4/xlrq5eVAGzleRcnDG9xegcxZ7Ts= github.com/gravitational/moby v1.4.2-0.20191008111026-2adf434ca696 h1:GI3Wyc/dLHL8snw18nLrSWTyKqMHF+pB8APFw0bQe3I= github.com/gravitational/moby v1.4.2-0.20191008111026-2adf434ca696/go.mod h1:bDzK0jA4s+TisyeqAiirQQULXKzSLYhmAd3NHeVgaHs= github.com/gravitational/oxy v0.0.0-20180629203109-e4a7e35311e6 h1:244Hc0XnOrqZxR0Fbwt9nwlvM5HnqKWJE+r5EdG6v4A= @@ -707,6 +705,8 @@ github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM52 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/knisbet/mage v1.9.1-0.20210609142646-749a704341ac h1:MOsgxjsdw6rKsxuCKmS8RIVLLPwDctvC1leKhYPl9cE= +github.com/knisbet/mage v1.9.1-0.20210609142646-749a704341ac/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= diff --git a/mage/help.go b/mage/help.go deleted file mode 100644 index 75a3a92a99..0000000000 --- a/mage/help.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2021 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mage - -import ( - "os" - - "github.com/gravitational/magnet/common" - - "github.com/magefile/mage/mg" -) - -type Help mg.Namespace - -// Envs outputs the current environment configuration -func (Help) Envs() (err error) { - m := root.Target("help:envs") - defer func() { m.Complete(err) }() - - return common.WriteEnvs(root.Env(), os.Stdout) -} diff --git a/mage/misc.go b/mage/misc.go index 7ea08b92d1..d1a2669f1b 100644 --- a/mage/misc.go +++ b/mage/misc.go @@ -42,15 +42,6 @@ func Clean() (err error) { return trace.Wrap(os.RemoveAll(root.buildDir)) } -// Env outputs the list of imported environment variables -func Env() (err error) { - for k, v := range root.Config.ImportEnv { - fmt.Println(k, ":", v) - } - - return -} - // IsUpToDate returns true iff all of the (source, sources...) are older or the same // age as dst. func IsUpToDate(dst string, source string, sources ...string) (uptodate bool) { diff --git a/mage/vars.go b/mage/vars.go index bdf89385b5..578fe61ba1 100644 --- a/mage/vars.go +++ b/mage/vars.go @@ -17,43 +17,48 @@ limitations under the License. package mage import ( + "bytes" "fmt" + "os" "os/exec" "path/filepath" - "strings" "github.com/gravitational/magnet" "github.com/gravitational/trace" "github.com/coreos/go-semver/semver" - log "github.com/sirupsen/logrus" ) -var env = importEnvFromMakefile() - var root = mustRoot(magnet.Config{ - Version: env.getBuildVersion(), + Version: buildVersion, PrintConfig: true, - LogDir: fmt.Sprintf("%s/logs", env.getBuildDir()), - CacheDir: fmt.Sprintf("%s/cache", env.getBuildDir()), - ImportEnv: env, - PlainProgress: env.isPlainProgress(), -}, env.getBuildDir()) + LogDir: fmt.Sprintf("%s/logs", buildDir), + CacheDir: fmt.Sprintf("%s/cache", buildDir), + PlainProgress: isPlainProgress(), +}, buildDir) var ( + env = magnet.NewEnviron(importEnvFromMakefile) // Go version - golangVersion = root.E(magnet.EnvVar{ + golangVersion = env.E(magnet.EnvVar{ Key: "GOLANG_VER", Short: "The version of Go from container: quay.io/gravitational/debian-venti:go${GOLANG_VER}", }) + // Build directory + buildDir = env.E(magnet.EnvVar{ + Key: "BUILDDIR", + Default: "_build", + Short: "Directory to store all build artefacts", + }) + // golangciVersion is the version of golangci-lint to use for linting // https://github.com/golangci/golangci-lint/releases golangciVersion = "v1.39.0" // FIO vars - fioVersion = root.E(magnet.EnvVar{ + fioVersion = env.E(magnet.EnvVar{ Key: "FIO_VER", Short: "The version of fio to include for volume IO testing", }) @@ -61,23 +66,23 @@ var ( fioPkgTag = fmt.Sprintf("%v.0", fioVersion) // Teleport - teleportTag = root.E(magnet.EnvVar{ + teleportTag = env.E(magnet.EnvVar{ Key: "TELEPORT_TAG", Short: "The teleport tag to build and include with gravity", }) teleportRepoTag = fmt.Sprintf("v%s", teleportTag) // Adapts teleportTag to the teleport tagging scheme // Grpc - grpcProtocVersion = root.E(magnet.EnvVar{ + grpcProtocVersion = env.E(magnet.EnvVar{ Key: "GRPC_PROTOC_VER", Short: "The protoc version to use", }) grpcProtocPlatform = "linux-x86_64" - grpcGoGoTag = root.E(magnet.EnvVar{ + grpcGoGoTag = env.E(magnet.EnvVar{ Key: "GOGO_PROTO_TAG", Short: "The grpc gogo version to use", }) - grpcGatewayTag = root.E(magnet.EnvVar{ + grpcGatewayTag = env.E(magnet.EnvVar{ Key: "GRPC_GATEWAY_TAG", Short: "The grpc gateway version to use", }) @@ -85,7 +90,7 @@ var ( // internal repos // BuildVersion - buildVersion = root.E(magnet.EnvVar{ + buildVersion = env.E(magnet.EnvVar{ Key: "BUILD_VERSION", Default: magnet.DefaultVersion(), Short: "The version to assign when building artifacts", @@ -94,7 +99,7 @@ var ( // Planet // k8sVersion is the version of kubernetes we're shipping - k8sVersion = root.E(magnet.EnvVar{ + k8sVersion = env.E(magnet.EnvVar{ Key: "K8S_VER", Short: "The k8s version to use (and locate the planet tag)", }) @@ -105,88 +110,88 @@ var ( //planetTag = fmt.Sprintf("7.1.4-%v", k8sVersionToPlanetFormat(k8sVersion)) planetTag = "" - planetBranch = root.E(magnet.EnvVar{ + planetBranch = env.E(magnet.EnvVar{ Key: "PLANET_BRANCH", Default: planetTag, Short: "Alternate branch to build planet", }) - planetVersion = root.E(magnet.EnvVar{ + planetVersion = env.E(magnet.EnvVar{ Key: "PLANET_TAG", Default: planetTag, Short: "Planet application tag/branch to build", }) // Gravity Internal Applications - appIngressVersion = root.E(magnet.EnvVar{ + appIngressVersion = env.E(magnet.EnvVar{ Key: "INGRESS_APP_VERSION", Short: "Ingress application - version to assign to internal application", }) - appIngressBranch = root.E(magnet.EnvVar{ + appIngressBranch = env.E(magnet.EnvVar{ Key: "INGRESS_APP_BRANCH", Default: appIngressVersion, Short: "Ingress application - tag/branch to build the application from on upstream repo", }) - appIngressRepo = root.E(magnet.EnvVar{ + appIngressRepo = env.E(magnet.EnvVar{ Key: "INGRESS_APP_REPO", Default: "https://github.com/gravitational/ingress-app", Short: "Ingress application - public repository to pull the application sources from for build", }) - appStorageVersion = root.E(magnet.EnvVar{ + appStorageVersion = env.E(magnet.EnvVar{ Key: "STORAGE_APP_VERSION", Short: "Storage application - version to assign to internal application", }) - appStorageBranch = root.E(magnet.EnvVar{ + appStorageBranch = env.E(magnet.EnvVar{ Key: "STORAGE_APP_BRANCH", Default: appStorageVersion, Short: "Storage application - tag/branch to build the application from on upstream repo", }) - appStorageRepo = root.E(magnet.EnvVar{ + appStorageRepo = env.E(magnet.EnvVar{ Key: "STORAGE_APP_REPO", Default: "https://github.com/gravitational/storage-app", Short: "Storage application - public repository to pull the application sources from for build", }) - appLoggingVersion = root.E(magnet.EnvVar{ + appLoggingVersion = env.E(magnet.EnvVar{ Key: "LOGGING_APP_VERSION", Short: "Logging application - version to assign to internal application", }) - appLoggingBranch = root.E(magnet.EnvVar{ + appLoggingBranch = env.E(magnet.EnvVar{ Key: "LOGGING_APP_BRANCH", Default: appLoggingVersion, Short: "Logging application - tag/branch to build the application from on upstream repo", }) - appLoggingRepo = root.E(magnet.EnvVar{ + appLoggingRepo = env.E(magnet.EnvVar{ Key: "LOGGING_APP_REPO", Default: "https://github.com/gravitational/logging-app", Short: "Storage application - public repository to pull the application sources from for build", }) - appMonitoringVersion = root.E(magnet.EnvVar{ + appMonitoringVersion = env.E(magnet.EnvVar{ Key: "MONITORING_APP_VERSION", Short: "Monitoring application - version to assign to internal application", }) - appMonitoringBranch = root.E(magnet.EnvVar{ + appMonitoringBranch = env.E(magnet.EnvVar{ Key: "MONITORING_APP_BRANCH", Default: appMonitoringVersion, Short: "Monitoring application - tag/branch to build the application from on upstream repo", }) - appMonitoringRepo = root.E(magnet.EnvVar{ + appMonitoringRepo = env.E(magnet.EnvVar{ Key: "MONITORING_APP_REPO", Default: "https://github.com/gravitational/monitoring-app", Short: "Monitoring application - public repository to pull the application sources from for build", }) - appBandwagonVersion = root.E(magnet.EnvVar{ + appBandwagonVersion = env.E(magnet.EnvVar{ Key: "BANDWAGON_APP_TAG", Short: "Bandwagon application - version to assign to internal application", }) - appBandwagonBranch = root.E(magnet.EnvVar{ + appBandwagonBranch = env.E(magnet.EnvVar{ Key: "BANDWAGON_APP_BRANCH", Default: appBandwagonVersion, Short: "Bandwagon application - tag/branch to build the application from on upstream repo", }) - appBandwagonRepo = root.E(magnet.EnvVar{ + appBandwagonRepo = env.E(magnet.EnvVar{ Key: "BANDWAGON_APP_REPO", Default: "https://github.com/gravitational/bandwagon", Short: "Bandwagon application - public repository to pull the application sources from for build", @@ -194,74 +199,74 @@ var ( // applications within the gravity master repository - appDNSVersion = root.E(magnet.EnvVar{ + appDNSVersion = env.E(magnet.EnvVar{ Key: "DNS_APP_VERSION", Short: "DNS application - version to assign to internal application", }) - appRBACVersion = root.E(magnet.EnvVar{ + appRBACVersion = env.E(magnet.EnvVar{ Key: "RBAC_APP_TAG", Default: buildVersion, Short: "Logging application tag/branch to build", }) - appTillerVersion = root.E(magnet.EnvVar{ + appTillerVersion = env.E(magnet.EnvVar{ Key: "TILLER_APP_TAG", Short: "Logging application tag/branch to build", }) // Dependency Versions - tillerVersion = root.E(magnet.EnvVar{ + tillerVersion = env.E(magnet.EnvVar{ Key: "TILLER_VERSION", Short: "Tiller version to include", }) - selinuxVersion = root.E(magnet.EnvVar{ + selinuxVersion = env.E(magnet.EnvVar{ Key: "SELINUX_VERSION", Short: "", }) - selinuxBranch = root.E(magnet.EnvVar{ + selinuxBranch = env.E(magnet.EnvVar{ Key: "SELINUX_BRANCH", Default: "distro/centos_rhel/7", Short: "", }) - selinuxRepo = root.E(magnet.EnvVar{ + selinuxRepo = env.E(magnet.EnvVar{ Key: "SELINUX_REPO", Default: "git@github.com:gravitational/selinux.git", Short: "", }) // which container to include for builds using wormhole networking - wormholeImage = root.E(magnet.EnvVar{ + wormholeImage = env.E(magnet.EnvVar{ Key: "WORMHOLE_IMG", Short: "ImagePath to wormhole docker container", }) // Image Vulnerability Scanning on Publishing - scanCopyToRegistry = root.E(magnet.EnvVar{ + scanCopyToRegistry = env.E(magnet.EnvVar{ Key: "TELE_COPY_TO_REGISTRY", Default: "quay.io/gravitational", Short: "Registry /to upload container to for scanning", }) - scanCopyToRepository = root.E(magnet.EnvVar{ + scanCopyToRepository = env.E(magnet.EnvVar{ Key: "TELE_COPY_TO_REPOSITORY", Default: "gravitational/gravity-scan", Short: "The repository on the registry server to use /", }) - scanCopyToPrefix = root.E(magnet.EnvVar{ + scanCopyToPrefix = env.E(magnet.EnvVar{ Key: "TELE_COPY_TO_PREFIX", Default: buildVersion, Short: "The prefix to add to each image name when uploading to the registry", }) - scanCopyToUser = root.E(magnet.EnvVar{ + scanCopyToUser = env.E(magnet.EnvVar{ Key: "TELE_COPY_TO_USER", Short: "User to use with the registry", }) - scanCopyToPassword = root.E(magnet.EnvVar{ + scanCopyToPassword = env.E(magnet.EnvVar{ Key: "TELE_COPY_TO_PASS", Short: "Password for the registry", Secret: true, }) // Publishing - distributionOpsCenter = root.E(magnet.EnvVar{ + distributionOpsCenter = env.E(magnet.EnvVar{ Key: "DISTRIBUTION_OPSCENTER", Default: "https://get.gravitational.io", Short: "Address of OpsCenter used to publish gravity enterprise artifacts to", @@ -287,40 +292,24 @@ func buildFlags() []string { } } -func importEnvFromMakefile() (env environ) { - env = make(environ) +func importEnvFromMakefile() (env map[string]string) { cmd := exec.Command("make", "-f", "Makefile.buildx", "magnet-vars") out, err := cmd.CombinedOutput() if err != nil { - log.WithError(err).Warn("Failed to import environ from makefile.") - return nil + panic(fmt.Sprint("failed to import environ from makefile:", err)) } - for _, line := range strings.Split(string(out), "\n") { - cols := strings.SplitN(line, "=", 2) - if len(cols) != 2 || !strings.HasPrefix(cols[0], "MAGNET_") { - log.Debug("Skip line that does not look like magnet envar.") - continue - } - key, value := strings.TrimPrefix(cols[0], "MAGNET_"), cols[1] - env[key] = value + env, err = magnet.ImportEnvFromReader(bytes.NewReader(out)) + if err != nil { + panic(fmt.Sprint("failed to import environ from makefile:", err)) } return env } -func (r environ) getBuildDir() string { - return r["BUILDDIR"] +func isPlainProgress() *bool { + enabled := os.Getenv("CI") != "" + return &enabled } -func (r environ) getBuildVersion() string { - return r["BUILD_VERSION"] -} - -func (r environ) isPlainProgress() bool { - return r["CI"] != "" -} - -type environ map[string]string - func mustRoot(config magnet.Config, buildDir string) *rootTarget { root, err := magnet.Root(config) if err != nil { diff --git a/vendor/github.com/gravitational/magnet/common/common.go b/vendor/github.com/gravitational/magnet/common/common.go index 49d4c76f66..77607120ef 100644 --- a/vendor/github.com/gravitational/magnet/common/common.go +++ b/vendor/github.com/gravitational/magnet/common/common.go @@ -14,31 +14,34 @@ limitations under the License. package common import ( - "io" + "os" "sort" "github.com/gravitational/magnet" + "github.com/magefile/mage/mg" "github.com/olekukonko/tablewriter" ) -// WriteEnvs outputs environment variables that can override build options to w -func WriteEnvs(environ map[string]magnet.EnvVar, w io.Writer) error { +// Help defines the utility namespace for help targets +type Help mg.Namespace + +// Envs outputs the current environment configuration +func (Help) Envs() (err error) { var result [][]string - for key, value := range environ { + for key, value := range magnet.Env() { if value.Secret { result = append(result, []string{key, "", "", value.Short}) } else { result = append(result, []string{key, value.Value, value.Default, value.Short}) } } - sort.SliceStable(result, func(i, j int) bool { return result[i][0] < result[j][0] }) - table := tablewriter.NewWriter(w) + table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Env", "Value", "Default", "Short Description"}) table.SetBorder(false) table.SetAutoWrapText(false) diff --git a/vendor/github.com/gravitational/magnet/dl.go b/vendor/github.com/gravitational/magnet/dl.go index bd43685a66..64a571577f 100644 --- a/vendor/github.com/gravitational/magnet/dl.go +++ b/vendor/github.com/gravitational/magnet/dl.go @@ -1,6 +1,7 @@ package magnet import ( + "context" "crypto/sha256" "fmt" "io" @@ -22,31 +23,32 @@ type downloadMetadata struct { SHA2Sum string } -// Download begins a download of a url but doesn't block. +// DownloadFuture begins a download of a url but doesn't block. // Returns a future that when called will block until it can return the path to the file on disk or an error. -func (m *MagnetTarget) DownloadFuture(url string) func() (url string, path string, err error) { - errC := make(chan error, 1) - - var path string +func (m *MagnetTarget) DownloadFuture(ctx context.Context, url string) DownloadFutureFunc { + type result struct { + path string + err error + } + resultC := make(chan result, 1) go func() { - p, err := m.Download(url) - path = p - errC <- err + p, err := m.Download(ctx, url) + resultC <- result{path: p, err: err} }() - return func() (string, string, error) { - err := <-errC - if err != nil { - return url, "", trace.Wrap(err) + return func() (url, path string, err error) { + result := <-resultC + if result.err != nil { + return url, "", trace.Wrap(result.err) } - return url, path, nil + return url, result.path, nil } } // Download will download a file from a remote URL. It's optimized for working with a local cache, and will send // request headers to the upstream server and only download the file if cached or missing from the local cache. -func (m *MagnetTarget) Download(url string) (path string, err error) { +func (m *MagnetTarget) Download(ctx context.Context, url string) (path string, err error) { progress := dlProgressWriter{ m: m, url: url, @@ -66,7 +68,7 @@ func (m *MagnetTarget) Download(url string) (path string, err error) { metadata = downloadMetadata{} } - resp, err := httpGetRequest(url, metadata.ETag) + resp, err := httpGetRequest(ctx, url, metadata.ETag) if err != nil { return "", trace.Wrap(err) } @@ -129,6 +131,9 @@ func (m *MagnetTarget) Download(url string) (path string, err error) { return path, nil } +// DownloadFutureFunc defines the function type returned from the DownloadFuture API +type DownloadFutureFunc func() (url, path string, err error) + type dlProgressWriter struct { m *MagnetTarget url string @@ -190,13 +195,14 @@ func (d *dlProgressWriter) Write(data []byte) (int, error) { return len(data), nil } -func httpGetRequest(url, etag string) (*http.Response, error) { +func httpGetRequest(ctx context.Context, url, etag string) (*http.Response, error) { client := &http.Client{} req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, trace.Wrap(err) } + req = req.WithContext(ctx) if etag != "" { req.Header.Add("If-None-Match", etag) diff --git a/vendor/github.com/gravitational/magnet/docker.go b/vendor/github.com/gravitational/magnet/docker.go index 69ad251480..67381fe0e1 100644 --- a/vendor/github.com/gravitational/magnet/docker.go +++ b/vendor/github.com/gravitational/magnet/docker.go @@ -382,9 +382,9 @@ func (m *DockerConfigRun) SetRemove(remove bool) *DockerConfigRun { return m } -// AddVolume attaches a filesystem mount to the container. -func (m *DockerConfigRun) AddVolume(volume DockerBindMount) *DockerConfigRun { - m.Volumes = append(m.Volumes, volume) +// AddVolume attaches a set of filesystem mounts to the container. +func (m *DockerConfigRun) AddVolume(volumes ...DockerBindMount) *DockerConfigRun { + m.Volumes = append(m.Volumes, volumes...) return m } diff --git a/vendor/github.com/gravitational/magnet/env.go b/vendor/github.com/gravitational/magnet/env.go index 698ffbcda8..de33df8892 100644 --- a/vendor/github.com/gravitational/magnet/env.go +++ b/vendor/github.com/gravitational/magnet/env.go @@ -1,8 +1,17 @@ package magnet import ( + "bufio" + "bytes" "fmt" + "io" + "log" "os" + "os/exec" + "strings" + "sync" + + "github.com/gravitational/trace" ) // EnvVar represents a configuration with optional defaults @@ -21,31 +30,66 @@ type EnvVar struct { // given to previously imported environment variables. // If the variable was not previously imported and no value // has been specified, the default is returned -func (m *Magnet) E(e EnvVar) string { +func E(e EnvVar) string { + return env.E(e) +} + +// MustGetEnv returns the value of the environment variable given with key. +// The variable is assumed to have been registered either with E or +// imported from existing environment - otherwise the function will panic. +// For non-panicking version use GetEnv +func MustGetEnv(key string) (value string) { + return env.MustGetEnv(key) +} + +// GetEnv returns the value of the environment variable given with key. +// The variable is assumed to have been registered either with E or +// imported from existing environment +func GetEnv(key string) (value string, exists bool) { + return env.GetEnv(key) +} + +// Env returns the complete environment +func Env() map[string]EnvVar { + return env.Env() +} + +// NewEnviron creates a new configuration environment +func NewEnviron(importer EnvImporterFunc) *Environ { + env = newEnviron(importer) + return env +} + +// E defines a new environment variable specified with e. +// Returns the current value of the variable with precedence +// given to previously imported environment variables. +// If the variable was not previously imported and no value +// has been specified, the default is returned +func (r *Environ) E(e EnvVar) string { if e.Key == "" { - panic("Key shouldn't be empty") + panic("key shouldn't be empty") } - if e.Secret && len(e.Default) > 0 { - panic("Secrets shouldn't be embedded with defaults") + panic("secrets shouldn't be embedded with defaults") } - if v, ok := m.ImportEnv[e.Key]; ok { + r.importOnce() + if v, ok := r.imported[e.Key]; ok { e.Value = v } else { e.Value = os.Getenv(e.Key) } - m.env[e.Key] = e + r.env[e.Key] = e - return m.MustGetEnv(e.Key) + return r.MustGetEnv(e.Key) } // MustGetEnv returns the value of the environment variable given with key. // The variable is assumed to have been registered either with E or // imported from existing environment - otherwise the function will panic. // For non-panicking version use GetEnv -func (m *Magnet) MustGetEnv(key string) (value string) { - if v, ok := m.GetEnv(key); ok { +func (r *Environ) MustGetEnv(key string) (value string) { + if v, ok := r.GetEnv(key); ok { return v } panic(fmt.Sprintf("Requested environment variable %q hasn't been registered", key)) @@ -54,9 +98,10 @@ func (m *Magnet) MustGetEnv(key string) (value string) { // GetEnv returns the value of the environment variable given with key. // The variable is assumed to have been registered either with E or // imported from existing environment -func (m *Magnet) GetEnv(key string) (value string, exists bool) { +func (r *Environ) GetEnv(key string) (value string, exists bool) { + r.importOnce() var v EnvVar - if v, exists = m.env[key]; !exists { + if v, exists = r.env[key]; !exists { return "", false } if v.Value != "" { @@ -66,6 +111,100 @@ func (m *Magnet) GetEnv(key string) (value string, exists bool) { } // Env returns the complete environment -func (m *Magnet) Env() map[string]EnvVar { - return m.env +func (r *Environ) Env() map[string]EnvVar { + m := make(map[string]EnvVar, len(r.env)) + for key, value := range r.env { + def := value.Default + if def == "" { + def = r.imported[key] + } + value.Default = def + m[key] = value + } + return m +} + +// Environ represents the environment with configuration +type Environ struct { + // env specifies the builder's configuration from environment + env map[string]EnvVar + // imported optionally specifies environment overrides + imported map[string]string + importer EnvImporterFunc + once sync.Once +} + +// ImportEnvFromMakefile invokes `make` to generate configuration for this mage script. +// The makefile target is assumed to be named `magnet-vars`. +// Assumes the makefile is named `Makefile` +// +// The script outputs a set of environment variables prefixed with `MAGNET_` which +// are used as default values for the configuration variables defined by the script. +// Any errors are ignored since this is a best-effort operation. +func ImportEnvFromMakefile() (env map[string]string) { + cmd := exec.Command("make", "magnet-vars") + out, err := cmd.CombinedOutput() + if err != nil { + return nil + } + env, _ = ImportEnvFromReader(bytes.NewReader(out)) + return env +} + +// EnvImporterFunc defines a function type to import environment from external source +type EnvImporterFunc func() map[string]string + +// ImportEnvFromReader consumes configuration for this mage script from the specified reader. +// Expects the reader to produce a list of environment variables as key=value pairs with a single +// variable per line. +// Only the environment variables prefixed with `MAGNET_` are considered which +// are used as default values for the configuration variables defined by the script itself. +func ImportEnvFromReader(r io.Reader) (env map[string]string, err error) { + env = make(map[string]string) + + s := bufio.NewScanner(r) + for s.Scan() { + line := s.Text() + if strings.TrimSpace(line) == "" { + continue + } + cols := strings.SplitN(line, "=", 2) + if len(cols) != 2 || !strings.HasPrefix(cols[0], "MAGNET_") { + log.Printf("Skip line that does not look like magnet envar: %q\n", line) + continue + } + key, value := strings.TrimPrefix(cols[0], "MAGNET_"), cols[1] + env[key] = value + } + if s.Err() != nil { + return nil, trace.Wrap(s.Err()) + } + return env, nil } + +// env represents the default configuration environment +var env = newEnviron(ImportEnvFromMakefile) + +func newEnviron(importer EnvImporterFunc) *Environ { + return &Environ{ + importer: importer, + env: make(map[string]EnvVar), + } +} + +func (r *Environ) importOnce() { + r.once.Do(func() { + r.imported = r.importer() + }) +} + +var debianFrontend = E(EnvVar{ + Key: "DEBIAN_FRONTEND", + Short: "Set to noninteractive or stderr to null to enable non-interactive output", +}) + +var cacheDir = E(EnvVar{ + Key: "XDG_CACHE_HOME", + Short: "Location to store/cache build assets", + Default: "_build/cache", +}) diff --git a/vendor/github.com/gravitational/magnet/exec.go b/vendor/github.com/gravitational/magnet/exec.go index 34ca2fb1c4..966631823a 100644 --- a/vendor/github.com/gravitational/magnet/exec.go +++ b/vendor/github.com/gravitational/magnet/exec.go @@ -89,7 +89,7 @@ func (e *ExecConfig) Run(ctx context.Context, cmd string, args ...string) (bool, return ran, trace.Wrap(err) } -// Outout runs the provided command, returning the output +// Output runs the provided command, returning the output // Note: output / trace won't be present in magnet logs func Output(ctx context.Context, cmd string, args ...string) (string, error) { buf := &bytes.Buffer{} diff --git a/vendor/github.com/gravitational/magnet/go.mod b/vendor/github.com/gravitational/magnet/go.mod index 5229027cee..f8ead77221 100644 --- a/vendor/github.com/gravitational/magnet/go.mod +++ b/vendor/github.com/gravitational/magnet/go.mod @@ -6,7 +6,7 @@ replace ( github.com/containerd/containerd => github.com/containerd/containerd v1.3.1-0.20200512144102-f13ba8f2f2fd github.com/docker/docker => github.com/docker/docker v17.12.0-ce-rc1.0.20200310163718-4634ce647cf2+incompatible github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305 - github.com/magefile/mage => github.com/a-palchikov/mage v1.9.1-0.20210510142914-0614fcadda0b + github.com/magefile/mage => github.com/knisbet/mage v1.9.1-0.20210609142646-749a704341ac ) require ( diff --git a/vendor/github.com/gravitational/magnet/go.sum b/vendor/github.com/gravitational/magnet/go.sum index c7149e627a..c87eb562be 100644 --- a/vendor/github.com/gravitational/magnet/go.sum +++ b/vendor/github.com/gravitational/magnet/go.sum @@ -1,5 +1,3 @@ -github.com/a-palchikov/mage v1.9.1-0.20210510142914-0614fcadda0b h1:4zD3qgA1LWkxwXN2p8U6set5w5ZIAWWlDy78CTb2FIg= -github.com/a-palchikov/mage v1.9.1-0.20210510142914-0614fcadda0b/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/containerd/console v1.0.0 h1:fU3UuQapBs+zLJu82NhR11Rif1ny2zfMMAyPJzSN5tQ= github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -9,6 +7,8 @@ github.com/gravitational/trace v1.1.11 h1:c+saoho5rBdj5lPEGzfHaYjh4Do4d+jX7hh4Bg github.com/gravitational/trace v1.1.11/go.mod h1:RvdOUHE4SHqR3oXlFFKnGzms8a5dugHygGw1bqDstYI= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/knisbet/mage v1.9.1-0.20210609142646-749a704341ac h1:MOsgxjsdw6rKsxuCKmS8RIVLLPwDctvC1leKhYPl9cE= +github.com/knisbet/mage v1.9.1-0.20210609142646-749a704341ac/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= diff --git a/vendor/github.com/gravitational/magnet/golang.go b/vendor/github.com/gravitational/magnet/golang.go index 4ca214bc2e..1c6c1ce744 100644 --- a/vendor/github.com/gravitational/magnet/golang.go +++ b/vendor/github.com/gravitational/magnet/golang.go @@ -3,7 +3,6 @@ package magnet import ( "context" "fmt" - "go/build" "os" "path/filepath" "strings" @@ -12,12 +11,12 @@ import ( ) type GolangConfigCommon struct { - // BuildContainer is the container image to use when running go commands + // BuildContainer is the container image to use when running Go commands BuildContainer string - // GOOS to pass to the go compiler as an env variable + // GOOS to pass to the Go compiler as an env variable GOOS string - // GOARCH to pass to the go compiler as an env variable + // GOARCH to pass to the Go compiler as an env variable GOARCH string // Env is a set of environment variables to pass to the compiler @@ -66,6 +65,9 @@ type GolangConfigCommon struct { // Tags is a list of build tags to consider as satisified during the build Tags []string + + // Volumes lists additional docker bind mounts for container workflows + Volumes []DockerBindMount } func (m *GolangConfigCommon) genFlags() []string { @@ -136,11 +138,20 @@ type GolangConfigBuild struct { type BuildContainer struct { // Name identifies the container image Name string + // GOPath optionally overrides the gopath inside the container. + // + // If set, the module path inside the container (ContainerPath) + // will be automatically deduced as `GOPath/src/ModulePath`. + // The value of the attribute will be written as `GOPATH` envar + // inside the container. + // Also sets `GO111MODULE=auto` envar if unspecified. + GOPath string // HostPath optionally specifies the repository path on host. - // Defaults to working directory of the process if unspecified + // Defaults to working directory of the process if unspecified. HostPath string - // ContainerPath optionally specifies the repository path inside the container. - // Will be computed based on host's GOPATH configuration if unspecified + // ContainerPath optionally specifies the module path inside the container. + // If unspecified and GOPath is given, will be computed as described above. + // If unspecified and GOPath is empty, defaults to `/host`. ContainerPath string } @@ -156,13 +167,19 @@ func (m *GolangConfigBuild) cacheDir() (path string, err error) { func (m *MagnetTarget) GolangBuild() *GolangConfigBuild { return &GolangConfigBuild{ TrimPath: true, - target: m, + GolangConfigCommon: GolangConfigCommon{ + Env: make(map[string]string), + }, + target: m, } } // GolangTest returns a builder that can be used to run golang tests against a set of sources. func (m *MagnetTarget) GolangTest() *GolangConfigTest { return &GolangConfigTest{ + GolangConfigCommon: GolangConfigCommon{ + Env: make(map[string]string), + }, target: m, } } @@ -170,6 +187,9 @@ func (m *MagnetTarget) GolangTest() *GolangConfigTest { // GolangCover returns a builder that can be used to work with the coverage tool func (m *MagnetTarget) GolangCover() *GolangConfigCover { return &GolangConfigCover{ + GolangConfigCommon: GolangConfigCommon{ + Env: make(map[string]string), + }, target: m, } } @@ -206,13 +226,19 @@ func (m *GolangConfigBuild) AddLDFlags(flags []string) *GolangConfigBuild { return m } +// AddVolumes adds a set of docker bind mounts to the container configuration +func (m *GolangConfigBuild) AddVolumes(volumes ...DockerBindMount) *GolangConfigBuild { + m.Volumes = append(m.Volumes, volumes...) + return m +} + // AddGCFlag adds a flag to the go tool compile program. func (m *GolangConfigBuild) AddGCFlag(flag string) *GolangConfigBuild { m.GCFlags = append(m.GCFlags, flag) return m } -// AddGCCGOFlag adds a flag to pass to the gcc go compiler. +// AddGCCGOFlag adds a flag to pass to the GCC Go compiler. func (m *GolangConfigBuild) AddGCCGOFlag(flag string) *GolangConfigBuild { m.GCCGOFlags = append(m.GCCGOFlags, flag) return m @@ -229,7 +255,7 @@ const ( BuildModePlugin = "plugin" ) -// SetBuildMode sets the golang build mode (see go help buildmode). +// SetBuildMode sets the Go build mode (see go help buildmode). func (m *GolangConfigBuild) SetBuildMode(mode string) *GolangConfigBuild { m.BuildMode = mode return m @@ -273,10 +299,6 @@ func (m *GolangConfigBuild) SetTrimpath(b bool) *GolangConfigBuild { // SetEnv sets an environment variable on the build tools. func (m *GolangConfigBuild) SetEnv(key, value string) *GolangConfigBuild { - if m.Env == nil { - m.Env = make(map[string]string) - } - m.Env[key] = value return m @@ -284,10 +306,6 @@ func (m *GolangConfigBuild) SetEnv(key, value string) *GolangConfigBuild { // SetEnvs allows setting multiple environment variables on the build tools. func (m *GolangConfigBuild) SetEnvs(envs map[string]string) *GolangConfigBuild { - if m.Env == nil { - m.Env = make(map[string]string) - } - for key, value := range envs { m.Env[key] = value } @@ -297,18 +315,12 @@ func (m *GolangConfigBuild) SetEnvs(envs map[string]string) *GolangConfigBuild { // SetGOOS allows overriding the GOOS env to a specific value. func (m *GolangConfigBuild) SetGOOS(value string) *GolangConfigBuild { - if m.Env == nil { - m.Env = make(map[string]string) - } m.Env["GOOS"] = value return m } // SetGOARCH allows overriding the default architecture for the resulting binary. func (m *GolangConfigBuild) SetGOARCH(value string) *GolangConfigBuild { - if m.Env == nil { - m.Env = make(map[string]string) - } m.Env["GOARCH"] = value return m } @@ -324,6 +336,7 @@ func (m *GolangConfigBuild) SetBuildContainer(value string) *GolangConfigBuild { // directly, a docker container will be used to map the sources and run the build within the consistent image. func (m *GolangConfigBuild) SetBuildContainerConfig(config BuildContainer) *GolangConfigBuild { m.BuildContainer = config.Name + m.paths.gopath = config.GOPath m.paths.hostPath = config.HostPath m.paths.containerPath = config.ContainerPath return m @@ -339,7 +352,7 @@ func (m *GolangConfigBuild) Build(ctx context.Context, packages ...string) error } func (m *GolangConfigBuild) buildDocker(ctx context.Context, packages ...string) (err error) { - if err := m.paths.checkAndSetDefaults(); err != nil { + if err := m.paths.compute(m.target.root.ModulePath, m.Env); err != nil { return trace.Wrap(err) } @@ -354,7 +367,6 @@ func (m *GolangConfigBuild) buildDocker(ctx context.Context, packages ...string) SetGID(fmt.Sprint(os.Getgid())). SetEnv("XDG_CACHE_HOME", "/cache"). SetEnv("GOCACHE", "/cache/go"). - SetEnv("GOPATH", m.paths.gopath). SetEnvs(m.Env). SetWorkDir(m.paths.containerPath). AddVolume(DockerBindMount{ @@ -366,7 +378,8 @@ func (m *GolangConfigBuild) buildDocker(ctx context.Context, packages ...string) Source: cacheDir, Destination: "/cache", Consistency: "delegated", - }) + }). + AddVolume(m.Volumes...) gocmd := m.buildCmd(packages...) @@ -431,7 +444,7 @@ func (m *GolangConfigTest) Test(ctx context.Context, packages ...string) error { } func (m *GolangConfigTest) testDocker(ctx context.Context, packages ...string) error { - if err := m.paths.checkAndSetDefaults(); err != nil { + if err := m.paths.compute(m.target.root.ModulePath, m.Env); err != nil { return trace.Wrap(err) } @@ -446,7 +459,6 @@ func (m *GolangConfigTest) testDocker(ctx context.Context, packages ...string) e SetGID(fmt.Sprint(os.Getgid())). SetEnv("XDG_CACHE_HOME", "/cache"). SetEnv("GOCACHE", "/cache/go"). - SetEnv("GOPATH", m.paths.gopath). SetEnvs(m.Env). SetWorkDir(m.paths.containerPath). AddVolume(DockerBindMount{ @@ -458,7 +470,8 @@ func (m *GolangConfigTest) testDocker(ctx context.Context, packages ...string) e Source: cacheDir, Destination: "/cache", Consistency: "delegated", - }) + }). + AddVolume(m.Volumes...) gocmd := m.buildCmd(packages...) @@ -516,7 +529,7 @@ func (m *GolangConfigTest) SetCount(count int) *GolangConfigTest { return m } -// AddTag adds a build tag for the golang compiler to consider during the build. +// AddTag adds a build tag for the Go compiler to consider during the build. func (m *GolangConfigTest) AddTag(tag string) *GolangConfigTest { m.Tags = append(m.Tags, tag) return m @@ -541,13 +554,19 @@ func (m *GolangConfigTest) AddLDFlags(flags []string) *GolangConfigTest { return m } +// AddVolumes adds a set of docker bind mounts to the container configuration +func (m *GolangConfigTest) AddVolumes(volumes ...DockerBindMount) *GolangConfigTest { + m.Volumes = append(m.Volumes, volumes...) + return m +} + // AddGCFlag adds a flag to the go tool compile program. func (m *GolangConfigTest) AddGCFlag(flag string) *GolangConfigTest { m.GCFlags = append(m.GCFlags, flag) return m } -// AddGCCGOFlag adds a flag to pass to the gcc go compiler. +// AddGCCGOFlag adds a flag to pass to the GCC Go compiler. func (m *GolangConfigTest) AddGCCGOFlag(flag string) *GolangConfigTest { m.GCCGOFlags = append(m.GCCGOFlags, flag) return m @@ -591,10 +610,6 @@ func (m *GolangConfigTest) SetRebuild(b bool) *GolangConfigTest { // SetEnv sets an environment variable on the build tools. func (m *GolangConfigTest) SetEnv(key, value string) *GolangConfigTest { - if m.Env == nil { - m.Env = make(map[string]string) - } - m.Env[key] = value return m @@ -602,10 +617,6 @@ func (m *GolangConfigTest) SetEnv(key, value string) *GolangConfigTest { // SetEnvs allows setting multiple environment variables on the build tools. func (m *GolangConfigTest) SetEnvs(envs map[string]string) *GolangConfigTest { - if m.Env == nil { - m.Env = make(map[string]string) - } - for key, value := range envs { m.Env[key] = value } @@ -615,18 +626,12 @@ func (m *GolangConfigTest) SetEnvs(envs map[string]string) *GolangConfigTest { // SetGOOS allows overriding the GOOS env to a specific value. func (m *GolangConfigTest) SetGOOS(value string) *GolangConfigTest { - if m.Env == nil { - m.Env = make(map[string]string) - } m.Env["GOOS"] = value return m } // SetGOARCH allows overriding the default architecture for the resulting binary. func (m *GolangConfigTest) SetGOARCH(value string) *GolangConfigTest { - if m.Env == nil { - m.Env = make(map[string]string) - } m.Env["GOARCH"] = value return m } @@ -642,6 +647,7 @@ func (m *GolangConfigTest) SetBuildContainer(value string) *GolangConfigTest { // directly, a docker container will be used to map the sources and run the build within the consistent image. func (m *GolangConfigTest) SetBuildContainerConfig(config BuildContainer) *GolangConfigTest { m.BuildContainer = config.Name + m.paths.gopath = config.GOPath m.paths.hostPath = config.HostPath m.paths.containerPath = config.ContainerPath return m @@ -674,12 +680,13 @@ type GolangConfigCover struct { // directly, a docker container will be used to map the sources and run the tool within the consistent image. func (m *GolangConfigCover) SetBuildContainerConfig(config BuildContainer) *GolangConfigCover { m.BuildContainer = config.Name + m.paths.gopath = config.GOPath m.paths.hostPath = config.HostPath m.paths.containerPath = config.ContainerPath return m } -// SetCoverProfile sets the path to the cover profile previously generated +// SetProfile sets the path to the cover profile previously generated // with 'go test' func (m *GolangConfigCover) SetProfile(path string) *GolangConfigCover { m.profilePath = path @@ -702,18 +709,12 @@ func (m *GolangConfigCover) SetMode(mode string) *GolangConfigCover { // SetEnv sets an environment variable on the tool. func (m *GolangConfigCover) SetEnv(key, value string) *GolangConfigCover { - if m.Env == nil { - m.Env = make(map[string]string) - } m.Env[key] = value return m } // SetEnvs allows setting multiple environment variables on the tool. func (m *GolangConfigCover) SetEnvs(envs map[string]string) *GolangConfigCover { - if m.Env == nil { - m.Env = make(map[string]string) - } for key, value := range envs { m.Env[key] = value } @@ -737,7 +738,7 @@ func (m *GolangConfigCover) Run(ctx context.Context) error { // docker runs the coverage analysis inside a docker container func (m *GolangConfigCover) docker(ctx context.Context) error { - if err := m.paths.checkAndSetDefaults(); err != nil { + if err := m.paths.compute(m.target.root.ModulePath, m.Env); err != nil { return trace.Wrap(err) } cacheDir, err := m.cacheDir() @@ -750,7 +751,6 @@ func (m *GolangConfigCover) docker(ctx context.Context) error { SetGID(fmt.Sprint(os.Getgid())). SetEnv("XDG_CACHE_HOME", "/cache"). SetEnv("GOCACHE", "/cache/go"). - SetEnv("GOPATH", m.paths.gopath). SetEnvs(m.Env). SetWorkDir(m.paths.containerPath). AddVolume(DockerBindMount{ @@ -762,7 +762,8 @@ func (m *GolangConfigCover) docker(ctx context.Context) error { Source: cacheDir, Destination: "/cache", Consistency: "delegated", - }) + }). + AddVolume(m.Volumes...) gocmd := m.toolCmd() return trace.Wrap(cmd.Run(ctx, m.BuildContainer, gocmd[0], gocmd[1:]...)) } @@ -795,7 +796,7 @@ func (m *GolangConfigCover) cacheDir() (path string, err error) { return path, nil } -func (r *containerPathMapping) checkAndSetDefaults() (err error) { +func (r *containerPathMapping) compute(modulePath string, env map[string]string) (err error) { if r.hostPath == "" { r.hostPath, err = os.Getwd() if err != nil { @@ -805,9 +806,15 @@ func (r *containerPathMapping) checkAndSetDefaults() (err error) { // Different build containers have an inconsistent directory layout. // So use a distinct directory for sources if r.containerPath == "" { - r.containerPath = dockerSrcPathFromGopath(r.hostPath, "/host", build.Default.SrcDirs()) - if r.containerPath != "/host" { - r.gopath = "/host" + if r.gopath != "" { + r.containerPath = filepath.Join(r.gopath, "src", modulePath) + env["GOPATH"] = r.gopath + if _, ok := env["GO111MODULE"]; !ok { + env["GO111MODULE"] = "auto" + } + } else { + delete(env, "GOPATH") + r.containerPath = "/host" } } return nil @@ -818,24 +825,11 @@ type containerPathMapping struct { // Defaults to process' working directory hostPath string - // continerPath optionally specifies the path to the package repository inside + // containerPath optionally specifies the path to the package repository inside // the container. // Will be computed automatically based on host's GOPATH configuration if unspecified containerPath string - // gopath optionally specifies the container's GOPATH in GOPATH mode. - // GOPATH can be overridden with GO111MODULE=on in Go 1.11+ + // gopath optionally overrides the container's GOPATH in GOPATH mode. gopath string } - -// dockerSrcPathFromGopath builds a path for Go repository at root -// using host's GOPATH configuration. -func dockerSrcPathFromGopath(root, containerRootPath string, srcDirs []string) string { - for _, srcDir := range srcDirs { - if strings.HasPrefix(root, srcDir) { - fmt.Printf("return rooted at container root %q:\n", containerRootPath) - return filepath.Join(containerRootPath, strings.TrimPrefix(root, srcDir)) - } - } - return containerRootPath -} diff --git a/vendor/github.com/gravitational/magnet/magnet.go b/vendor/github.com/gravitational/magnet/magnet.go index 3c66510791..dc26c72c41 100644 --- a/vendor/github.com/gravitational/magnet/magnet.go +++ b/vendor/github.com/gravitational/magnet/magnet.go @@ -8,6 +8,8 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" + "strings" "sync" "time" @@ -29,20 +31,35 @@ type Config struct { // ModulePath specifies the path of the Go module being built ModulePath string - // Version specifies the module version + // Version specifies the module version. + // If unspecified, will be synthesized based on a git tag. Version string // PrintConfig configures whether magnet will output its configuration PrintConfig bool + // PlainProgress specifies whether the logger uses fancy progress reporting. // Set to true to see streaming output (e.g. on CI) - PlainProgress bool - // ImportEnv optionally specifies the external configuration as a set of - // environment variables - ImportEnv map[string]string + PlainProgress *bool } func (c *Config) checkAndSetDefaults() error { + if c.CacheDir == "" { + c.CacheDir = defaultCacheDir() + if c.CacheDir == "" { + return trace.BadParameter("expected CacheDir to be set") + } + } + + wd, err := os.Getwd() + if err != nil { + return trace.Wrap(err) + } + + if !filepath.IsAbs(c.CacheDir) { + c.CacheDir = filepath.Join(wd, c.CacheDir) + } + if c.Version == "" { c.Version = DefaultVersion() } @@ -55,19 +72,11 @@ func (c *Config) checkAndSetDefaults() error { return nil } - wd, err := os.Getwd() - if err != nil { - return trace.Wrap(err) - } - c.ModulePath, err = getModulePath(wd) if err != nil { return trace.Wrap(err) } - if !filepath.IsAbs(c.CacheDir) { - c.CacheDir = filepath.Join(wd, c.CacheDir) - } return nil } @@ -90,8 +99,6 @@ type Magnet struct { statusLogger *SolveStatusLogger root MagnetTarget - env map[string]EnvVar - wg sync.WaitGroup ctx context.Context // cancel cancels the logger process @@ -100,6 +107,7 @@ type Magnet struct { } // MagnetTarget describes a child logging target +//nolint:revive // TODO(dima): rename to Target type MagnetTarget struct { root *Magnet vertex *progressui.Vertex @@ -108,6 +116,10 @@ type MagnetTarget struct { // Root creates a root vertex for executing and capturing status of each build target. func Root(c Config) (*Magnet, error) { + if root != nil { + panic("already initialized") + } + if err := c.checkAndSetDefaults(); err != nil { return nil, trace.Wrap(err) } @@ -119,7 +131,7 @@ func Root(c Config) (*Magnet, error) { now := time.Now() ctx, cancel := context.WithCancel(context.Background()) - root := &Magnet{ + root = &Magnet{ Config: c, root: MagnetTarget{ vertex: &progressui.Vertex{ @@ -130,7 +142,6 @@ func Root(c Config) (*Magnet, error) { }, status: statusLogger.source, statusLogger: statusLogger, - env: make(map[string]EnvVar), ctx: ctx, cancel: cancel, } @@ -138,6 +149,9 @@ func Root(c Config) (*Magnet, error) { return root, nil } +// root specifies the Magnet instance singleton +var root *Magnet + func newSecretsRedactor(env map[string]EnvVar) secretsRedactor { var secrets []EnvVar for _, value := range env { @@ -213,7 +227,7 @@ func (c Config) AbsCacheDir() (path string) { // initOutput starts the internal progress logging process func (m *Magnet) initOutput() { m.initOutputOnce.Do(func() { - redactor := newSecretsRedactor(m.env) + redactor := newSecretsRedactor(env.env) m.statusLogger.start(redactor) if m.PrintConfig { @@ -222,7 +236,7 @@ func (m *Magnet) initOutput() { var c console.Console - if !m.PlainProgress { + if !m.plainProgress() { if cn, err := console.ConsoleFromFile(os.Stderr); err == nil { c = cn } @@ -230,7 +244,7 @@ func (m *Magnet) initOutput() { m.wg.Add(1) go func() { - progressui.DisplaySolveStatus( + _ = progressui.DisplaySolveStatus( m.ctx, m.root.vertex.Name, c, @@ -242,6 +256,20 @@ func (m *Magnet) initOutput() { }) } +func defaultCacheDir() string { + if runtime.GOOS != "linux" { + return "" + } + return cacheDir +} + +func (c Config) plainProgress() bool { + if c.PlainProgress != nil { + return *c.PlainProgress + } + return debianFrontend == "noninteractive" +} + func (c Config) cacheDir() string { return filepath.Join(c.CacheDir, "magnet", c.ModulePath) } @@ -295,11 +323,9 @@ func getModulePath(root string) (path string, err error) { if err == nil { return modfile.ModulePath(buf), nil } - var modulePath string for _, srcDir := range build.Default.SrcDirs() { - modulePath, err = filepath.Rel(srcDir, root) - if err == nil { - return modulePath, nil + if strings.HasPrefix(root, srcDir) { + return strings.TrimPrefix(root, srcDir), nil } } return "", trace.Wrap(err, "invalid working directory %s in GOPATH mode", root) diff --git a/vendor/github.com/gravitational/magnet/pkg/progressui/printer.go b/vendor/github.com/gravitational/magnet/pkg/progressui/printer.go index 78c0f61d40..83d5156c87 100644 --- a/vendor/github.com/gravitational/magnet/pkg/progressui/printer.go +++ b/vendor/github.com/gravitational/magnet/pkg/progressui/printer.go @@ -127,7 +127,7 @@ func (p *textMux) printVtx(t *trace, dgst digest.Digest) { if i == 0 { l = l[v.logsOffset:] } - fmt.Fprintf(p.w, "%s", []byte(l)) + fmt.Fprintf(p.w, "%s", l) if i != len(v.logs)-1 || !v.logsPartial { fmt.Fprintln(p.w, "") } diff --git a/vendor/github.com/magefile/mage/parse/parse.go b/vendor/github.com/magefile/mage/parse/parse.go index 5a58474763..0751cb415d 100644 --- a/vendor/github.com/magefile/mage/parse/parse.go +++ b/vendor/github.com/magefile/mage/parse/parse.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "log" "os" + "reflect" "strings" "time" @@ -549,11 +550,22 @@ func setDeinit(pi *PkgInfo) { return } pi.DeinitFunc = f + removeDeinitFuncFromTargets(pi) return } } } +func removeDeinitFuncFromTargets(pi *PkgInfo) { + for i, f := range pi.Funcs { + if !reflect.DeepEqual(f, pi.DeinitFunc) { + continue + } + pi.Funcs = append(pi.Funcs[:i], pi.Funcs[i+1:]...) + return + } +} + func lit2string(l *ast.BasicLit) (string, bool) { if !strings.HasPrefix(l.Value, `"`) || !strings.HasSuffix(l.Value, `"`) { return "", false diff --git a/vendor/modules.txt b/vendor/modules.txt index 43249fd485..eba00008e0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -454,7 +454,7 @@ github.com/gravitational/kingpin github.com/gravitational/license github.com/gravitational/license/authority github.com/gravitational/license/constants -# github.com/gravitational/magnet v0.2.6 => github.com/a-palchikov/magnet v0.2.7-0.20210520113349-2a5f8ce635ab +# github.com/gravitational/magnet v0.2.7-0.20210609203954-0a59f53f530e github.com/gravitational/magnet github.com/gravitational/magnet/common github.com/gravitational/magnet/pkg/cp @@ -602,7 +602,7 @@ github.com/lib/pq/oid github.com/lib/pq/scram # github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de github.com/liggitt/tabwriter -# github.com/magefile/mage v1.9.0 => github.com/a-palchikov/mage v1.9.1-0.20210510142914-0614fcadda0b +# github.com/magefile/mage v1.9.0 => github.com/knisbet/mage v1.9.1-0.20210609142646-749a704341ac github.com/magefile/mage/internal github.com/magefile/mage/mage github.com/magefile/mage/mg