Update go version to 1.14.4 (#19753) (#19792)

This PR adds support for Golang 1.14.4 in Beats. The dependency `golang.org/x/tools` is pinned to `release-branch.go1.14`.

The generated Beats no longer contain a vendor folder.

Go 1.14 has been out for quite some time. It is high time we update.

Closes #18829
Closes #16566

(cherry picked from commit 7580c72)

.go-version

@@ -1 +1 @@
-1.13.10
+1.14.4

CHANGELOG-developer.next.asciidoc

@@ -88,3 +88,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Add fields validation for histogram subfields. {pull}17759[17759]
 - Events intended for the Elasticsearch output can now take an `op_type` metadata field of type events.OpType or string to indicate the `op_type` to use for bulk indexing. {pull}12606[12606]
 - Remove vendor folder from repository. {pull}18655[18655]
+- Update Go version to 1.14.4. {pull}19753[19753]

Makefile

@@ -98,6 +98,8 @@ check: python-env
 	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
 	@$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
 	@$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false)
+	# check if vendor folder does not exists
+	[ ! -d vendor ]
 	@# Validate that all updates were committed
 	@$(MAKE) update
 	@$(MAKE) check-headers

NOTICE.txt

@@ -15460,11 +15460,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/tools
-Version: v0.0.0-20200615191743-991d59a616de
+Version: v0.0.0-20200602230032-c00d67ef29d0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.0.0-20200615191743-991d59a616de/LICENSE:
+Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.0.0-20200602230032-c00d67ef29d0/LICENSE:
 
 Copyright (c) 2009 The Go Authors. All rights reserved.
 

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.10
+FROM golang:1.14.4
 
 RUN \
     apt-get update \

dev-tools/mage/build.go

@@ -118,16 +118,6 @@ func Build(params BuildArgs) error {
 	}
 	env["CGO_ENABLED"] = cgoEnabled
 
-	if UseVendor {
-		var goFlags string
-		goFlags, ok := env["GOFLAGS"]
-		if !ok {
-			env["GOFLAGS"] = "-mod=vendor"
-		} else {
-			env["GOFLAGS"] = strings.Join([]string{goFlags, "-mod=vendor"}, " ")
-		}
-	}
-
 	// Spec
 	args := []string{
 		"build",

dev-tools/mage/common.go

@@ -49,8 +49,6 @@ import (
 	"github.com/magefile/mage/sh"
 	"github.com/magefile/mage/target"
 	"github.com/pkg/errors"
-
-	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
 
 // Expand expands the given Go text/template string.
@@ -784,16 +782,6 @@ func binaryExtension(goos string) string {
 	return ""
 }
 
-// listModuleDir calls gotool.ListModuleVendorDir or
-// gotool.ListModuleCacheDir, depending on the value of
-// UseVendor.
-func listModuleDir(modpath string) (string, error) {
-	if UseVendor {
-		return gotool.ListModuleVendorDir(modpath)
-	}
-	return gotool.ListModuleCacheDir(modpath)
-}
-
 var parseVersionRegex = regexp.MustCompile(`(?m)^[^\d]*(?P<major>\d)+\.(?P<minor>\d)+(?:\.(?P<patch>\d)+.*)?$`)
 
 // ParseVersion extracts the major, minor, and optional patch number from a

dev-tools/mage/crossbuild.go

@@ -255,9 +255,6 @@ func (b GolangCrossBuilder) Build() error {
 	if versionQualified {
 		args = append(args, "--env", "VERSION_QUALIFIER="+versionQualifier)
 	}
-	if UseVendor {
-		args = append(args, "--env", "GOFLAGS=-mod=vendor")
-	}
 	if CrossBuildMountModcache {
 		// Mount $GOPATH/pkg/mod into the container, read-only.
 		hostDir := filepath.Join(build.Default.GOPATH, "pkg", "mod")

dev-tools/mage/fields.go

@@ -101,16 +101,12 @@ func generateFieldsYAML(baseDir, output string, moduleDirs ...string) error {
 		return err
 	}
 
-	cmd := []string{"run"}
-	if UseVendor {
-		cmd = append(cmd, "-mod", "vendor")
-	}
-	cmd = append(cmd,
+	cmd := []string{"run",
 		filepath.Join(beatsDir, globalFieldsCmdPath),
 		"-es_beats_path", beatsDir,
 		"-beat_path", baseDir,
 		"-out", CreateDir(output),
-	)
+	}
 	globalFieldsCmd := sh.RunCmd("go", cmd...)
 
 	return globalFieldsCmd(moduleDirs...)
@@ -130,18 +126,14 @@ func GenerateFieldsGo(fieldsYML, out string) error {
 		return err
 	}
 
-	cmd := []string{"run"}
-	if UseVendor {
-		cmd = append(cmd, "-mod", "vendor")
-	}
-	cmd = append(cmd,
+	cmd := []string{"run",
 		filepath.Join(beatsDir, assetCmdPath),
 		"-pkg", "include",
 		"-in", fieldsYML,
 		"-out", CreateDir(out),
 		"-license", toLibbeatLicenseName(BeatLicense),
 		BeatName,
-	)
+	}
 	assetCmd := sh.RunCmd("go", cmd...)
 
 	return assetCmd()
@@ -162,16 +154,12 @@ func GenerateModuleFieldsGo(moduleDir string) error {
 		moduleDir = CWD(moduleDir)
 	}
 
-	cmd := []string{"run"}
-	if UseVendor {
-		cmd = append(cmd, "-mod", "vendor")
-	}
-	cmd = append(cmd,
+	cmd := []string{"run",
 		filepath.Join(beatsDir, moduleFieldsCmdPath),
 		"-beat", BeatName,
 		"-license", toLibbeatLicenseName(BeatLicense),
 		moduleDir,
-	)
+	}
 	moduleFieldsCmd := sh.RunCmd("go", cmd...)
 
 	return moduleFieldsCmd()
@@ -194,16 +182,12 @@ func GenerateIncludeListGo(options IncludeListOptions) error {
 		return err
 	}
 
-	cmd := []string{"run"}
-	if UseVendor {
-		cmd = append(cmd, "-mod", "vendor")
-	}
-	cmd = append(cmd,
+	cmd := []string{"run",
 		filepath.Join(beatsDir, moduleIncludeListCmdPath),
 		"-license", toLibbeatLicenseName(BeatLicense),
 		"-out", options.Outfile, "-buildTags", options.BuildTags,
 		"-pkg", options.Pkg,
-	)
+	}
 
 	includeListCmd := sh.RunCmd("go", cmd...)
 

dev-tools/mage/fmt.go

@@ -66,19 +66,10 @@ func GoImports() error {
 	}
 
 	fmt.Println(">> fmt - goimports: Formatting Go code")
-	if UseVendor {
-		if err := gotool.Install(
-			gotool.Install.Vendored(),
-			gotool.Install.Package(filepath.Join(GoImportsImportPath)),
-		); err != nil {
-			return err
-		}
-	} else {
-		if err := gotool.Install(
-			gotool.Install.Package(filepath.Join(GoImportsImportPath)),
-		); err != nil {
-			return err
-		}
+	if err := gotool.Install(
+		gotool.Install.Package(filepath.Join(GoImportsImportPath)),
+	); err != nil {
+		return err
 	}
 
 	args := append(

dev-tools/mage/godaemon.go

@@ -22,6 +22,8 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+
+	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
 
 var (
@@ -43,7 +45,7 @@ func BuildGoDaemon() error {
 	}
 
 	// Test if binaries are up-to-date.
-	godaemonDir, err := listModuleDir("github.com/tsg/go-daemon")
+	godaemonDir, err := gotool.ListModuleCacheDir("github.com/tsg/go-daemon")
 	if err != nil {
 		return err
 	}

dev-tools/mage/install.go

@@ -37,9 +37,6 @@ func InstallVendored(importPath string) error {
 
 // InstallGoLicenser target installs go-licenser
 func InstallGoLicenser() error {
-	if UseVendor {
-		return InstallVendored(GoLicenserImportPath)
-	}
 	return gotool.Get(
 		gotool.Get.Package(GoLicenserImportPath),
 	)

dev-tools/mage/integtest.go

@@ -245,9 +245,6 @@ func initRunner(tester IntegrationTester, dir string, passInEnv map[string]strin
 	if mg.Verbose() {
 		env["MAGEFILE_VERBOSE"] = "1"
 	}
-	if UseVendor {
-		env["GOFLAGS"] = "-mod=vendor"
-	}
 
 	runner := &IntegrationRunner{
 		steps:  runnerSteps,

dev-tools/mage/settings.go

@@ -60,7 +60,6 @@ var (
 	XPackDir     = "../x-pack"
 	RaceDetector = false
 	TestCoverage = false
-	UseVendor    = false
 
 	// CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into
 	// the crossbuild images at /go/pkg/mod, read-only.
@@ -110,10 +109,6 @@ func init() {
 	if err != nil {
 		panic(errors.Wrap(err, "failed to parse TEST_COVERAGE env value"))
 	}
-	UseVendor, err = strconv.ParseBool(EnvOr("USE_VENDOR", "false"))
-	if err != nil {
-		panic(errors.Wrap(err, "failed to parse USE_VENDOR env value"))
-	}
 
 	Snapshot, err = strconv.ParseBool(EnvOr("SNAPSHOT", "false"))
 	if err != nil {
@@ -286,7 +281,7 @@ func findElasticBeatsDir() (string, error) {
 	if repo.IsElasticBeats() {
 		return repo.RootDir, nil
 	}
-	return listModuleDir(elasticBeatsModulePath)
+	return gotool.ListModuleCacheDir(elasticBeatsModulePath)
 }
 
 var (

dev-tools/packaging/packages.yml

@@ -72,8 +72,8 @@ shared:
   - &macos_agent_pkg_spec
     <<: *common
     extra_vars:
-      # OS X 10.11 Mountain Lion is the oldest supported by Go 1.13.
-      # https://golang.org/doc/go1.13#ports
+      # OS X 10.11 El Capitan is the oldest supported by Go 1.14.
+      # https://golang.org/doc/go1.14#ports
       min_supported_osx_version: 10.11
       identifier: 'co.{{.BeatVendor | tolower}}.beats.{{.BeatName}}'
       install_path: /Library/Application Support

docs/devguide/newbeat.asciidoc

@@ -515,9 +515,6 @@ To depend on the latest `master` of `github.com/elastic/beats` run the following
 go get github.com/elastic/beats@master
 ----------------------------------------------------------------------
 
-To move the dependencies to vendor, you need to manually fetch the new
-`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`.
-
 We suggest you read the following section to learn about maintaining dependencies
 using go modules:
 * https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies]
@@ -526,6 +523,5 @@ Please note that it is your choice whether you put dependencies in the folder ve
 
 If you choose to abandon the vendor folder, you have to adjust the following things in your Beat:
 
-* add `devtools.UseVendor = false` to the `magefile.go` of your Beat
 * add `devtools.CrossBuildMountModcache = true` to the `magefile.go` of your Beat
 * modify the path to `ES_BEATS` in the `Makefile` so it points to the folder under the module cache (`go list -m -f '{{.Dir}}' $ES_BEATS_IMPORT_PATH`)

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.10
+FROM golang:1.14.4
 
 RUN \
     apt-get update \

generator/_templates/beat/{beat}/Makefile

@@ -4,7 +4,7 @@ BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
 ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
-ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
+ES_BEATS?=$(shell go list -m -f '{{.Dir}}' ${ES_BEATS_IMPORT_PATH})
 LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile
 GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
 GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"

generator/_templates/beat/{beat}/magefile.go

@@ -14,7 +14,6 @@ import (
 	"github.com/elastic/beats/v7/dev-tools/mage/target/common"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/pkg"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
-	"github.com/elastic/beats/v7/generator/common/beatgen"
 )
 
 func init() {
@@ -23,11 +22,7 @@ func init() {
 	devtools.BeatDescription = "One sentence description of the Beat."
 	devtools.BeatVendor = "{full_name}"
 	devtools.BeatProjectType = devtools.CommunityProject
-}
-
-// VendorUpdate updates the vendor dir
-func VendorUpdate() error {
-	return beatgen.VendorUpdate()
+	devtools.CrossBuildMountModcache = true
 }
 
 // Package packages the Beat for distribution.

generator/_templates/metricbeat/{beat}/Makefile

@@ -4,15 +4,11 @@ BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
 ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
-ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
+ES_BEATS?=$(shell go list -m -f '{{.Dir}}' ${ES_BEATS_IMPORT_PATH})
 GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
 GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"
 MAGE_IMPORT_PATH=github.com/magefile/mage
 CHECK_HEADERS_DISABLED=true
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/metricbeat/Makefile
-
-.PHONY: copy-vendor
-copy-vendor:
-	mage vendorUpdate

generator/_templates/metricbeat/{beat}/magefile.go

@@ -14,7 +14,6 @@ import (
 	"github.com/elastic/beats/v7/dev-tools/mage/target/common"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/pkg"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
-	"github.com/elastic/beats/v7/generator/common/beatgen"
 	metricbeat "github.com/elastic/beats/v7/metricbeat/scripts/mage"
 
 	// mage:import
@@ -26,11 +25,7 @@ func init() {
 
 	devtools.BeatDescription = "One sentence description of the Beat."
 	devtools.BeatVendor = "{full_name}"
-}
-
-// VendorUpdate updates elastic/beats/v7 in the vendor dir
-func VendorUpdate() error {
-	return beatgen.VendorUpdate()
+	devtools.CrossBuildMountModcache = true
 }
 
 // CollectAll generates the docs and the fields.

generator/common/beatgen/beatgen.go

@@ -126,7 +126,6 @@ func Generate() error {
 		return errors.Wrap(err, "error while getting required beats version")
 	}
 
-	mg.Deps(setup.CopyVendor)
 	mg.Deps(setup.GitInit)
 
 	if cfg["type"] == "metricbeat" {

go.mod

@@ -1,6 +1,6 @@
 module github.com/elastic/beats/v7
 
-go 1.13
+go 1.14
 
 require (
 	4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b
@@ -190,5 +190,5 @@ replace (
 	github.com/google/gopacket => github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41
 	github.com/insomniacslk/dhcp => github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 // indirect
 	github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c
-	golang.org/x/tools => golang.org/x/tools v0.0.0-20200615191743-991d59a616de // release 1.13
+	golang.org/x/tools => golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0 // release 1.14
 )

go.sum

@@ -844,8 +844,9 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20200615191743-991d59a616de h1:kFKSx8iHlOzmtGWtfJW+b2UzcJ+rMWHHyUBpjrZq8To=
-golang.org/x/tools v0.0.0-20200615191743-991d59a616de/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0 h1:6txNFSnY+tteYoO+hf01EpdYcYZiurdC9MDIrcUzEu4=
+golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=