abandon vendoring in generated beats

dev-tools/mage/gomod.go

@@ -36,7 +36,7 @@ type CopyModule struct {
 // CopyFilesToVendor copies packages which require the whole tree
 func CopyFilesToVendor(vendorFolder string, modulesToCopy []CopyModule) error {
 	for _, p := range modulesToCopy {
-		path, err := gotool.ListModuleForceCacheDir(p.Name)
+		path, err := gotool.ListModuleCacheDir(p.Name)
 		if err != nil {
 			return errors.Wrapf(err, "error while looking up cached dir of module: %s", p.Name)
 		}

dev-tools/mage/gotool/go.go

@@ -97,32 +97,24 @@ func ListTestFiles(pkg string) ([]string, error) {
 // the specified module. If the module does not exist in the cache,
 // an error will be returned.
 func ListModuleCacheDir(pkg string) (string, error) {
-	return listModuleDir(pkg, false, false)
+	return listModuleDir(pkg, false)
 }
 
 // ListModuleVendorDir returns the vendor directory containing the
 // specified module. If the module has not been vendored, an error
 // will be returned.
 func ListModuleVendorDir(pkg string) (string, error) {
-	return listModuleDir(pkg, true, false)
+	return listModuleDir(pkg, true)
 }
 
-// ListModuleForceCacheDir returns the module cache directory containing the
-// specified module even if the repo has a vendor folder.
-func ListModuleForceCacheDir(pkg string) (string, error) {
-	return listModuleDir(pkg, false, true)
-}
-
-func listModuleDir(pkg string, vendor, force bool) (string, error) {
+func listModuleDir(pkg string, vendor bool) (string, error) {
 	env := map[string]string{
 		// Make sure GOFLAGS does not influence behaviour.
 		"GOFLAGS": "",
 	}
 	args := []string{"-m", "-f", "{{.Dir}}"}
 	if vendor {
 		args = append(args, "-mod=vendor")
-	} else if force {
-		args = append(args, "-mod=mod")
 	}
 	args = append(args, pkg)
 	lines, err := getLines(callGo(env, "list", args...))

docs/devguide/newbeat.asciidoc

@@ -515,9 +515,6 @@ To depend on the latest `master` of `github.com/elastic/beats` run the following
 go get github.com/elastic/beats@master
 ----------------------------------------------------------------------
 
-To move the dependencies to vendor, you need to manually fetch the new
-`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`.
-
 We suggest you read the following section to learn about maintaining dependencies
 using go modules:
 * https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies]

generator/_templates/beat/{beat}/Makefile

@@ -4,7 +4,7 @@ BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
 ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
-ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
+ES_BEATS?=$(shell go list -m -f '{{.Dir}}' ${ES_BEATS_IMPORT_PATH})
 LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile
 GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
 GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"

generator/_templates/beat/{beat}/magefile.go

@@ -23,6 +23,7 @@ func init() {
 	devtools.BeatDescription = "One sentence description of the Beat."
 	devtools.BeatVendor = "{full_name}"
 	devtools.BeatProjectType = devtools.CommunityProject
+	devtools.CrossBuildMountModcache = true
 }
 
 // VendorUpdate updates the vendor dir

generator/_templates/metricbeat/{beat}/Makefile

@@ -4,15 +4,11 @@ BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
 ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
-ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
+ES_BEATS?=$(shell go list -m -f '{{.Dir}}' ${ES_BEATS_IMPORT_PATH})
 GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
 GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"
 MAGE_IMPORT_PATH=github.com/magefile/mage
 CHECK_HEADERS_DISABLED=true
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/metricbeat/Makefile
-
-.PHONY: copy-vendor
-copy-vendor:
-	mage vendorUpdate

generator/_templates/metricbeat/{beat}/magefile.go

@@ -26,6 +26,7 @@ func init() {
 
 	devtools.BeatDescription = "One sentence description of the Beat."
 	devtools.BeatVendor = "{full_name}"
+	devtools.CrossBuildMountModcache = true
 }
 
 // VendorUpdate updates elastic/beats/v7 in the vendor dir

generator/common/Makefile

@@ -38,7 +38,7 @@ prepare-test:: mage
 	export NEWBEAT_TYPE=${BEAT_TYPE} ; \
 	export NEWBEAT_DEV=1 ; \
 	export NEWBEAT_BEATS_REVISION=$(shell git rev-parse origin/master) ; \
-	mage -v GenerateCustomBeat
+	mage GenerateCustomBeat
 
 # Runs test build for the created beat
 .PHONY: test-build

generator/common/beatgen/beatgen.go

@@ -126,7 +126,6 @@ func Generate() error {
 		return errors.Wrap(err, "error while getting required beats version")
 	}
 
-	mg.Deps(setup.CopyVendor)
 	mg.Deps(setup.GitInit)
 
 	if cfg["type"] == "metricbeat" {
@@ -156,17 +155,6 @@ func getConfiguredBeatsRevision(beatsModule, revision string) error {
 	)
 }
 
-// VendorUpdate updates the vendor directory if used
-func VendorUpdate() error {
-	err := sh.Rm("./vendor/github.com/elastic/beats")
-	if err != nil {
-		return errors.Wrap(err, "error removing vendor dir")
-	}
-
-	devtools.SetElasticBeatsDir(getAbsoluteBeatsPath())
-	return setup.CopyVendor()
-}
-
 // returns a "compleated" config object with everything we need
 func getConfig() (map[string]string, error) {
 	userCfg := make(map[string]string)

generator/common/beatgen/setup/setup.go

@@ -95,42 +95,6 @@ func copyReplacedModules() error {
 	return w.Flush()
 }
 
-// CopyVendor copies a new version of the dependencies to the vendor folder
-func CopyVendor() error {
-	err := gotool.Mod.Vendor()
-	if err != nil {
-		return errors.Wrapf(err, "error while running go mod vendor")
-	}
-
-	err = devtools.CopyFilesToVendor(
-		"./vendor",
-		[]devtools.CopyModule{
-			devtools.CopyModule{
-				Name: "github.com/elastic/beats/v7",
-				FilesToCopy: []string{
-					"dev-tools",
-					"libbeat",
-					"licenses",
-					"metricbeat",
-					"script",
-					".go-version",
-				},
-			},
-			devtools.CopyModule{
-				Name: "github.com/tsg/go-daemon",
-				FilesToCopy: []string{
-					"src",
-				},
-			},
-		},
-	)
-	if err != nil {
-		return errors.Wrapf(err, "error while copying required files to vendor")
-	}
-
-	return nil
-}
-
 // GitInit initializes a new git repo in the current directory
 func GitInit() error {
 	return sh.Run("git", "init")