Fix generator jobs for go modules (#16288) (#16828)

* Set the correct path to `mage` for generator jobs
* Generated beats use go modules for dependency management
   * New option `beats_revision` is added so users can select which beats revision they want to vendor
    * if one does not want to depend on `elastic/beats`, he/she needs to add the appropriate `replace` directive to the generated `go.mod`

As Beats is moving to go modules for dependency management, generated Beats should use that as well.

Documentation is provided to help users migrate to go modules in their own Beats. Also, about the new option.
(cherry picked from commit a846029)

.travis.yml

@@ -148,24 +148,23 @@ jobs:
       stage: test
 
     # Generators
-    # Temporarily disable generator jobs
-    #- os: linux
-    #  env: TARGETS="-C generator/_templates/metricbeat test test-package"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
-    #- os: linux
-    #  env: TARGETS="-C generator/_templates/beat test test-package"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
+    - os: linux
+      env: TARGETS="-C generator/_templates/metricbeat test test-package"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      env: TARGETS="-C generator/_templates/beat test test-package"
+      go: $TRAVIS_GO_VERSION
+      stage: test
 
-    #- os: osx
-    #  env: TARGETS="-C generator/_templates/metricbeat test"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
-    #- os: osx
-    #  env: TARGETS="-C generator/_templates/beat test"
-    #  go: $TRAVIS_GO_VERSION
-    #  stage: test
+    - os: osx
+      env: TARGETS="-C generator/_templates/metricbeat test"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: osx
+      env: TARGETS="-C generator/_templates/beat test"
+      go: $TRAVIS_GO_VERSION
+      stage: test
 
     # Kubernetes
     - os: linux

dev-tools/mage/gomod.go

@@ -18,7 +18,6 @@
 package mage
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 
@@ -80,7 +79,6 @@ func Vendor() error {
 		if err != nil {
 			return err
 		}
-		fmt.Println(path)
 
 		for _, f := range p.filesToCopy {
 			from := filepath.Join(path, f)

dev-tools/mage/gotool/get.go

@@ -27,5 +27,6 @@ func runGoGet(opts ...ArgOpt) error {
 	return runVGo("get", args)
 }
 
+func (goGet) Download() ArgOpt          { return flagBoolIf("-d", true) }
 func (goGet) Update() ArgOpt            { return flagBoolIf("-u", true) }
 func (goGet) Package(pkg string) ArgOpt { return posArg(pkg) }

dev-tools/mage/gotool/go.go

@@ -105,7 +105,7 @@ func ListModulePath(pkg string) (string, error) {
 		return "", err
 	}
 	if n := len(lines); n != 1 {
-		return "", fmt.Errorf("expected 1 line, got %d", n)
+		return "", fmt.Errorf("expected 1 line, got %d while looking for %s", n, pkg)
 	}
 	return lines[0], nil
 }

dev-tools/mage/gotool/modules.go

@@ -19,6 +19,7 @@ package gotool
 
 // Mod is the command go mod.
 var Mod = goMod{
+	Init:   modCommand{"init"}.run,
 	Tidy:   modCommand{"tidy"}.run,
 	Verify: modCommand{"verify"}.run,
 	Vendor: modCommand{"vendor"}.run,
@@ -39,11 +40,15 @@ func (cmd modCommand) run(opts ...ArgOpt) error {
 }
 
 type goMod struct {
+	Init   modInit
 	Tidy   modTidy
 	Verify modVerify
 	Vendor modVendor
 }
 
+// modInit initializes a new go module in folder.
+type modInit func(opts ...ArgOpt) error
+
 // modTidy cleans the go.mod file
 type modTidy func(opts ...ArgOpt) error
 

dev-tools/mage/settings.go

@@ -23,9 +23,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"path"
 	"path/filepath"
-	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
@@ -275,20 +273,15 @@ func ElasticBeatsDir() (string, error) {
 // it will return the root directory of the module from within the module cache or vendor
 // directory.
 func findElasticBeatsDir() (string, error) {
-	// Find the import path for the package containing this file.
-	type foo struct{}
-	typ := reflect.TypeOf(foo{})
-	magepkgpath := typ.PkgPath()
-
-	// Walk up the import path until we find the elastic/beats module path.
-	pkgpath := magepkgpath
-	for extractCanonicalRootImportPath(pkgpath) != elasticBeatsImportPath {
-		pkgpath = path.Dir(pkgpath)
-		if pkgpath == "." {
-			return "", errors.Errorf("failed to find %q from %q", elasticBeatsImportPath, magepkgpath)
-		}
+	repo, err := GetProjectRepoInfo()
+	if err != nil {
+		return "", err
 	}
-	return gotool.ListModulePath(pkgpath)
+	if repo.IsElasticBeats() {
+		return repo.RootDir, nil
+	}
+
+	return gotool.ListModulePath("github.com/elastic/beats/v7")
 }
 
 var (

docs/devguide/newbeat.asciidoc

@@ -26,6 +26,10 @@ The following topics describe how to build a new Beat:
 * <<newbeat-sharing>>
 * <<event-conventions>>
 
+The following topic provide information about moving generated Beats to go modules:
+
+* <<newbeat-migrate-gomodules>>
+
 [[newbeat-getting-ready]]
 === Getting Ready
 
@@ -128,18 +132,21 @@ mage GenerateCustomBeat
 --------------------
 
 The mage script will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`.
-For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname.
+For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname. Finally, pick the revision of elastic/beats you would like to depend on.
 
 [source,shell]
 ---------
 Enter a project name [examplebeat]:  Countbeat
 Enter a github name [your-github-name]: {username}
 Enter a beat path [github.com/{username}/countbeat]: 
 Enter a full name [Firstname Lastname]: {Full Name}
+Enter the github.com/elastic/beats revision [master]:
 ---------
 
 The Beat generator creates a directory called `countbeat` inside of your project folder (e.g. {project folder}/github.com/{github id}/countbeat).
 
+Please note that it is not possible to use revisions of Beats which does not support go modules.
+
 You now have a raw template of the Beat, but you still need to <<setting-up-beat,fetch dependencies and set up the Beat>>.
 
 [[setting-up-beat]]
@@ -480,3 +487,35 @@ func main() {
 
 When you're done with your new Beat, how about letting everyone know? Open
 a pull request to add your link to the {beats-ref}/community-beats.html[Community Beats] list.
+
+[[newbeat-migrate-gomodules]]
+=== Migrate existing Beat to go modules
+
+Get started by making sure the contents of the vendored beats folder is
+pushed to a remote repository. Otherwise, go will overwrite the folder
+with the selected revision of `github.com/elastic/beats`.
+
+The following command will initialize a new module in your repo.
+
+[source,shell]
+----------------------------------------------------------------------
+go mod init
+----------------------------------------------------------------------
+
+Make sure to add the approprite `replace` directives from the `go.mod` file
+of Beats: https://github.com/elastic/beats/tree/master/go.mod
+Unfortunately, this workaround is needed to make sure your Beat can be compiled.
+
+To depend on the latest `master` of `github.com/elastic/beats` run the following command:
+
+[source,shell]
+----------------------------------------------------------------------
+go get github.com/elastic/beats@master
+----------------------------------------------------------------------
+
+To move the dependencies to vendor, you need to manually fetch the new
+`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`.
+
+We suggest you read the following section to learn about maintaining dependencies
+using go modules:
+* https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies]

generator/Makefile

@@ -1,11 +1,11 @@
 # Runs test build for the count metricset
 .PHONY: test
 test:
-	$(MAKE) -C beat test
-	$(MAKE) -C metricbeat test
+	$(MAKE) -C _templates/beat test
+	$(MAKE) -C _templates/metricbeat test
 
 # Cleans up environment
 .PHONY: clean
 clean:
-	@$(MAKE) -C beat clean
-	@$(MAKE) -C metricbeat clean
+	@$(MAKE) -C _templates/beat clean
+	@$(MAKE) -C _templates/metricbeat clean

generator/_templates/beat/{beat}/Makefile

@@ -3,11 +3,12 @@ BEAT_PATH={beat_path}
 BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
-ES_BEATS?=./vendor/github.com/elastic/beats
+ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
+ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
 LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile
-GOPACKAGES=$(shell govendor list -no-status +local)
-GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
-MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
+GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
+GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"
+MAGE_IMPORT_PATH=github.com/magefile/mage
 NO_COLLECT=true
 CHECK_HEADERS_DISABLED=true
 
@@ -16,4 +17,4 @@ CHECK_HEADERS_DISABLED=true
 
 .PHONY: copy-vendor
 copy-vendor:
-	mage vendorUpdate
+	mage vendorUpdate

generator/_templates/beat/{beat}/_meta/beat.reference.yml

@@ -0,0 +1,7 @@
+################### {Beat} Configuration Example #########################
+
+############################# {Beat} ######################################
+
+{beat}:
+  # Defines how often an event is sent to the output
+  period: 1s

generator/_templates/beat/{beat}/include/include.go

@@ -0,0 +1 @@
+package include

generator/_templates/beat/{beat}/magefile.go

@@ -25,7 +25,7 @@ func init() {
 	devtools.BeatProjectType = devtools.CommunityProject
 }
 
-// VendorUpdate updates elastic/beats/v7 in the vendor dir
+// VendorUpdate updates the vendor dir
 func VendorUpdate() error {
 	return beatgen.VendorUpdate()
 }

generator/_templates/beat/{beat}/tools/tools.go

@@ -0,0 +1,14 @@
+// +build tools
+
+// This package contains the tool dependencies of the project.
+
+package tools
+
+import (
+	_ "github.com/pierrre/gotestcover"
+	_ "golang.org/x/tools/cmd/goimports"
+
+	_ "github.com/mitchellh/gox"
+	_ "github.com/reviewdog/reviewdog/cmd/reviewdog"
+	_ "golang.org/x/lint/golint"
+)

generator/_templates/metricbeat/{beat}/Makefile

@@ -3,15 +3,16 @@ BEAT_PATH={beat_path}
 BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
-ES_BEATS?=./vendor/github.com/elastic/beats
-GOPACKAGES=$(shell govendor list -no-status +local)
-GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
-MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
+ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7
+ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH}
+GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools)
+GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)"
+MAGE_IMPORT_PATH=github.com/magefile/mage
 CHECK_HEADERS_DISABLED=true
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/metricbeat/Makefile
 
 .PHONY: copy-vendor
 copy-vendor:
-	mage vendorUpdate
+	mage vendorUpdate

generator/_templates/metricbeat/{beat}/README.md

@@ -33,14 +33,11 @@ make collect
 
 This updates all fields and docs with the most recent changes.
 
-## Use vendoring
+## Dependency management
 
-We recommend to use vendoring for your beat. This means the dependencies are put into your beat folder. The beats team currently uses [govendor](https://github.com/kardianos/govendor) for vendoring.
+Beats uses go modules for dependency management.
 
-```
-govendor init
-govendor update +e
-```
+In order to maintain your `vendor` folder, run `mage copyVendor`.
 
 This will create a directory `vendor` inside your repository. To make sure all dependencies for the Makefile commands are loaded from the vendor directory, find the following line in your Makefile:
 
@@ -50,7 +47,7 @@ ES_BEATS=${GOPATH}/src/github.com/elastic/beats
 
 Replace it with:
 ```
-ES_BEATS=./vendor/github.com/elastic/beats
+ES_BEATS=./vendor/github.com/elastic/beats/v7
 ```
 
 

generator/_templates/metricbeat/{beat}/include/include.go

@@ -0,0 +1 @@
+package include

generator/_templates/metricbeat/{beat}/tools/tools.go

@@ -0,0 +1,14 @@
+// +build tools
+
+// This package contains the tool dependencies of the project.
+
+package tools
+
+import (
+	_ "github.com/pierrre/gotestcover"
+	_ "golang.org/x/tools/cmd/goimports"
+
+	_ "github.com/mitchellh/gox"
+	_ "github.com/reviewdog/reviewdog/cmd/reviewdog"
+	_ "golang.org/x/lint/golint"
+)

generator/common/Makefile

@@ -5,6 +5,8 @@ BEAT_PATH=${GOPATH}/src/${BEAT_NAME}
 ES_BEATS=${GOPATH}/src/github.com/elastic/beats
 PREPARE_COMMAND?=
 
+-include ${ES_BEATS}/dev-tools/make/mage.mk
+
 # Runs test build for mock beat
 .PHONY: test
 test: prepare-test
@@ -23,36 +25,28 @@ test-package: test
 	mage package
 
 .PHONY: prepare-test
-prepare-test:: ${GOPATH}/bin/mage
+prepare-test:: mage
 	rm -fr ${BEAT_PATH}
 	mkdir -p ${BEAT_PATH}
 	cd ${GOPATH}/src/github.com/elastic/beats/ ; \
 	export MODULE=elastic ; \
 	export METRICSET=test ; \
-	export GO111MODULE=off; \
 	export NEWBEAT_PROJECT_NAME=Testbeat ; \
 	export NEWBEAT_GITHUB_NAME=ruflin ; \
 	export NEWBEAT_BEAT_PATH=${BEAT_NAME} ; \
 	export NEWBEAT_FULL_NAME="Nicolas Ruflin" ; \
 	export NEWBEAT_TYPE=${BEAT_TYPE} ; \
 	export NEWBEAT_DEV=1 ; \
+	export NEWBEAT_BEATS_REVISION=$(shell git rev-parse origin/master) ; \
 	mage GenerateCustomBeat
 
 # Runs test build for the created beat
 .PHONY: test-build
 test-build: test
-	# Copy dev tools
-	cp -r ${PWD}/../../../dev-tools ${BEAT_PATH}/vendor/github.com/elastic/beats/
-
 	cd ${BEAT_PATH}/dev-tools/packer ; \
 	$(MAKE) deps ; \
 	$(MAKE) images
 
-${GOPATH}/bin/mage:
-	go get -u -d github.com/magefile/mage
-	cd $${GOPATH}/src/github.com/magefile/mage;  \
-	go run bootstrap.go
-
 # Cleans up environment
 .PHONY: clean
 clean: