From a6ab74bf84b97bd080bd6c68c4c2b3f7ebf5936b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 5 Mar 2020 12:15:42 +0100 Subject: [PATCH] Fix generator jobs for go modules (#16288) (#16828) * Set the correct path to `mage` for generator jobs * Generated beats use go modules for dependency management * New option `beats_revision` is added so users can select which beats revision they want to vendor * if one does not want to depend on `elastic/beats`, he/she needs to add the appropriate `replace` directive to the generated `go.mod` As Beats is moving to go modules for dependency management, generated Beats should use that as well. Documentation is provided to help users migrate to go modules in their own Beats. Also, about the new option. (cherry picked from commit a846029d5bbd0e83d45bc38e8dce3c6e02483274) --- .travis.yml | 33 ++--- dev-tools/mage/gomod.go | 2 - dev-tools/mage/gotool/get.go | 1 + dev-tools/mage/gotool/go.go | 2 +- dev-tools/mage/gotool/modules.go | 5 + dev-tools/mage/settings.go | 23 +-- docs/devguide/newbeat.asciidoc | 41 +++++- generator/Makefile | 8 +- generator/_templates/beat/{beat}/Makefile | 11 +- .../beat/{beat}/_meta/beat.reference.yml | 7 + .../_templates/beat/{beat}/include/include.go | 1 + generator/_templates/beat/{beat}/magefile.go | 2 +- .../_templates/beat/{beat}/tools/tools.go | 14 ++ .../_templates/metricbeat/{beat}/Makefile | 11 +- .../_templates/metricbeat/{beat}/README.md | 11 +- .../metricbeat/{beat}/include/include.go | 1 + .../metricbeat/{beat}/tools/tools.go | 14 ++ generator/common/Makefile | 14 +- generator/common/beatgen/beatgen.go | 36 ++++- generator/common/beatgen/setup/creator.go | 2 +- generator/common/beatgen/setup/setup.go | 137 +++++++++--------- 21 files changed, 237 insertions(+), 139 deletions(-) create mode 100644 generator/_templates/beat/{beat}/_meta/beat.reference.yml create mode 100644 generator/_templates/beat/{beat}/include/include.go create mode 100644 generator/_templates/beat/{beat}/tools/tools.go create mode 100644 generator/_templates/metricbeat/{beat}/include/include.go create mode 100644 generator/_templates/metricbeat/{beat}/tools/tools.go diff --git a/.travis.yml b/.travis.yml index 9c5eae3e17c..82f87c1e08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -148,24 +148,23 @@ jobs: stage: test # Generators - # Temporarily disable generator jobs - #- os: linux - # env: TARGETS="-C generator/_templates/metricbeat test test-package" - # go: $TRAVIS_GO_VERSION - # stage: test - #- os: linux - # env: TARGETS="-C generator/_templates/beat test test-package" - # go: $TRAVIS_GO_VERSION - # stage: test + - os: linux + env: TARGETS="-C generator/_templates/metricbeat test test-package" + go: $TRAVIS_GO_VERSION + stage: test + - os: linux + env: TARGETS="-C generator/_templates/beat test test-package" + go: $TRAVIS_GO_VERSION + stage: test - #- os: osx - # env: TARGETS="-C generator/_templates/metricbeat test" - # go: $TRAVIS_GO_VERSION - # stage: test - #- os: osx - # env: TARGETS="-C generator/_templates/beat test" - # go: $TRAVIS_GO_VERSION - # stage: test + - os: osx + env: TARGETS="-C generator/_templates/metricbeat test" + go: $TRAVIS_GO_VERSION + stage: test + - os: osx + env: TARGETS="-C generator/_templates/beat test" + go: $TRAVIS_GO_VERSION + stage: test # Kubernetes - os: linux diff --git a/dev-tools/mage/gomod.go b/dev-tools/mage/gomod.go index 17f4e8917f6..7cc6b7d8c68 100644 --- a/dev-tools/mage/gomod.go +++ b/dev-tools/mage/gomod.go @@ -18,7 +18,6 @@ package mage import ( - "fmt" "os" "path/filepath" @@ -80,7 +79,6 @@ func Vendor() error { if err != nil { return err } - fmt.Println(path) for _, f := range p.filesToCopy { from := filepath.Join(path, f) diff --git a/dev-tools/mage/gotool/get.go b/dev-tools/mage/gotool/get.go index 6a147ca9392..43046019fdd 100644 --- a/dev-tools/mage/gotool/get.go +++ b/dev-tools/mage/gotool/get.go @@ -27,5 +27,6 @@ func runGoGet(opts ...ArgOpt) error { return runVGo("get", args) } +func (goGet) Download() ArgOpt { return flagBoolIf("-d", true) } func (goGet) Update() ArgOpt { return flagBoolIf("-u", true) } func (goGet) Package(pkg string) ArgOpt { return posArg(pkg) } diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go index 30eaa6ff87d..9e6cc08a471 100644 --- a/dev-tools/mage/gotool/go.go +++ b/dev-tools/mage/gotool/go.go @@ -105,7 +105,7 @@ func ListModulePath(pkg string) (string, error) { return "", err } if n := len(lines); n != 1 { - return "", fmt.Errorf("expected 1 line, got %d", n) + return "", fmt.Errorf("expected 1 line, got %d while looking for %s", n, pkg) } return lines[0], nil } diff --git a/dev-tools/mage/gotool/modules.go b/dev-tools/mage/gotool/modules.go index 239425ee321..ebaf2869ec5 100644 --- a/dev-tools/mage/gotool/modules.go +++ b/dev-tools/mage/gotool/modules.go @@ -19,6 +19,7 @@ package gotool // Mod is the command go mod. var Mod = goMod{ + Init: modCommand{"init"}.run, Tidy: modCommand{"tidy"}.run, Verify: modCommand{"verify"}.run, Vendor: modCommand{"vendor"}.run, @@ -39,11 +40,15 @@ func (cmd modCommand) run(opts ...ArgOpt) error { } type goMod struct { + Init modInit Tidy modTidy Verify modVerify Vendor modVendor } +// modInit initializes a new go module in folder. +type modInit func(opts ...ArgOpt) error + // modTidy cleans the go.mod file type modTidy func(opts ...ArgOpt) error diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index cc1a42f86ef..5b96e58e9f0 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -23,9 +23,7 @@ import ( "io/ioutil" "log" "os" - "path" "path/filepath" - "reflect" "regexp" "strconv" "strings" @@ -275,20 +273,15 @@ func ElasticBeatsDir() (string, error) { // it will return the root directory of the module from within the module cache or vendor // directory. func findElasticBeatsDir() (string, error) { - // Find the import path for the package containing this file. - type foo struct{} - typ := reflect.TypeOf(foo{}) - magepkgpath := typ.PkgPath() - - // Walk up the import path until we find the elastic/beats module path. - pkgpath := magepkgpath - for extractCanonicalRootImportPath(pkgpath) != elasticBeatsImportPath { - pkgpath = path.Dir(pkgpath) - if pkgpath == "." { - return "", errors.Errorf("failed to find %q from %q", elasticBeatsImportPath, magepkgpath) - } + repo, err := GetProjectRepoInfo() + if err != nil { + return "", err } - return gotool.ListModulePath(pkgpath) + if repo.IsElasticBeats() { + return repo.RootDir, nil + } + + return gotool.ListModulePath("github.com/elastic/beats/v7") } var ( diff --git a/docs/devguide/newbeat.asciidoc b/docs/devguide/newbeat.asciidoc index 02bae35078b..42e9dbf7056 100644 --- a/docs/devguide/newbeat.asciidoc +++ b/docs/devguide/newbeat.asciidoc @@ -26,6 +26,10 @@ The following topics describe how to build a new Beat: * <> * <> +The following topic provide information about moving generated Beats to go modules: + +* <> + [[newbeat-getting-ready]] === Getting Ready @@ -128,7 +132,7 @@ mage GenerateCustomBeat -------------------- The mage script will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`. -For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname. +For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname. Finally, pick the revision of elastic/beats you would like to depend on. [source,shell] --------- @@ -136,10 +140,13 @@ Enter a project name [examplebeat]: Countbeat Enter a github name [your-github-name]: {username} Enter a beat path [github.com/{username}/countbeat]: Enter a full name [Firstname Lastname]: {Full Name} +Enter the github.com/elastic/beats revision [master]: --------- The Beat generator creates a directory called `countbeat` inside of your project folder (e.g. {project folder}/github.com/{github id}/countbeat). +Please note that it is not possible to use revisions of Beats which does not support go modules. + You now have a raw template of the Beat, but you still need to <>. [[setting-up-beat]] @@ -480,3 +487,35 @@ func main() { When you're done with your new Beat, how about letting everyone know? Open a pull request to add your link to the {beats-ref}/community-beats.html[Community Beats] list. + +[[newbeat-migrate-gomodules]] +=== Migrate existing Beat to go modules + +Get started by making sure the contents of the vendored beats folder is +pushed to a remote repository. Otherwise, go will overwrite the folder +with the selected revision of `github.com/elastic/beats`. + +The following command will initialize a new module in your repo. + +[source,shell] +---------------------------------------------------------------------- +go mod init +---------------------------------------------------------------------- + +Make sure to add the approprite `replace` directives from the `go.mod` file +of Beats: https://github.com/elastic/beats/tree/master/go.mod +Unfortunately, this workaround is needed to make sure your Beat can be compiled. + +To depend on the latest `master` of `github.com/elastic/beats` run the following command: + +[source,shell] +---------------------------------------------------------------------- +go get github.com/elastic/beats@master +---------------------------------------------------------------------- + +To move the dependencies to vendor, you need to manually fetch the new +`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`. + +We suggest you read the following section to learn about maintaining dependencies +using go modules: +* https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies] diff --git a/generator/Makefile b/generator/Makefile index 92e1eddc61b..6f0122cc360 100644 --- a/generator/Makefile +++ b/generator/Makefile @@ -1,11 +1,11 @@ # Runs test build for the count metricset .PHONY: test test: - $(MAKE) -C beat test - $(MAKE) -C metricbeat test + $(MAKE) -C _templates/beat test + $(MAKE) -C _templates/metricbeat test # Cleans up environment .PHONY: clean clean: - @$(MAKE) -C beat clean - @$(MAKE) -C metricbeat clean + @$(MAKE) -C _templates/beat clean + @$(MAKE) -C _templates/metricbeat clean diff --git a/generator/_templates/beat/{beat}/Makefile b/generator/_templates/beat/{beat}/Makefile index 0246f73dc98..6ad41d81456 100644 --- a/generator/_templates/beat/{beat}/Makefile +++ b/generator/_templates/beat/{beat}/Makefile @@ -3,11 +3,12 @@ BEAT_PATH={beat_path} BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH})) SYSTEM_TESTS=false TEST_ENVIRONMENT=false -ES_BEATS?=./vendor/github.com/elastic/beats +ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7 +ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH} LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile -GOPACKAGES=$(shell govendor list -no-status +local) -GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)" -MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage +GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools) +GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)" +MAGE_IMPORT_PATH=github.com/magefile/mage NO_COLLECT=true CHECK_HEADERS_DISABLED=true @@ -16,4 +17,4 @@ CHECK_HEADERS_DISABLED=true .PHONY: copy-vendor copy-vendor: - mage vendorUpdate \ No newline at end of file + mage vendorUpdate diff --git a/generator/_templates/beat/{beat}/_meta/beat.reference.yml b/generator/_templates/beat/{beat}/_meta/beat.reference.yml new file mode 100644 index 00000000000..7d9bb3942c3 --- /dev/null +++ b/generator/_templates/beat/{beat}/_meta/beat.reference.yml @@ -0,0 +1,7 @@ +################### {Beat} Configuration Example ######################### + +############################# {Beat} ###################################### + +{beat}: + # Defines how often an event is sent to the output + period: 1s diff --git a/generator/_templates/beat/{beat}/include/include.go b/generator/_templates/beat/{beat}/include/include.go new file mode 100644 index 00000000000..8e731cdfc38 --- /dev/null +++ b/generator/_templates/beat/{beat}/include/include.go @@ -0,0 +1 @@ +package include diff --git a/generator/_templates/beat/{beat}/magefile.go b/generator/_templates/beat/{beat}/magefile.go index b79448c2aaa..d924f3c7946 100644 --- a/generator/_templates/beat/{beat}/magefile.go +++ b/generator/_templates/beat/{beat}/magefile.go @@ -25,7 +25,7 @@ func init() { devtools.BeatProjectType = devtools.CommunityProject } -// VendorUpdate updates elastic/beats/v7 in the vendor dir +// VendorUpdate updates the vendor dir func VendorUpdate() error { return beatgen.VendorUpdate() } diff --git a/generator/_templates/beat/{beat}/tools/tools.go b/generator/_templates/beat/{beat}/tools/tools.go new file mode 100644 index 00000000000..d816e58647f --- /dev/null +++ b/generator/_templates/beat/{beat}/tools/tools.go @@ -0,0 +1,14 @@ +// +build tools + +// This package contains the tool dependencies of the project. + +package tools + +import ( + _ "github.com/pierrre/gotestcover" + _ "golang.org/x/tools/cmd/goimports" + + _ "github.com/mitchellh/gox" + _ "github.com/reviewdog/reviewdog/cmd/reviewdog" + _ "golang.org/x/lint/golint" +) diff --git a/generator/_templates/metricbeat/{beat}/Makefile b/generator/_templates/metricbeat/{beat}/Makefile index 932b0508378..453344c8feb 100644 --- a/generator/_templates/metricbeat/{beat}/Makefile +++ b/generator/_templates/metricbeat/{beat}/Makefile @@ -3,10 +3,11 @@ BEAT_PATH={beat_path} BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH})) SYSTEM_TESTS=false TEST_ENVIRONMENT=false -ES_BEATS?=./vendor/github.com/elastic/beats -GOPACKAGES=$(shell govendor list -no-status +local) -GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)" -MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage +ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7 +ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH} +GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools) +GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)" +MAGE_IMPORT_PATH=github.com/magefile/mage CHECK_HEADERS_DISABLED=true # Path to the libbeat Makefile @@ -14,4 +15,4 @@ CHECK_HEADERS_DISABLED=true .PHONY: copy-vendor copy-vendor: - mage vendorUpdate \ No newline at end of file + mage vendorUpdate diff --git a/generator/_templates/metricbeat/{beat}/README.md b/generator/_templates/metricbeat/{beat}/README.md index cf71741f5fe..10e88d88b51 100644 --- a/generator/_templates/metricbeat/{beat}/README.md +++ b/generator/_templates/metricbeat/{beat}/README.md @@ -33,14 +33,11 @@ make collect This updates all fields and docs with the most recent changes. -## Use vendoring +## Dependency management -We recommend to use vendoring for your beat. This means the dependencies are put into your beat folder. The beats team currently uses [govendor](https://github.com/kardianos/govendor) for vendoring. +Beats uses go modules for dependency management. -``` -govendor init -govendor update +e -``` +In order to maintain your `vendor` folder, run `mage copyVendor`. This will create a directory `vendor` inside your repository. To make sure all dependencies for the Makefile commands are loaded from the vendor directory, find the following line in your Makefile: @@ -50,7 +47,7 @@ ES_BEATS=${GOPATH}/src/github.com/elastic/beats Replace it with: ``` -ES_BEATS=./vendor/github.com/elastic/beats +ES_BEATS=./vendor/github.com/elastic/beats/v7 ``` diff --git a/generator/_templates/metricbeat/{beat}/include/include.go b/generator/_templates/metricbeat/{beat}/include/include.go new file mode 100644 index 00000000000..8e731cdfc38 --- /dev/null +++ b/generator/_templates/metricbeat/{beat}/include/include.go @@ -0,0 +1 @@ +package include diff --git a/generator/_templates/metricbeat/{beat}/tools/tools.go b/generator/_templates/metricbeat/{beat}/tools/tools.go new file mode 100644 index 00000000000..d816e58647f --- /dev/null +++ b/generator/_templates/metricbeat/{beat}/tools/tools.go @@ -0,0 +1,14 @@ +// +build tools + +// This package contains the tool dependencies of the project. + +package tools + +import ( + _ "github.com/pierrre/gotestcover" + _ "golang.org/x/tools/cmd/goimports" + + _ "github.com/mitchellh/gox" + _ "github.com/reviewdog/reviewdog/cmd/reviewdog" + _ "golang.org/x/lint/golint" +) diff --git a/generator/common/Makefile b/generator/common/Makefile index d19c6e73778..0a7b3608dae 100644 --- a/generator/common/Makefile +++ b/generator/common/Makefile @@ -5,6 +5,8 @@ BEAT_PATH=${GOPATH}/src/${BEAT_NAME} ES_BEATS=${GOPATH}/src/github.com/elastic/beats PREPARE_COMMAND?= +-include ${ES_BEATS}/dev-tools/make/mage.mk + # Runs test build for mock beat .PHONY: test test: prepare-test @@ -23,36 +25,28 @@ test-package: test mage package .PHONY: prepare-test -prepare-test:: ${GOPATH}/bin/mage +prepare-test:: mage rm -fr ${BEAT_PATH} mkdir -p ${BEAT_PATH} cd ${GOPATH}/src/github.com/elastic/beats/ ; \ export MODULE=elastic ; \ export METRICSET=test ; \ - export GO111MODULE=off; \ export NEWBEAT_PROJECT_NAME=Testbeat ; \ export NEWBEAT_GITHUB_NAME=ruflin ; \ export NEWBEAT_BEAT_PATH=${BEAT_NAME} ; \ export NEWBEAT_FULL_NAME="Nicolas Ruflin" ; \ export NEWBEAT_TYPE=${BEAT_TYPE} ; \ export NEWBEAT_DEV=1 ; \ + export NEWBEAT_BEATS_REVISION=$(shell git rev-parse origin/master) ; \ mage GenerateCustomBeat # Runs test build for the created beat .PHONY: test-build test-build: test - # Copy dev tools - cp -r ${PWD}/../../../dev-tools ${BEAT_PATH}/vendor/github.com/elastic/beats/ - cd ${BEAT_PATH}/dev-tools/packer ; \ $(MAKE) deps ; \ $(MAKE) images -${GOPATH}/bin/mage: - go get -u -d github.com/magefile/mage - cd $${GOPATH}/src/github.com/magefile/mage; \ - go run bootstrap.go - # Cleans up environment .PHONY: clean clean: diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go index cd1d1900e88..a4a1f436a13 100644 --- a/generator/common/beatgen/beatgen.go +++ b/generator/common/beatgen/beatgen.go @@ -30,6 +30,7 @@ import ( "github.com/pkg/errors" devtools "github.com/elastic/beats/v7/dev-tools/mage" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" "github.com/elastic/beats/v7/generator/common/beatgen/setup" ) @@ -80,17 +81,29 @@ var configList = []ConfigItem{ return "beat" }, }, + { + Key: "beats_revision", + Help: "Enter the github.com/elastic/beats revision", + Default: func(cfg map[string]string) string { + return "master" + }, + }, } // Generate generates a new custom beat func Generate() error { cfg, err := getConfig() if err != nil { - return errors.Wrap(err, "Error getting config") + return errors.Wrap(err, "error getting config") } + beatsModuleName, err := gotool.GetModuleName() + if err != nil { + return err + } + err = setup.GenNewBeat(cfg) if err != nil { - return errors.Wrap(err, "Error generating new beat") + return errors.Wrap(err, "error generating new beat") } absBeatPath := filepath.Join(build.Default.GOPATH, "src", cfg["beat_path"]) @@ -100,8 +113,14 @@ func Generate() error { return errors.Wrap(err, "error changing directory") } + mg.Deps(setup.InitModule) + + err = getConfiguredBeatsRevision(beatsModuleName, cfg["beats_revision"]) + if err != nil { + return errors.Wrap(err, "error while getting required beats version") + } + mg.Deps(setup.CopyVendor) - mg.Deps(setup.RunSetup) mg.Deps(setup.GitInit) if cfg["type"] == "metricbeat" { @@ -122,7 +141,16 @@ func Generate() error { return nil } -// VendorUpdate updates the beat vendor directory +func getConfiguredBeatsRevision(beatsModule, revision string) error { + beatsPkg := beatsModule + "@" + revision + return gotool.Get( + gotool.Get.Download(), + gotool.Get.Update(), + gotool.Get.Package(beatsPkg), + ) +} + +// VendorUpdate updates the vendor directory func VendorUpdate() error { err := sh.Rm("./vendor/github.com/elastic/beats") if err != nil { diff --git a/generator/common/beatgen/setup/creator.go b/generator/common/beatgen/setup/creator.go index 8656bf5a5f1..d6e07d2228f 100644 --- a/generator/common/beatgen/setup/creator.go +++ b/generator/common/beatgen/setup/creator.go @@ -40,7 +40,7 @@ func GenNewBeat(config map[string]string) error { return fmt.Errorf("%s is not a valid custom beat type. Valid types are 'beat' and 'metricbeat'", config["type"]) } - genPath := devtools.OSSBeatDir("generator", config["type"], "{beat}") + genPath := devtools.OSSBeatDir("generator", "_templates", config["type"], "{beat}") err := filepath.Walk(genPath, func(path string, info os.FileInfo, err error) error { newBase := filepath.Join(build.Default.GOPATH, "src", config["beat_path"]) replacePath := strings.Replace(path, genPath, newBase, -1) diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go index b0bb5427e22..c4ef95c7654 100644 --- a/generator/common/beatgen/setup/setup.go +++ b/generator/common/beatgen/setup/setup.go @@ -18,7 +18,7 @@ package setup import ( - "fmt" + "bufio" "os" "path/filepath" @@ -26,103 +26,108 @@ import ( "github.com/pkg/errors" devtools "github.com/elastic/beats/v7/dev-tools/mage" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) -// RunSetup runs any remaining setup commands after the vendor directory has been setup -func RunSetup() error { - vendorPath := "./vendor/github.com/" - - //Copy mage stuff - toMkdir := filepath.Join(vendorPath, "magefile") - err := os.MkdirAll(toMkdir, 0755) - if err != nil { - return errors.Wrapf(err, "error making mage directory at %s", toMkdir) +var ( + makefileDeps = []string{ + "dev-tools", + "libbeat", + "licenses", + "metricbeat", + "script", } +) - err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/magefile/mage"), filepath.Join(vendorPath, "magefile")) +func InitModule() error { + err := gotool.Mod.Init() if err != nil { - return errors.Wrapf(err, "error copying vendored magefile to %s", filepath.Join(vendorPath, "magefile")) + return errors.Wrap(err, "error initializing a module for the Beat") } - //Copy the pkg helper - err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/pkg"), vendorPath) - if err != nil { - return errors.Wrapf(err, "error copying pkg to %s", vendorPath) - } - return nil + return copyReplacedModules() } -// CopyVendor copies a new version of beats into the vendor directory of PWD -// By default this uses git archive, meaning any uncommitted changes will not be copied. -// Set the NEWBEAT_DEV env variable to use a slow `cp` copy that will catch uncommited changes -func CopyVendor() error { - vendorPath := "./vendor/github.com/elastic/" +func copyReplacedModules() error { + const goModPath = "go.mod" + beatPath, err := devtools.ElasticBeatsDir() if err != nil { - return errors.Wrap(err, "Could not find ElasticBeatsDir") + return errors.Wrap(err, "error determining path to github.com/elastic/beats") } - err = os.MkdirAll(vendorPath, 0755) + + inMod, err := os.Open(filepath.Join(beatPath, goModPath)) if err != nil { - return errors.Wrap(err, "error creating vendor dir") + return errors.Wrap(err, "error opening go.mod file of github.com/elastic/beats") } + defer inMod.Close() - isClean, err := checkBeatsDirClean() - if err != nil { - return errors.Wrap(err, "error in checkIfBeatsDirClean") + s := bufio.NewScanner(inMod) + inReplaceSection := false + replacedLines := []string{ + "", + "replace (", } + for s.Scan() { + if err := s.Err(); err != nil { + return errors.Wrap(err, "error reading go.mod file") + } - if !isClean { - //Dev mode. Use CP. - fmt.Printf("You have uncommited changes in elastic/beats. Running CopyVendor running in dev mode, elastic/beats will be copied into the vendor directory with cp\n") - vendorPath = filepath.Join(vendorPath, "beats") + line := s.Text() + if inReplaceSection { + replacedLines = append(replacedLines, line) - err = sh.Run("cp", "-R", beatPath, vendorPath) - if err != nil { - return errors.Wrap(err, "error copying vendor dir") - } - err = sh.Rm(filepath.Join(vendorPath, ".git")) - if err != nil { - return errors.Wrap(err, "error removing vendor git directory") - } - err = sh.Rm(filepath.Join(vendorPath, "x-pack")) - if err != nil { - return errors.Wrap(err, "error removing x-pack directory") + if line == ")" { + break + } + continue } - } else { - //not dev mode. Use git archive - vendorPath = filepath.Join(vendorPath, "beats") - err = os.MkdirAll(vendorPath, 0755) - if err != nil { - return errors.Wrap(err, "error creating vendor dir") + + if line == "replace (" { + inReplaceSection = true } - err = sh.Run("sh", - "-c", - "git archive --remote "+beatPath+" HEAD | tar -x --exclude=x-pack -C "+vendorPath) + } + + outMod, err := os.OpenFile(goModPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) + if err != nil { + return errors.Wrap(err, "error opening the go.mod file of the generated Beat") + } + defer outMod.Close() + + w := bufio.NewWriter(outMod) + for _, rep := range replacedLines { + _, err = w.WriteString(rep + "\n") if err != nil { - return errors.Wrap(err, "error running git archive") + return errors.Wrap(err, "error writing replace lines to go.mod file") } } - return nil - + return w.Flush() } -// checkIfBeatsDirClean checks to see if the working elastic/beats dir is modified. -// If it is, we'll use a different method to copy beats to vendor/ -func checkBeatsDirClean() (bool, error) { - beatPath, err := devtools.ElasticBeatsDir() +// CopyVendor copies a new version of the dependencies to the vendor folder +func CopyVendor() error { + err := gotool.Mod.Vendor() if err != nil { - return false, errors.Wrap(err, "Could not find ElasticBeatsDir") + return err } - out, err := sh.Output("git", "-C", beatPath, "status", "--porcelain") + + path, err := gotool.ListModulePath("github.com/elastic/beats/v7") if err != nil { - return false, errors.Wrap(err, "Error checking status of elastic/beats repo") + return err } - if len(out) == 0 { - return true, nil + vendorPath := "./vendor/github.com/elastic/beats/v7" + for _, d := range makefileDeps { + from := filepath.Join(path, d) + to := filepath.Join(vendorPath, d) + copyTask := &devtools.CopyTask{Source: from, Dest: to, Mode: 0640, DirMode: os.ModeDir | 0750} + err = copyTask.Execute() + if err != nil { + return err + } } - return false, nil + return nil } // GitInit initializes a new git repo in the current directory