diff --git a/.travis.yml b/.travis.yml index 9c5eae3e17c..82f87c1e08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -148,24 +148,23 @@ jobs: stage: test # Generators - # Temporarily disable generator jobs - #- os: linux - # env: TARGETS="-C generator/_templates/metricbeat test test-package" - # go: $TRAVIS_GO_VERSION - # stage: test - #- os: linux - # env: TARGETS="-C generator/_templates/beat test test-package" - # go: $TRAVIS_GO_VERSION - # stage: test + - os: linux + env: TARGETS="-C generator/_templates/metricbeat test test-package" + go: $TRAVIS_GO_VERSION + stage: test + - os: linux + env: TARGETS="-C generator/_templates/beat test test-package" + go: $TRAVIS_GO_VERSION + stage: test - #- os: osx - # env: TARGETS="-C generator/_templates/metricbeat test" - # go: $TRAVIS_GO_VERSION - # stage: test - #- os: osx - # env: TARGETS="-C generator/_templates/beat test" - # go: $TRAVIS_GO_VERSION - # stage: test + - os: osx + env: TARGETS="-C generator/_templates/metricbeat test" + go: $TRAVIS_GO_VERSION + stage: test + - os: osx + env: TARGETS="-C generator/_templates/beat test" + go: $TRAVIS_GO_VERSION + stage: test # Kubernetes - os: linux diff --git a/dev-tools/mage/gomod.go b/dev-tools/mage/gomod.go index 17f4e8917f6..7cc6b7d8c68 100644 --- a/dev-tools/mage/gomod.go +++ b/dev-tools/mage/gomod.go @@ -18,7 +18,6 @@ package mage import ( - "fmt" "os" "path/filepath" @@ -80,7 +79,6 @@ func Vendor() error { if err != nil { return err } - fmt.Println(path) for _, f := range p.filesToCopy { from := filepath.Join(path, f) diff --git a/dev-tools/mage/gotool/get.go b/dev-tools/mage/gotool/get.go index 6a147ca9392..43046019fdd 100644 --- a/dev-tools/mage/gotool/get.go +++ b/dev-tools/mage/gotool/get.go @@ -27,5 +27,6 @@ func runGoGet(opts ...ArgOpt) error { return runVGo("get", args) } +func (goGet) Download() ArgOpt { return flagBoolIf("-d", true) } func (goGet) Update() ArgOpt { return flagBoolIf("-u", true) } func (goGet) Package(pkg string) ArgOpt { return posArg(pkg) } diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go index 30eaa6ff87d..9e6cc08a471 100644 --- a/dev-tools/mage/gotool/go.go +++ b/dev-tools/mage/gotool/go.go @@ -105,7 +105,7 @@ func ListModulePath(pkg string) (string, error) { return "", err } if n := len(lines); n != 1 { - return "", fmt.Errorf("expected 1 line, got %d", n) + return "", fmt.Errorf("expected 1 line, got %d while looking for %s", n, pkg) } return lines[0], nil } diff --git a/dev-tools/mage/gotool/modules.go b/dev-tools/mage/gotool/modules.go index 239425ee321..ebaf2869ec5 100644 --- a/dev-tools/mage/gotool/modules.go +++ b/dev-tools/mage/gotool/modules.go @@ -19,6 +19,7 @@ package gotool // Mod is the command go mod. var Mod = goMod{ + Init: modCommand{"init"}.run, Tidy: modCommand{"tidy"}.run, Verify: modCommand{"verify"}.run, Vendor: modCommand{"vendor"}.run, @@ -39,11 +40,15 @@ func (cmd modCommand) run(opts ...ArgOpt) error { } type goMod struct { + Init modInit Tidy modTidy Verify modVerify Vendor modVendor } +// modInit initializes a new go module in folder. +type modInit func(opts ...ArgOpt) error + // modTidy cleans the go.mod file type modTidy func(opts ...ArgOpt) error diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index cc1a42f86ef..5b96e58e9f0 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -23,9 +23,7 @@ import ( "io/ioutil" "log" "os" - "path" "path/filepath" - "reflect" "regexp" "strconv" "strings" @@ -275,20 +273,15 @@ func ElasticBeatsDir() (string, error) { // it will return the root directory of the module from within the module cache or vendor // directory. func findElasticBeatsDir() (string, error) { - // Find the import path for the package containing this file. - type foo struct{} - typ := reflect.TypeOf(foo{}) - magepkgpath := typ.PkgPath() - - // Walk up the import path until we find the elastic/beats module path. - pkgpath := magepkgpath - for extractCanonicalRootImportPath(pkgpath) != elasticBeatsImportPath { - pkgpath = path.Dir(pkgpath) - if pkgpath == "." { - return "", errors.Errorf("failed to find %q from %q", elasticBeatsImportPath, magepkgpath) - } + repo, err := GetProjectRepoInfo() + if err != nil { + return "", err } - return gotool.ListModulePath(pkgpath) + if repo.IsElasticBeats() { + return repo.RootDir, nil + } + + return gotool.ListModulePath("github.com/elastic/beats/v7") } var ( diff --git a/docs/devguide/newbeat.asciidoc b/docs/devguide/newbeat.asciidoc index 02bae35078b..42e9dbf7056 100644 --- a/docs/devguide/newbeat.asciidoc +++ b/docs/devguide/newbeat.asciidoc @@ -26,6 +26,10 @@ The following topics describe how to build a new Beat: * <> * <> +The following topic provide information about moving generated Beats to go modules: + +* <> + [[newbeat-getting-ready]] === Getting Ready @@ -128,7 +132,7 @@ mage GenerateCustomBeat -------------------- The mage script will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`. -For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname. +For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname. Finally, pick the revision of elastic/beats you would like to depend on. [source,shell] --------- @@ -136,10 +140,13 @@ Enter a project name [examplebeat]: Countbeat Enter a github name [your-github-name]: {username} Enter a beat path [github.com/{username}/countbeat]: Enter a full name [Firstname Lastname]: {Full Name} +Enter the github.com/elastic/beats revision [master]: --------- The Beat generator creates a directory called `countbeat` inside of your project folder (e.g. {project folder}/github.com/{github id}/countbeat). +Please note that it is not possible to use revisions of Beats which does not support go modules. + You now have a raw template of the Beat, but you still need to <>. [[setting-up-beat]] @@ -480,3 +487,35 @@ func main() { When you're done with your new Beat, how about letting everyone know? Open a pull request to add your link to the {beats-ref}/community-beats.html[Community Beats] list. + +[[newbeat-migrate-gomodules]] +=== Migrate existing Beat to go modules + +Get started by making sure the contents of the vendored beats folder is +pushed to a remote repository. Otherwise, go will overwrite the folder +with the selected revision of `github.com/elastic/beats`. + +The following command will initialize a new module in your repo. + +[source,shell] +---------------------------------------------------------------------- +go mod init +---------------------------------------------------------------------- + +Make sure to add the approprite `replace` directives from the `go.mod` file +of Beats: https://github.com/elastic/beats/tree/master/go.mod +Unfortunately, this workaround is needed to make sure your Beat can be compiled. + +To depend on the latest `master` of `github.com/elastic/beats` run the following command: + +[source,shell] +---------------------------------------------------------------------- +go get github.com/elastic/beats@master +---------------------------------------------------------------------- + +To move the dependencies to vendor, you need to manually fetch the new +`magefile.go` for newly generated Beats from the dev-tools of `elastic/beats`. + +We suggest you read the following section to learn about maintaining dependencies +using go modules: +* https://github.com/golang/go/wiki/Modules#how-to-upgrade-and-downgrade-dependencies[How to upgrade and downgrade dependencies] diff --git a/generator/Makefile b/generator/Makefile index 92e1eddc61b..6f0122cc360 100644 --- a/generator/Makefile +++ b/generator/Makefile @@ -1,11 +1,11 @@ # Runs test build for the count metricset .PHONY: test test: - $(MAKE) -C beat test - $(MAKE) -C metricbeat test + $(MAKE) -C _templates/beat test + $(MAKE) -C _templates/metricbeat test # Cleans up environment .PHONY: clean clean: - @$(MAKE) -C beat clean - @$(MAKE) -C metricbeat clean + @$(MAKE) -C _templates/beat clean + @$(MAKE) -C _templates/metricbeat clean diff --git a/generator/_templates/beat/{beat}/Makefile b/generator/_templates/beat/{beat}/Makefile index 0246f73dc98..6ad41d81456 100644 --- a/generator/_templates/beat/{beat}/Makefile +++ b/generator/_templates/beat/{beat}/Makefile @@ -3,11 +3,12 @@ BEAT_PATH={beat_path} BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH})) SYSTEM_TESTS=false TEST_ENVIRONMENT=false -ES_BEATS?=./vendor/github.com/elastic/beats +ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7 +ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH} LIBBEAT_MAKEFILE=$(ES_BEATS)/libbeat/scripts/Makefile -GOPACKAGES=$(shell govendor list -no-status +local) -GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)" -MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage +GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools) +GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)" +MAGE_IMPORT_PATH=github.com/magefile/mage NO_COLLECT=true CHECK_HEADERS_DISABLED=true @@ -16,4 +17,4 @@ CHECK_HEADERS_DISABLED=true .PHONY: copy-vendor copy-vendor: - mage vendorUpdate \ No newline at end of file + mage vendorUpdate diff --git a/generator/_templates/beat/{beat}/_meta/beat.reference.yml b/generator/_templates/beat/{beat}/_meta/beat.reference.yml new file mode 100644 index 00000000000..7d9bb3942c3 --- /dev/null +++ b/generator/_templates/beat/{beat}/_meta/beat.reference.yml @@ -0,0 +1,7 @@ +################### {Beat} Configuration Example ######################### + +############################# {Beat} ###################################### + +{beat}: + # Defines how often an event is sent to the output + period: 1s diff --git a/generator/_templates/beat/{beat}/include/include.go b/generator/_templates/beat/{beat}/include/include.go new file mode 100644 index 00000000000..8e731cdfc38 --- /dev/null +++ b/generator/_templates/beat/{beat}/include/include.go @@ -0,0 +1 @@ +package include diff --git a/generator/_templates/beat/{beat}/magefile.go b/generator/_templates/beat/{beat}/magefile.go index b79448c2aaa..d924f3c7946 100644 --- a/generator/_templates/beat/{beat}/magefile.go +++ b/generator/_templates/beat/{beat}/magefile.go @@ -25,7 +25,7 @@ func init() { devtools.BeatProjectType = devtools.CommunityProject } -// VendorUpdate updates elastic/beats/v7 in the vendor dir +// VendorUpdate updates the vendor dir func VendorUpdate() error { return beatgen.VendorUpdate() } diff --git a/generator/_templates/beat/{beat}/tools/tools.go b/generator/_templates/beat/{beat}/tools/tools.go new file mode 100644 index 00000000000..d816e58647f --- /dev/null +++ b/generator/_templates/beat/{beat}/tools/tools.go @@ -0,0 +1,14 @@ +// +build tools + +// This package contains the tool dependencies of the project. + +package tools + +import ( + _ "github.com/pierrre/gotestcover" + _ "golang.org/x/tools/cmd/goimports" + + _ "github.com/mitchellh/gox" + _ "github.com/reviewdog/reviewdog/cmd/reviewdog" + _ "golang.org/x/lint/golint" +) diff --git a/generator/_templates/metricbeat/{beat}/Makefile b/generator/_templates/metricbeat/{beat}/Makefile index 932b0508378..453344c8feb 100644 --- a/generator/_templates/metricbeat/{beat}/Makefile +++ b/generator/_templates/metricbeat/{beat}/Makefile @@ -3,10 +3,11 @@ BEAT_PATH={beat_path} BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH})) SYSTEM_TESTS=false TEST_ENVIRONMENT=false -ES_BEATS?=./vendor/github.com/elastic/beats -GOPACKAGES=$(shell govendor list -no-status +local) -GOBUILD_FLAGS=-i -ldflags "-X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)" -MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage +ES_BEATS_IMPORT_PATH=github.com/elastic/beats/v7 +ES_BEATS?=./vendor/${ES_BEATS_IMPORT_PATH} +GOPACKAGES=$(shell go list ${BEAT_PATH}/... | grep -v /tools) +GOBUILD_FLAGS=-i -ldflags "-X ${ES_BEATS_IMPORT_PATH}/libbeat/version.buildTime=$(NOW) -X ${ES_BEATS_IMPORT_PATH}/libbeat/version.commit=$(COMMIT_ID)" +MAGE_IMPORT_PATH=github.com/magefile/mage CHECK_HEADERS_DISABLED=true # Path to the libbeat Makefile @@ -14,4 +15,4 @@ CHECK_HEADERS_DISABLED=true .PHONY: copy-vendor copy-vendor: - mage vendorUpdate \ No newline at end of file + mage vendorUpdate diff --git a/generator/_templates/metricbeat/{beat}/README.md b/generator/_templates/metricbeat/{beat}/README.md index cf71741f5fe..10e88d88b51 100644 --- a/generator/_templates/metricbeat/{beat}/README.md +++ b/generator/_templates/metricbeat/{beat}/README.md @@ -33,14 +33,11 @@ make collect This updates all fields and docs with the most recent changes. -## Use vendoring +## Dependency management -We recommend to use vendoring for your beat. This means the dependencies are put into your beat folder. The beats team currently uses [govendor](https://github.com/kardianos/govendor) for vendoring. +Beats uses go modules for dependency management. -``` -govendor init -govendor update +e -``` +In order to maintain your `vendor` folder, run `mage copyVendor`. This will create a directory `vendor` inside your repository. To make sure all dependencies for the Makefile commands are loaded from the vendor directory, find the following line in your Makefile: @@ -50,7 +47,7 @@ ES_BEATS=${GOPATH}/src/github.com/elastic/beats Replace it with: ``` -ES_BEATS=./vendor/github.com/elastic/beats +ES_BEATS=./vendor/github.com/elastic/beats/v7 ``` diff --git a/generator/_templates/metricbeat/{beat}/include/include.go b/generator/_templates/metricbeat/{beat}/include/include.go new file mode 100644 index 00000000000..8e731cdfc38 --- /dev/null +++ b/generator/_templates/metricbeat/{beat}/include/include.go @@ -0,0 +1 @@ +package include diff --git a/generator/_templates/metricbeat/{beat}/tools/tools.go b/generator/_templates/metricbeat/{beat}/tools/tools.go new file mode 100644 index 00000000000..d816e58647f --- /dev/null +++ b/generator/_templates/metricbeat/{beat}/tools/tools.go @@ -0,0 +1,14 @@ +// +build tools + +// This package contains the tool dependencies of the project. + +package tools + +import ( + _ "github.com/pierrre/gotestcover" + _ "golang.org/x/tools/cmd/goimports" + + _ "github.com/mitchellh/gox" + _ "github.com/reviewdog/reviewdog/cmd/reviewdog" + _ "golang.org/x/lint/golint" +) diff --git a/generator/common/Makefile b/generator/common/Makefile index d19c6e73778..0a7b3608dae 100644 --- a/generator/common/Makefile +++ b/generator/common/Makefile @@ -5,6 +5,8 @@ BEAT_PATH=${GOPATH}/src/${BEAT_NAME} ES_BEATS=${GOPATH}/src/github.com/elastic/beats PREPARE_COMMAND?= +-include ${ES_BEATS}/dev-tools/make/mage.mk + # Runs test build for mock beat .PHONY: test test: prepare-test @@ -23,36 +25,28 @@ test-package: test mage package .PHONY: prepare-test -prepare-test:: ${GOPATH}/bin/mage +prepare-test:: mage rm -fr ${BEAT_PATH} mkdir -p ${BEAT_PATH} cd ${GOPATH}/src/github.com/elastic/beats/ ; \ export MODULE=elastic ; \ export METRICSET=test ; \ - export GO111MODULE=off; \ export NEWBEAT_PROJECT_NAME=Testbeat ; \ export NEWBEAT_GITHUB_NAME=ruflin ; \ export NEWBEAT_BEAT_PATH=${BEAT_NAME} ; \ export NEWBEAT_FULL_NAME="Nicolas Ruflin" ; \ export NEWBEAT_TYPE=${BEAT_TYPE} ; \ export NEWBEAT_DEV=1 ; \ + export NEWBEAT_BEATS_REVISION=$(shell git rev-parse origin/master) ; \ mage GenerateCustomBeat # Runs test build for the created beat .PHONY: test-build test-build: test - # Copy dev tools - cp -r ${PWD}/../../../dev-tools ${BEAT_PATH}/vendor/github.com/elastic/beats/ - cd ${BEAT_PATH}/dev-tools/packer ; \ $(MAKE) deps ; \ $(MAKE) images -${GOPATH}/bin/mage: - go get -u -d github.com/magefile/mage - cd $${GOPATH}/src/github.com/magefile/mage; \ - go run bootstrap.go - # Cleans up environment .PHONY: clean clean: diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go index cd1d1900e88..a4a1f436a13 100644 --- a/generator/common/beatgen/beatgen.go +++ b/generator/common/beatgen/beatgen.go @@ -30,6 +30,7 @@ import ( "github.com/pkg/errors" devtools "github.com/elastic/beats/v7/dev-tools/mage" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" "github.com/elastic/beats/v7/generator/common/beatgen/setup" ) @@ -80,17 +81,29 @@ var configList = []ConfigItem{ return "beat" }, }, + { + Key: "beats_revision", + Help: "Enter the github.com/elastic/beats revision", + Default: func(cfg map[string]string) string { + return "master" + }, + }, } // Generate generates a new custom beat func Generate() error { cfg, err := getConfig() if err != nil { - return errors.Wrap(err, "Error getting config") + return errors.Wrap(err, "error getting config") } + beatsModuleName, err := gotool.GetModuleName() + if err != nil { + return err + } + err = setup.GenNewBeat(cfg) if err != nil { - return errors.Wrap(err, "Error generating new beat") + return errors.Wrap(err, "error generating new beat") } absBeatPath := filepath.Join(build.Default.GOPATH, "src", cfg["beat_path"]) @@ -100,8 +113,14 @@ func Generate() error { return errors.Wrap(err, "error changing directory") } + mg.Deps(setup.InitModule) + + err = getConfiguredBeatsRevision(beatsModuleName, cfg["beats_revision"]) + if err != nil { + return errors.Wrap(err, "error while getting required beats version") + } + mg.Deps(setup.CopyVendor) - mg.Deps(setup.RunSetup) mg.Deps(setup.GitInit) if cfg["type"] == "metricbeat" { @@ -122,7 +141,16 @@ func Generate() error { return nil } -// VendorUpdate updates the beat vendor directory +func getConfiguredBeatsRevision(beatsModule, revision string) error { + beatsPkg := beatsModule + "@" + revision + return gotool.Get( + gotool.Get.Download(), + gotool.Get.Update(), + gotool.Get.Package(beatsPkg), + ) +} + +// VendorUpdate updates the vendor directory func VendorUpdate() error { err := sh.Rm("./vendor/github.com/elastic/beats") if err != nil { diff --git a/generator/common/beatgen/setup/creator.go b/generator/common/beatgen/setup/creator.go index 8656bf5a5f1..d6e07d2228f 100644 --- a/generator/common/beatgen/setup/creator.go +++ b/generator/common/beatgen/setup/creator.go @@ -40,7 +40,7 @@ func GenNewBeat(config map[string]string) error { return fmt.Errorf("%s is not a valid custom beat type. Valid types are 'beat' and 'metricbeat'", config["type"]) } - genPath := devtools.OSSBeatDir("generator", config["type"], "{beat}") + genPath := devtools.OSSBeatDir("generator", "_templates", config["type"], "{beat}") err := filepath.Walk(genPath, func(path string, info os.FileInfo, err error) error { newBase := filepath.Join(build.Default.GOPATH, "src", config["beat_path"]) replacePath := strings.Replace(path, genPath, newBase, -1) diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go index b0bb5427e22..c4ef95c7654 100644 --- a/generator/common/beatgen/setup/setup.go +++ b/generator/common/beatgen/setup/setup.go @@ -18,7 +18,7 @@ package setup import ( - "fmt" + "bufio" "os" "path/filepath" @@ -26,103 +26,108 @@ import ( "github.com/pkg/errors" devtools "github.com/elastic/beats/v7/dev-tools/mage" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) -// RunSetup runs any remaining setup commands after the vendor directory has been setup -func RunSetup() error { - vendorPath := "./vendor/github.com/" - - //Copy mage stuff - toMkdir := filepath.Join(vendorPath, "magefile") - err := os.MkdirAll(toMkdir, 0755) - if err != nil { - return errors.Wrapf(err, "error making mage directory at %s", toMkdir) +var ( + makefileDeps = []string{ + "dev-tools", + "libbeat", + "licenses", + "metricbeat", + "script", } +) - err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/magefile/mage"), filepath.Join(vendorPath, "magefile")) +func InitModule() error { + err := gotool.Mod.Init() if err != nil { - return errors.Wrapf(err, "error copying vendored magefile to %s", filepath.Join(vendorPath, "magefile")) + return errors.Wrap(err, "error initializing a module for the Beat") } - //Copy the pkg helper - err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/pkg"), vendorPath) - if err != nil { - return errors.Wrapf(err, "error copying pkg to %s", vendorPath) - } - return nil + return copyReplacedModules() } -// CopyVendor copies a new version of beats into the vendor directory of PWD -// By default this uses git archive, meaning any uncommitted changes will not be copied. -// Set the NEWBEAT_DEV env variable to use a slow `cp` copy that will catch uncommited changes -func CopyVendor() error { - vendorPath := "./vendor/github.com/elastic/" +func copyReplacedModules() error { + const goModPath = "go.mod" + beatPath, err := devtools.ElasticBeatsDir() if err != nil { - return errors.Wrap(err, "Could not find ElasticBeatsDir") + return errors.Wrap(err, "error determining path to github.com/elastic/beats") } - err = os.MkdirAll(vendorPath, 0755) + + inMod, err := os.Open(filepath.Join(beatPath, goModPath)) if err != nil { - return errors.Wrap(err, "error creating vendor dir") + return errors.Wrap(err, "error opening go.mod file of github.com/elastic/beats") } + defer inMod.Close() - isClean, err := checkBeatsDirClean() - if err != nil { - return errors.Wrap(err, "error in checkIfBeatsDirClean") + s := bufio.NewScanner(inMod) + inReplaceSection := false + replacedLines := []string{ + "", + "replace (", } + for s.Scan() { + if err := s.Err(); err != nil { + return errors.Wrap(err, "error reading go.mod file") + } - if !isClean { - //Dev mode. Use CP. - fmt.Printf("You have uncommited changes in elastic/beats. Running CopyVendor running in dev mode, elastic/beats will be copied into the vendor directory with cp\n") - vendorPath = filepath.Join(vendorPath, "beats") + line := s.Text() + if inReplaceSection { + replacedLines = append(replacedLines, line) - err = sh.Run("cp", "-R", beatPath, vendorPath) - if err != nil { - return errors.Wrap(err, "error copying vendor dir") - } - err = sh.Rm(filepath.Join(vendorPath, ".git")) - if err != nil { - return errors.Wrap(err, "error removing vendor git directory") - } - err = sh.Rm(filepath.Join(vendorPath, "x-pack")) - if err != nil { - return errors.Wrap(err, "error removing x-pack directory") + if line == ")" { + break + } + continue } - } else { - //not dev mode. Use git archive - vendorPath = filepath.Join(vendorPath, "beats") - err = os.MkdirAll(vendorPath, 0755) - if err != nil { - return errors.Wrap(err, "error creating vendor dir") + + if line == "replace (" { + inReplaceSection = true } - err = sh.Run("sh", - "-c", - "git archive --remote "+beatPath+" HEAD | tar -x --exclude=x-pack -C "+vendorPath) + } + + outMod, err := os.OpenFile(goModPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0640) + if err != nil { + return errors.Wrap(err, "error opening the go.mod file of the generated Beat") + } + defer outMod.Close() + + w := bufio.NewWriter(outMod) + for _, rep := range replacedLines { + _, err = w.WriteString(rep + "\n") if err != nil { - return errors.Wrap(err, "error running git archive") + return errors.Wrap(err, "error writing replace lines to go.mod file") } } - return nil - + return w.Flush() } -// checkIfBeatsDirClean checks to see if the working elastic/beats dir is modified. -// If it is, we'll use a different method to copy beats to vendor/ -func checkBeatsDirClean() (bool, error) { - beatPath, err := devtools.ElasticBeatsDir() +// CopyVendor copies a new version of the dependencies to the vendor folder +func CopyVendor() error { + err := gotool.Mod.Vendor() if err != nil { - return false, errors.Wrap(err, "Could not find ElasticBeatsDir") + return err } - out, err := sh.Output("git", "-C", beatPath, "status", "--porcelain") + + path, err := gotool.ListModulePath("github.com/elastic/beats/v7") if err != nil { - return false, errors.Wrap(err, "Error checking status of elastic/beats repo") + return err } - if len(out) == 0 { - return true, nil + vendorPath := "./vendor/github.com/elastic/beats/v7" + for _, d := range makefileDeps { + from := filepath.Join(path, d) + to := filepath.Join(vendorPath, d) + copyTask := &devtools.CopyTask{Source: from, Dest: to, Mode: 0640, DirMode: os.ModeDir | 0750} + err = copyTask.Execute() + if err != nil { + return err + } } - return false, nil + return nil } // GitInit initializes a new git repo in the current directory