Go modules (#15853)

This PR introduces the new dependency management system for beats. From now on we are using `go mod` instead of `govendor`. The name of the module we provide is named `github.com/elastic/beats/v7`.

We adopted go modules, but we still keep dependencies under the folder `vendor`. However, it is maintained by `go mod` now by running `mage vendor`. Thus, it is not possible to add local changes to the dependencies there because next time someone runs the command it will be overwritten. **If you need to apply patches to dependencies either fork the repository or try to contribute it back to the original repo.**

This PR does not address the changes in Beat generators. Those are going to be updated in a follow-up PR.

### FAQ

#### How can I add a new dependency?

TL;DR
```sh
go get {{ module_name }}@{{ required_version }}
mage vendor
make notice
```

Run `go get {{ module_name }}@{{ required_version }}` in the repository. This adds the module to the requirements list in `go.mod`. In order to add the dependency to the vendor folder, run `mage vendor`. If the dependency contains files which are not copied by this command (e.g. C source files), add them to this list: https://github.com/elastic/beats/tree/master/dev-tools/mage/gomod.go

#### How can I upgrade a dependency?

The process is similar to adding a new dependency. Except for one step. If the dependency is updated to a newer major version, make sure to follow up changes in the root import paths of those libs. I suggest using the tool named `mod` to upgrade it: https://github.com/marwan-at-work/mod

#### How can I use a fork of a dependency?

If you need to use a fork of a dependency, add it to `go.mod` either manually or by running the following command:

```sh
go mod edit -replace {{ dependency_name }}={{ fork_name }}@{{ fork_version }}
```

#### Why is the major version part of the import path?

In a nutshell, it is needed to tell apart major version changes. Their import paths are different because those are incompatible. See more: https://blog.golang.org/v2-go-modules

Thus, when we are releasing v8.0.0, we need to change the import path again. The tool named mod is going to be useful for us in this case, too.

#### Why is `make check` failing?

Are you seeing the following error?

```
make[1]: Leaving directory `/home/travis/gopath/src/github.com/elastic/beats'
diff --git a/go.mod b/go.mod
index 170f6660..fc5fb6ee 100644
--- a/go.mod
+++ b/go.mod
@@ -53,6 +53,7 @@ require (
 	github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6
 	github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4
 	github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2
+	github.com/elastic/beats v7.6.0+incompatible
 	github.com/elastic/ecs v1.4.0
 	github.com/elastic/go-libaudit v0.4.0
 	github.com/elastic/go-licenser v0.2.1
```

If your `go.mod` lists `github.com/elastic/beats` as a requirement, the repo contains an outdated root import path. Change `github.com/elastic/beats` to `github.com/elastic/beats/v7`.

#### What is next?

After this PR is merged, please rebase all of your open PRs. In most cases rebasing will lead to conflicts. Possible sources of conflicts:
1. New import path conflicts with the old. To resolve it switch every `github.com/elastic/beats` import to `github.com/elasitc/beats/v7`.
2. A new dependency has been added or existing was updated. Update the dependency using go modules.

### Further reading for the interested

- [Golang blog: Using Go modules](https://blog.golang.org/using-go-modules)
- [Golang wiki](https://github.com/golang/go/wiki/Modules)

### Related issues

#15868

.travis.yml

@@ -148,23 +148,24 @@ jobs:
       stage: test
 
     # Generators
-    - os: linux
-      env: TARGETS="-C generator/metricbeat test test-package"
-      go: $TRAVIS_GO_VERSION
-      stage: test
-    - os: linux
-      env: TARGETS="-C generator/beat test test-package"
-      go: $TRAVIS_GO_VERSION
-      stage: test
+    # Temporarily disable generator jobs
+    #- os: linux
+    #  env: TARGETS="-C generator/_templates/metricbeat test test-package"
+    #  go: $TRAVIS_GO_VERSION
+    #  stage: test
+    #- os: linux
+    #  env: TARGETS="-C generator/_templates/beat test test-package"
+    #  go: $TRAVIS_GO_VERSION
+    #  stage: test
 
-    - os: osx
-      env: TARGETS="-C generator/metricbeat test"
-      go: $TRAVIS_GO_VERSION
-      stage: test
-    - os: osx
-      env: TARGETS="-C generator/beat test"
-      go: $TRAVIS_GO_VERSION
-      stage: test
+    #- os: osx
+    #  env: TARGETS="-C generator/_templates/metricbeat test"
+    #  go: $TRAVIS_GO_VERSION
+    #  stage: test
+    #- os: osx
+    #  env: TARGETS="-C generator/_templates/beat test"
+    #  go: $TRAVIS_GO_VERSION
+    #  stage: test
 
     # Kubernetes
     - os: linux

Jenkinsfile

@@ -196,6 +196,19 @@ pipeline {
             makeTarget("Auditbeat x-pack Linux", "-C x-pack/auditbeat testsuite")
           }
         }
+        stage('Auditbeat x-pack'){
+          agent { label 'ubuntu && immutable' }
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            expression {
+              return env.BUILD_AUDITBEAT_XPACK != "false"
+            }
+          }
+          steps {
+            makeTarget("Auditbeat x-pack Linux", "-C x-pack/auditbeat testsuite")
+          }
+        }
         stage('Libbeat'){
           agent { label 'ubuntu && immutable' }
           options { skipDefaultCheckout() }
@@ -459,44 +472,45 @@ pipeline {
             }
           }
         }
-        stage('Generators'){
-          agent { label 'ubuntu && immutable' }
-          options { skipDefaultCheckout() }
-          when {
-            beforeAgent true
-            expression {
-              return env.BUILD_GENERATOR != "false"
-            }
-          }
-          stages {
-            stage('Generators Metricbeat Linux'){
-              steps {
-                makeTarget("Generators Metricbeat Linux", "-C generator/metricbeat test")
-                makeTarget("Generators Metricbeat Linux", "-C generator/metricbeat test-package")
-              }
-            }
-            stage('Generators Beat Linux'){
-              steps {
-                makeTarget("Generators Beat Linux", "-C generator/beat test")
-                makeTarget("Generators Beat Linux", "-C generator/beat test-package")
-              }
-            }
-            stage('Generators Metricbeat Mac OS X'){
-              agent { label 'macosx' }
-              options { skipDefaultCheckout() }
-              steps {
-                makeTarget("Generators Metricbeat Mac OS X", "-C generator/metricbeat test")
-              }
-            }
-            stage('Generators Beat Mac OS X'){
-              agent { label 'macosx' }
-              options { skipDefaultCheckout() }
-              steps {
-                makeTarget("Generators Beat Mac OS X", "-C generator/beat test")
-              }
-            }
-          }
-        }
+        // Temporarily disable generator jobs
+        //stage('Generators'){
+        //  agent { label 'ubuntu && immutable' }
+        //  options { skipDefaultCheckout() }
+        //  when {
+        //    beforeAgent true
+        //    expression {
+        //      return env.BUILD_GENERATOR != "false"
+        //    }
+        //  }
+        //  stages {
+        //    stage('Generators Metricbeat Linux'){
+        //      steps {
+        //        makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test")
+        //        makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package")
+        //      }
+        //    }
+        //    stage('Generators Beat Linux'){
+        //      steps {
+        //        makeTarget("Generators Beat Linux", "-C generator/_templates/beat test")
+        //        makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package")
+        //      }
+        //    }
+        //    stage('Generators Metricbeat Mac OS X'){
+        //      agent { label 'macosx' }
+        //      options { skipDefaultCheckout() }
+        //      steps {
+        //        makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test")
+        //      }
+        //    }
+        //    stage('Generators Beat Mac OS X'){
+        //      agent { label 'macosx' }
+        //      options { skipDefaultCheckout() }
+        //      steps {
+        //        makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test")
+        //      }
+        //    }
+        //  }
+        //}
         stage('Kubernetes'){
           agent { label 'ubuntu && immutable' }
           options { skipDefaultCheckout() }
@@ -672,7 +686,7 @@ def loadConfigEnvVars(){
     "^x-pack/functionbeat/.*",
     "^x-pack/libbeat/.*",
   ])
-  env.BUILD_GENERATOR = isChanged(["^generator/.*"])
+  //env.BUILD_GENERATOR = isChanged(["^generator/.*"])
   env.BUILD_HEARTBEAT = isChanged(["^heartbeat/.*"])
   env.BUILD_HEARTBEAT_XPACK = isChanged([
     "^heartbeat/.*",

Makefile

@@ -81,12 +81,6 @@ clean: mage
 	@$(MAKE) -C generator clean
 	@-mage -clean
 
-# Cleans up the vendor directory from unnecessary files
-# This should always be run after updating the dependencies
-.PHONY: clean-vendor
-clean-vendor:
-	@sh script/clean_vendor.sh
-
 .PHONY: check
 check: python-env
 	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
@@ -95,6 +89,7 @@ check: python-env
 	@# Validate that all updates were committed
 	@$(MAKE) update
 	@$(MAKE) check-headers
+	go mod tidy
 	@git diff | cat
 	@git update-index --refresh
 	@git diff-index --exit-code HEAD --