Merge remote-tracking branch 'upstream/master' into feature/windows-10

* upstream/master: (60 commits)
  libbeat/cmd/instance: report cgroup stats (elastic#21113)
  Configurable index template loading (elastic#21212)
  [Ingest Manager] Thread safe sorted set (elastic#21290)
  Change mirror of kafka download (elastic#19645)
  [Ingest manager] Copy Action store on upgrade (elastic#21298)
  [CI] Pipeline 2.0 for monorepos (elastic#20104)
  Stop running agent container as root by default (elastic#21213)
  Stop running auditbeat container as root by default (elastic#21202)
  Fix autodiscover flaky tests (elastic#21242)
  [Ingest Manager] Enabled dev builds (elastic#21241)
  Fix librpm installation in auditbeat build (elastic#21239)
  Fix prometheus default config (elastic#21253)
  Fix dev guide test command (elastic#21254)
  Move aws lambda metricset to GA (elastic#21255)
  [Docs] Typo in table syntax (elastic#20227)
  [ECS] Adds related.hosts to capture all hostnames and host identifiers on an event. (elastic#21160)
  Add recursive split to httpjson (elastic#21214)
  [DOCS] Add beat specific start widgets (elastic#21217)
  Fix timestamp handling in remote_write (elastic#21166)
  Fix aws, azure and googlecloud compute dashboards (elastic#21098)
  ...

.ci/beats-tester.groovy

@@ -0,0 +1,106 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    BASE_DIR = 'src/github.com/elastic/beats'
+    PIPELINE_LOG_LEVEL = "INFO"
+    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/master'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    disableConcurrentBuilds()
+  }
+  triggers {
+    issueCommentTrigger('(?i)^\\/beats-tester$')
+    upstream("Beats/packaging/${env.JOB_BASE_NAME}")
+  }
+  stages {
+    stage('Filter build') {
+      agent { label 'ubuntu && immutable' }
+      when {
+        beforeAgent true
+        anyOf {
+          triggeredBy cause: "IssueCommentCause"
+          expression {
+            def ret = isUserTrigger() || isUpstreamTrigger()
+            if(!ret){
+              currentBuild.result = 'NOT_BUILT'
+              currentBuild.description = "The build has been skipped"
+              currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+              echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+            }
+            return ret
+          }
+        }
+      }
+      stages {
+        stage('Checkout') {
+          options { skipDefaultCheckout() }
+          steps {
+            deleteDir()
+            gitCheckout(basedir: "${BASE_DIR}")
+            setEnvVar('VERSION', sh(script: "grep ':stack-version:' ${BASE_DIR}/libbeat/docs/version.asciidoc | cut -d' ' -f2", returnStdout: true).trim())
+          }
+        }
+        stage('Build master') {
+          options { skipDefaultCheckout() }
+          when { branch 'master' }
+          steps {
+            runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
+          }
+        }
+        stage('Build *.x branch') {
+          options { skipDefaultCheckout() }
+          when { branch '*.x' }
+          steps {
+            runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
+          }
+        }
+        stage('Build PullRequest') {
+          options { skipDefaultCheckout() }
+          when { changeRequest() }
+          steps {
+            runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT",
+                              apm: "https://storage.googleapis.com/apm-ci-artifacts/jobs/pull-requests/pr-${env.CHANGE_ID}",
+                              beats: "https://storage.googleapis.com/beats-ci-artifacts/pull-requests/pr-${env.CHANGE_ID}")
+          }
+        }
+        stage('Build release branch') {
+          options { skipDefaultCheckout() }
+          when {
+            not {
+              allOf {
+                branch comparator: 'REGEXP', pattern: '(master|.*x)'
+                changeRequest()
+              }
+            }
+           }
+          steps {
+            runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
+          }
+        }
+      }
+    }
+  }
+}
+
+def runBeatsTesterJob(Map args = [:]) {
+  if (args.apm && args.beats) {
+    build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false,
+          parameters: [
+            string(name: 'APM_URL_BASE', value: args.apm),
+            string(name: 'BEATS_URL_BASE', value: args.beats),
+            string(name: 'VERSION', value: args.version)
+          ])
+  } else {
+    build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: args.version) ])
+  }
+}

.ci/jobs/beats-tester.yml

@@ -0,0 +1,56 @@
+---
+- job:
+    name: Beats/beats-tester
+    display-name: Beats Tester
+    description: Run the beats-tester
+    view: Beats
+    disabled: false
+    project-type: multibranch
+    script-path: .ci/beats-tester.groovy
+    scm:
+      - github:
+          branch-discovery: 'no-pr'
+          discover-pr-forks-strategy: 'merge-current'
+          discover-pr-forks-trust: 'permission'
+          discover-pr-origin: 'merge-current'
+          discover-tags: true
+          head-filter-regex: '(master|7\.([x9]|1\d+)|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+          disable-pr-notifications: true
+          notification-context: 'beats-tester'
+          repo: 'beats'
+          repo-owner: 'elastic'
+          credentials-id: github-app-beats-ci
+          ssh-checkout:
+            credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+          build-strategies:
+          - skip-initial-build: true
+          - tags:
+              ignore-tags-older-than: -1
+              ignore-tags-newer-than: 30
+          - named-branches:
+              - exact-name:
+                  name: 'master'
+                  case-sensitive: true
+              - regex-name:
+                  regex: '7\.([x9]|1\d+)'
+                  case-sensitive: true
+              - regex-name:
+                  regex: '8\.\d+'
+                  case-sensitive: true
+          - change-request:
+              ignore-target-only-changes: true
+          clean:
+              after: true
+              before: true
+          prune: true
+          shallow-clone: true
+          depth: 3
+          do-not-fetch-tags: true
+          submodule:
+              disable: false
+              recursive: true
+              parent-credentials: true
+              timeout: 100
+          timeout: '15'
+          use-author: true
+          wipe-workspace: true

.ci/scripts/generate_build_table.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+import os
+import yaml
+
+if __name__ == "__main__":
+
+    print("| Beat  | Stage  | Command  | MODULE  | Platforms  | When |")
+    print("|-------|--------|----------|---------|------------|------|")
+    for root, dirs, files in os.walk("."):
+        dirs.sort()
+        for file in files:
+            if file.endswith("Jenkinsfile.yml") and root != ".":
+                with open(os.path.join(root, file), 'r') as f:
+                    doc = yaml.load(f, Loader=yaml.FullLoader)
+                module = root.replace(".{}".format(os.sep), '')
+                platforms = [doc["platform"]]
+                when = ""
+                if "branches" in doc["when"]:
+                    when = f"{when}/:palm_tree:"
+                if "changeset" in doc["when"]:
+                    when = f"{when}/:file_folder:"
+                if "comments" in doc["when"]:
+                    when = f"{when}/:speech_balloon:"
+                if "labels" in doc["when"]:
+                    when = f"{when}/:label:"
+                if "parameters" in doc["when"]:
+                    when = f"{when}/:smiley:"
+                if "tags" in doc["when"]:
+                    when = f"{when}/:taco:"
+                for stage in doc["stages"]:
+                    withModule = False
+                    if "make" in doc["stages"][stage]:
+                        command = doc["stages"][stage]["make"]
+                    if "mage" in doc["stages"][stage]:
+                        command = doc["stages"][stage]["mage"]
+                    if "platforms" in doc["stages"][stage]:
+                        platforms = doc["stages"][stage]["platforms"]
+                    if "withModule" in doc["stages"][stage]:
+                        withModule = doc["stages"][stage]["withModule"]
+                    if "when" in doc["stages"][stage]:
+                        when = f"{when}/:star:"
+                    print("| {} | {} | `{}` | {} | `{}` | {} |".format(
+                        module, stage, command, withModule, platforms, when))
+
+print("> :palm_tree: -> Git Branch based")
+print("> :label: -> GitHub Pull Request Label based")
+print("> :file_folder: -> Changeset based")
+print("> :speech_balloon: -> GitHub Pull Request comment based")
+print("> :taco: -> Git tag based")
+print("> :smiley: -> Manual UI interaction based")
+print("> :star: -> More specific cases based")

.ci/scripts/get-vendor-dependencies.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Given the go module it will list all the dependencies that will be later on
+# used by the CI to enable/disable specific stages as long as the changeset
+# matches any of those patterns.
+#
+
+GO_VERSION=${GO_VERSION:?"GO_VERSION environment variable is not set"}
+BEATS=${1:?"parameter missing."}
+eval "$(gvm "${GO_VERSION}")"
+
+go list -deps ./"${BEATS}" \
+| grep 'elastic/beats' \
+| sort \
+| sed -e "s#github.com/elastic/beats/v7/##g" \
+| awk '{print "^" $1 "/.*"}'

.ci/scripts/install-tools.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+.ci/scripts/install-go.sh
+.ci/scripts/install-docker-compose.sh
+.ci/scripts/install-terraform.sh
+make mage