Merge branch 'master' into feature-stack-monitoring-mb-ecs

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [{ "name": "7.9"}, { "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
+  "branches": [ { "name": "7.x", "checked": true }, "7.10" ],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.ci/apm-beats-update.groovy

@@ -2,7 +2,7 @@
 @Library('apm@current') _
 
 pipeline {
-  agent none
+  agent { label 'master' }
   environment {
     REPO = 'apm-server'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
@@ -31,7 +31,7 @@ pipeline {
   }
   stages {
     stage('Filter build') {
-      agent { label 'ubuntu && immutable' }
+      agent { label 'ubuntu-18 && immutable' }
       when {
         beforeAgent true
         anyOf {
@@ -53,6 +53,7 @@ pipeline {
         Checkout the code and stash it, to use it on other stages.
         */
         stage('Checkout') {
+          options { skipDefaultCheckout() }
           steps {
             deleteDir()
             gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)

.ci/beats-tester-bc.groovy

@@ -0,0 +1,47 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    BASE_DIR = 'src/github.com/elastic/beats'
+    PIPELINE_LOG_LEVEL = "INFO"
+    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/master'
+    BASE_URL = "https://staging.elastic.co/${params.version}/downloads"
+    APM_BASE_URL = "${env.BASE_URL}/apm-server"
+    BEATS_BASE_URL = "${env.BASE_URL}/beats"
+    VERSION = "${params.version?.split('-')[0]}"
+  }
+  options {
+    timeout(time: 2, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    disableConcurrentBuilds()
+  }
+  parameters {
+    string(name: 'version', defaultValue: '', description: 'Id of the Build Candidate (7.10.0-b55684ff).')
+    string(name: 'BRANCH_REFERENCE', defaultValue: 'master', description: 'Branch to grab the Groovy script(for test changes).')
+  }
+  stages {
+    stage('Run Beat Tester') {
+      options { skipDefaultCheckout() }
+      when {
+        expression {
+          return '' != "${VERSION}"
+        }
+      }
+      steps {
+        build(job: env.BEATS_TESTER_JOB, propagate: true, wait: true,
+          parameters: [
+            string(name: 'APM_URL_BASE', value: "${APM_BASE_URL}"),
+            string(name: 'BEATS_URL_BASE', value: "${BEATS_BASE_URL}"),
+            string(name: 'VERSION', value: "${VERSION}")
+        ])
+      }
+    }
+  }
+}

.ci/beats-tester.groovy

@@ -0,0 +1,121 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    BASE_DIR = 'src/github.com/elastic/beats'
+    PIPELINE_LOG_LEVEL = "INFO"
+    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/master'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    disableConcurrentBuilds()
+  }
+  triggers {
+    issueCommentTrigger('(?i)^\\/beats-tester$')
+    upstream("Beats/packaging/${env.JOB_BASE_NAME}")
+  }
+  stages {
+    stage('Filter build') {
+      agent { label 'ubuntu && immutable' }
+      when {
+        beforeAgent true
+        anyOf {
+          triggeredBy cause: "IssueCommentCause"
+          expression {
+            def ret = isUserTrigger() || isUpstreamTrigger()
+            if(!ret){
+              currentBuild.result = 'NOT_BUILT'
+              currentBuild.description = "The build has been skipped"
+              currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+              echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+            }
+            return ret
+          }
+        }
+      }
+      stages {
+        stage('Checkout') {
+          options { skipDefaultCheckout() }
+          steps {
+            deleteDir()
+            gitCheckout(basedir: "${BASE_DIR}")
+            setEnvVar('VERSION', sh(script: "grep ':stack-version:' ${BASE_DIR}/libbeat/docs/version.asciidoc | cut -d' ' -f2", returnStdout: true).trim())
+          }
+        }
+        stage('Build master') {
+          options { skipDefaultCheckout() }
+          when { branch 'master' }
+          steps {
+            runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
+          }
+        }
+        stage('Build *.x branch') {
+          options { skipDefaultCheckout() }
+          when { branch '*.x' }
+          steps {
+            runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
+          }
+        }
+        stage('Build PullRequest') {
+          options { skipDefaultCheckout() }
+          when { changeRequest() }
+          steps {
+            runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT",
+                              apm: "https://storage.googleapis.com/apm-ci-artifacts/jobs/pull-requests/pr-${env.CHANGE_ID}",
+                              beats: "https://storage.googleapis.com/beats-ci-artifacts/pull-requests/pr-${env.CHANGE_ID}")
+          }
+        }
+        stage('Build release branch') {
+          options { skipDefaultCheckout() }
+          when {
+            not {
+              anyOf {
+                branch comparator: 'REGEXP', pattern: '(master|.*x)'
+                changeRequest()
+              }
+            }
+           }
+          steps {
+            // TODO: to use the git commit that triggered the upstream build
+            runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
+          }
+        }
+      }
+    }
+  }
+}
+
+def runBeatsTesterJob(Map args = [:]) {
+  def apm = args.get('apm', '')
+  def beats = args.get('beats', '')
+  def version = args.version
+
+  if (isUpstreamTrigger()) {
+    copyArtifacts(filter: 'beats-tester.properties',
+                  flatten: true,
+                  projectName: "Beats/packaging/${env.JOB_BASE_NAME}",
+                  selector: upstream(fallbackToLastSuccessful: true))
+    def props = readProperties(file: 'beats-tester.properties')
+    apm = props.get('APM_URL_BASE', '')
+    beats = props.get('BEATS_URL_BASE', '')
+    version = props.get('VERSION', '8.0.0-SNAPSHOT')
+  }
+  if (apm?.trim() || beats?.trim()) {
+    build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false,
+          parameters: [
+            string(name: 'APM_URL_BASE', value: apm),
+            string(name: 'BEATS_URL_BASE', value: beats),
+            string(name: 'VERSION', value: version)
+          ])
+  } else {
+    build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: version) ])
+  }
+}

.ci/jobs/apm-beats-update.yml

@@ -18,7 +18,7 @@
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
         discover-tags: true
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         disable-pr-notifications: true
         notification-context: 'apm-beats-update'
         repo: 'beats'
@@ -38,6 +38,9 @@
             - regex-name:
                 regex: '7\.[x789]'
                 case-sensitive: true
+            - regex-name:
+                regex: '7\.1\d'
+                case-sensitive: true
             - regex-name:
                 regex: '8\.\d+'
                 case-sensitive: true
@@ -48,7 +51,7 @@
             before: true
         prune: true
         shallow-clone: true
-        depth: 3
+        depth: 10
         do-not-fetch-tags: true
         submodule:
             disable: false

.ci/jobs/beats-release-changelog.yml

@@ -0,0 +1,20 @@
+---
+- job:
+    name: Beats/Release/beats-release-changelog
+    display-name: 'Prepare the Changelog for a Release'
+    description: 'Automate the steps to prepare the Changelog for a Release'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: release_scripts/pipeline-release-changelog.groovy
+        scm:
+        - git:
+            url: git@github.com:elastic/ingest-dev.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+            branches:
+                - master

.ci/jobs/beats-release-minor-major.yml

@@ -0,0 +1,20 @@
+---
+- job:
+    name: Beats/Release/beats-release-minor-major
+    display-name: 'Prepare Major/minor Release'
+    description: 'Automate the steps to prepare a new Release branch'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: release_scripts/pipeline-release-minor-major.groovy
+        scm:
+        - git:
+            url: git@github.com:elastic/ingest-dev.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+            branches:
+                - master

.ci/jobs/beats-release-patch.yml

@@ -0,0 +1,20 @@
+---
+- job:
+    name: Beats/Release/beats-release-patch
+    display-name: 'Prepare Patch Release'
+    description: 'Automate the steps to prepare a new Patch'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: release_scripts/pipeline-release-patch.groovy
+        scm:
+        - git:
+            url: git@github.com:elastic/ingest-dev.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+            branches:
+                - master

.ci/jobs/beats-tester-bc.yml

@@ -0,0 +1,26 @@
+---
+- job:
+    name: Beats/beats-tester-bc
+    display-name: Beats Tester orchestrator for Build Candidates
+    description: Launch the beat-tester suit on Build Candidates easily
+    view: Beats
+    disabled: false
+    project-type: pipeline
+    parameters:
+      - string:
+          name: BRANCH_REFERENCE
+          default: master
+          description: the Git branch specifier
+    pipeline-scm:
+      script-path: .ci/beats-tester-bc.groovy
+      scm:
+        - git:
+            url: git@github.com:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: true
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - $BRANCH_REFERENCE