Merge remote-tracking branch 'origin/master' into cloud-tests-tags

jsoriano · Mar 12, 2020 · c5e2c28 · c5e2c28
2 parents 3ebda8b + 1d36da7
commit c5e2c28
Show file tree

Hide file tree

Showing 357 changed files with 38,121 additions and 3,079 deletions.
diff --git a/.ci/scripts/travis_has_changes.sh b/.ci/scripts/travis_has_changes.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+# commit range to check for. For example master...<PR branch>
+RANGE=$TRAVIS_COMMIT_RANGE
+DIRLIST=$@
+
+# find modified files in range and filter out docs only changes
+CHANGED_FILES=$(git diff --name-only $RANGE | grep -v '.asciidoc')
+
+beginswith() { case $2 in "$1"*) true;; *) false;; esac }
+
+for path in $DIRLIST; do
+  for changed in $CHANGED_FILES; do
+    if beginswith $path $changed; then
+      exit 0 # found a match -> continue testing
+    fi
+  done
+done
+
+echo "NOT testing required. Modified files: $CHANGED_FILES"
+exit 1
diff --git a/.travis.yml b/.travis.yml
@@ -34,168 +34,203 @@ jobs:
 
     # Filebeat
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh filebeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C filebeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh filebeat libbeat vendor || travis_terminate 0
       env: TARGETS="TEST_ENVIRONMENT=0 -C filebeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh filebeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C x-pack/filebeat testsuite"
       go: $(GO_VERSION)
       stage: test
 
     # Heartbeat
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh heartbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C heartbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh heartbeat libbeat vendor || travis_terminate 0
       env: TARGETS="TEST_ENVIRONMENT=0 -C heartbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Auditbeat
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh auditbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C auditbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh auditbeat libbeat vendor || travis_terminate 0
       env: TARGETS="TEST_ENVIRONMENT=0 -C auditbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh auditbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C auditbeat crosscompile"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh auditbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C x-pack/auditbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Libbeat
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh libbeat vendor || travis_terminate 0
       env: TARGETS="-C libbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh libbeat vendor || travis_terminate 0
       env: TARGETS="-C libbeat crosscompile"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh libbeat vendor || travis_terminate 0
       env: STRESS_TEST_OPTIONS="-timeout=20m -race -v -parallel 1" TARGETS="-C libbeat stress-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh libbeat vendor || travis_terminate 0
       env: TARGETS="-C x-pack/libbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Metricbeat
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C metricbeat unit-tests coverage-report"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C metricbeat integration-tests-environment coverage-report"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C metricbeat update system-tests-environment coverage-report"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
       env: TARGETS="TEST_ENVIRONMENT=0 -C metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C metricbeat crosscompile"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C x-pack/metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Packetbeat
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh packetbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C packetbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Winlogbeat
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh winlogbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C winlogbeat crosscompile"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Functionbeat
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/functionbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C x-pack/functionbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/functionbeat libbeat vendor || travis_terminate 0
       env: TARGETS="TEST_ENVIRONMENT=0 -C x-pack/functionbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Docker Log Driver
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/dockerlogbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C x-pack/dockerlogbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Journalbeat
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh journalbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C journalbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Agent
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/agent libbeat vendor || travis_terminate 0
       env: TARGETS="-C x-pack/agent testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/agent libbeat vendor || travis_terminate 0
       env: TARGETS="TEST_ENVIRONMENT=0 -C x-pack/agent testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Generators
-    - os: linux
-      env: TARGETS="-C generator/_templates/metricbeat test test-package"
-      go: $TRAVIS_GO_VERSION
-      stage: test
-    - os: linux
-      env: TARGETS="-C generator/_templates/beat test test-package"
-      go: $TRAVIS_GO_VERSION
-      stage: test
+    # https://github.com/elastic/beats/issues/16951
+    #- os: linux
+    #  before_install: .ci/scripts/travis_has_changes.sh generator metricbeat libbeat vendor || travis_terminate 0
+    #  env: TARGETS="-C generator/_templates/metricbeat test test-package"
+    #  go: $TRAVIS_GO_VERSION
+    #  stage: test
+    #- os: linux
+    #  before_install: .ci/scripts/travis_has_changes.sh generator libbeat vendor || travis_terminate 0
+    #  env: TARGETS="-C generator/_templates/beat test test-package"
+    #  go: $TRAVIS_GO_VERSION
+    #  stage: test
 
     - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh generator metricbeat libbeat vendor || travis_terminate 0
       env: TARGETS="-C generator/_templates/metricbeat test"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh generator libbeat vendor || travis_terminate 0
       env: TARGETS="-C generator/_templates/beat test"
       go: $TRAVIS_GO_VERSION
       stage: test
 
     # Kubernetes
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh deploy/kubernetes metricbeat filebeat libbeat vendor || travis_terminate 0
       install: deploy/kubernetes/.travis/setup.sh
       env:
         - TARGETS="-C deploy/kubernetes test"
         - TRAVIS_K8S_VERSION=v1.9.4
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh deploy/kubernetes metricbeat filebeat libbeat  || travis_terminate 0
       install: deploy/kubernetes/.travis/setup.sh
       env:
         - TARGETS="-C deploy/kubernetes test"
         - TRAVIS_K8S_VERSION=v1.10.0
       stage: test
     - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh deploy/kubernetes metricbeat filebeat libbeat || travis_terminate 0
       dist: xenial
       install: deploy/kubernetes/.travis/setup.sh
       env:
@@ -239,7 +274,12 @@ addons:
       - python3.6
       - python3.6-venv
 
-before_install:
+
+
+# Skips installations step
+install: true
+
+before_script:
   - if [ x$TRAVIS_DIST = xtrusty ]; then sudo ln -sf python3.6 /usr/bin/python3; fi
   - python --version
   - python3 --version
@@ -252,10 +292,6 @@ before_install:
   - sudo mv docker-compose /usr/local/bin
   - if [ $TRAVIS_OS_NAME = osx ]; then pip install virtualenv==16.7.9; fi
 
-
-# Skips installations step
-install: true
-
 script:
   # Replacement for travis_wait which doesn't print output in real time.
   # Default Travis timeout is 10min, so this workaround prints timestamps every 9min to reset the counter.

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -66,7 +66,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix k8s pods annotations broken schema. {pull}16554[16554]
 - Upgrade go-ucfg to latest v0.8.3. {pull}16450{16450}
 - Fix `NewContainerMetadataEnricher` to use default config for kubernetes module. {pull}16857[16857]
-- Improve some logging messages for add_kubernetes_metadata processor {pull}16866{16866}
+- Improve some logging messages for add_kubernetes_metadata processor {pull}16866[16866]
+- Fix k8s metadata issue regarding node labels not shown up on root level of metadata. {pull}16834[16834]
 
 *Auditbeat*
 
@@ -112,6 +113,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix detection and logging of some error cases with light modules. {pull}14706[14706]
 - Fix imports after PR was merged before rebase. {pull}16756[16756]
 - Add dashboard for `redisenterprise` module. {pull}16752[16752]
+- Dynamically choose a method for the system/service metricset to support older linux distros. {pull}16902[16902]
 
 *Packetbeat*
 
@@ -140,6 +142,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Kerberos support to Kafka input and output. {pull}16781[16781]
 - Add `add_cloudfoundry_metadata` processor to annotate events with Cloud Foundry application data. {pull}16621[16621
 - Add support for kubernetes provider to recognize namespace level defaults {pull}16321[16321]
+- Add `translate_sid` processor on Windows for converting Windows security identifier (SID) values to names. {issue}7451[7451] {pull}16013[16013]
 
 *Auditbeat*
 
@@ -162,6 +165,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307]
 - Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500]
 - Improve ECS field mappings in haproxy module. {issue}16162[16162] {pull}16529[16529]
+- Add cloudwatch fileset and ec2 fileset in aws module. {issue}13716[13716] {pull}16579[16579]
 - Improve ECS categorization field mappings in kibana module. {issue}16168[16168] {pull}16652[16652]
 - Improve the decode_cef processor by reducing the number of memory allocations. {pull}16587[16587]
 - Add `cloudfoundry` input to send events from Cloud Foundry. {pull}16586[16586]
@@ -170,6 +174,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Allow users to override pipeline ID in fileset input config. {issue}9531[9531] {pull}16561[16561]
 - Add `o365audit` input type for consuming events from Office 365 Management Activity API. {issue}16196[16196] {pull}16244[16244]
 - Improve ECS categorization field mappings in logstash module. {issue}16169[16169] {pull}16668[16668]
+- Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354]
+- Improve ECS categorization field mapping in icinga module. {issue}16164[16164] {pull}16533[16533]
+- Improve ECS categorization field mappings in ibmmq module. {issue}16163[16163] {pull}16532[16532]
 
 *Heartbeat*
 
@@ -213,13 +220,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add filtering option for prometheus collector. {pull}16420[16420]
 - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254]
 - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280]
+- Add collecting tags and tags_filter for rds metricset in aws module. {pull}16605[16605] {issue}16358[16358]
 - Add OpenMetrics Metricbeat module {pull}16596[16596]
 - Add `cloudfoundry` module to send events from Cloud Foundry. {pull}16671[16671]
 - Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269]
 - Add system/users metricset as beta {pull}16569[16569]
 - Align fields to ECS and add more tests for the azure module. {issue}16024[16024] {pull}16754[16754]
 - Add additional cgroup fields to docker/diskio{pull}16638[16638]
 - Add PubSub metricset to Google Cloud Platform module {pull}15536[15536]
+- Add Prometheus remote write endpoint {pull}16609[16609]
 
 *Packetbeat*