Merge remote-tracking branch 'upstream/master' into feature/ci-pipeli…

…ne-2.0 * upstream/master: (44 commits) Update users.asciidoc (elastic#20802) (elastic#21108) Fix docker provider builder. (elastic#21118) [Elastic Agent] Add docker composable dynamic provider. (elastic#20842) Add new modules/filesets from rsa2elk for 7.10 (elastic#20820) Fix broken links to external websites (elastic#21061) [docs] typo in the command line (elastic#20799) [Filebeat] add panos type and sub_type (elastic#20912) Move the `compute_vm_scalset` to a light metricset and map the cloud metadata (elastic#21038) [Filebeat] Add support for Cloudtrail digest files (elastic#21086) Add metrics collection from cost explorer into aws/billing metricset (elastic#20527) Add vendoring to Google Cloud Functions again (elastic#21070) [Elastic Agent] Add fleet.host.id for sending to endpoint. (elastic#21042) Do not need Google credentials before using it (elastic#21072) [Filebeat][New Module] Zoom webhook module (elastic#20414) Add support for GMT timezone offset in decode_cef (elastic#20993) Filebeat: Fix random error on harvester close (elastic#21048) Add ingress controller dashboards (elastic#21052) Fix loggers in composable module. (elastic#21047) [Ingest Manager] Increase kibana client timeout to 5 minutes (elastic#21037) Add changelog. (elastic#21041) ...
v1v · Sep 17, 2020 · 4c7e8da · 4c7e8da
2 parents b4a1faf + fd0fe73
commit 4c7e8da
Show file tree

Hide file tree

Showing 513 changed files with 392,768 additions and 31,331 deletions.
diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy
@@ -2,7 +2,7 @@
 @Library('apm@current') _
 
 pipeline {
-  agent none
+  agent { label 'master' }
   environment {
     REPO = 'apm-server'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
@@ -31,7 +31,7 @@ pipeline {
   }
   stages {
     stage('Filter build') {
-      agent { label 'ubuntu && immutable' }
+      agent { label 'ubuntu-18 && immutable' }
       when {
         beforeAgent true
         anyOf {
@@ -53,6 +53,7 @@ pipeline {
         Checkout the code and stash it, to use it on other stages.
         */
         stage('Checkout') {
+          options { skipDefaultCheckout() }
           steps {
             deleteDir()
             gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -3,6 +3,52 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.9.1]]
+=== Beats version 7.9.1
+https://github.com/elastic/beats/compare/v7.9.0...v7.9.1[View commits]
+
+==== Breaking changes
+
+*Affecting all Beats*
+
+- Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Update replicaset group to apps/v1 {pull}15854[15854]
+- Rename cloud.provider `az` value to `azure` inside the add_cloud_metadata processor. {pull}20689[20689]
+- Add missing country_name geo field in `add_host_metadata` and `add_observer_metadata` processors. {issue}20796[20796] {pull}20811[20811]
+
+*Filebeat*
+
+- Fix long registry migration times. {pull}20717[20717] {issue}20705[20705]
+- Fix event types and categories in auditd module to comply with ECS {pull}20652[20652]
+- Update documentation in the azure module filebeat. {pull}20815[20815]
+
+*Heartbeat*
+
+- Stop rescheduling tasks of stopped monitors. {pull}20570[20570]
+
+*Metricbeat*
+
+- Updates vm_compute metricset with more info on guest metrics. {pull}20448[20448]
+- Add fallback for PdhExpandWildCardPathW failing in perfmon metricset. {issue}20139[20139] {pull}20630[20630]
+- Fix resource tags in aws cloudwatch metricset {issue}20326[20326]  {pull}20385[20385]
+- Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736]
+
+*Winlogbeat*
+
+- Fix duplicated field error when exporting index-pattern with migration.6_to_7.enabled. {issue}20521[20521] {pull}20540[20540]
+- Fix `event.outcome` in the security module for non-English languages. {issue}20079[20079] {pull}20564[20564]
+
+==== Added
+
+*Affecting all Beats*
+
+- Added support for more message types for Cisco ASA and FTD. {pull}20565[20565]
+
 [[release-notes-7.9.0]]
 === Beats version 7.9.0
 https://github.com/elastic/beats/compare/v7.8.1...v7.9.0[View commits]

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -70,6 +70,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Tracking session end reason in panw module. {pull}18705[18705]
 - Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808]
 - Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]
+- Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
 
 *Heartbeat*
 
@@ -259,6 +260,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix long registry migration times. {pull}20717[20717] {issue}20705[20705]
 - Fix event types and categories in auditd module to comply with ECS {pull}20652[20652]
 - Update documentation in the azure module filebeat. {pull}20815[20815]
+- Provide backwards compatibility for the `set` processor when Elasticsearch is less than 7.9.0. {pull}20908[20908]
+- Remove wrongly mapped `tls.client.server_name` from `fortinet/firewall` fileset. {pull}20983[20983]
+- Fix an error updating file size being logged when EOF is reached. {pull}21048[21048]
+- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943]
 
 *Heartbeat*
 
@@ -267,6 +272,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for new `service_name` option to all monitors. {pull}19932[19932].
 - Stop rescheduling tasks of stopped monitors. {pull}20570[20570]
 
+*Heartbeat*
+
+
 *Journalbeat*
 
 
@@ -309,7 +317,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix "ID" event generator of Google Cloud module {issue}17160[17160] {pull}17608[17608]
 - Add privileged option for Auditbeat in Openshift {pull}17637[17637]
 - Fix storage metricset to allow config without region/zone. {issue}17623[17623] {pull}17624[17624]
-- Add a switch to the driver definition on SQL module to use pretty names. {pull}17378[17378]
 - Fix overflow on Prometheus rates when new buckets are added on the go. {pull}17753[17753]
 - Remove specific win32 api errors from events in perfmon. {issue}18292[18292] {pull}18361[18361]
 - Fix application_pool metricset after pdh changes. {pull}18477[18477]
@@ -330,15 +337,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601]
 - Fix k8s scheduler compatibility issue. {pull}19699[19699]
 - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898
-- Modify doc for app_insights metricset to contain example of config. {pull}20185[20185]
-- Add required option for `metrics` in app_insights. {pull}20406[20406]
-- Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403]
-- Updates vm_compute metricset with more info on guest metrics. {pull}20448[20448]
-- Add fallback for PdhExpandWildCardPathW failing in perfmon metricset. {issue}20139[20139] {pull}20630[20630]
-- Fix resource tags in aws cloudwatch metricset {issue}20326[20326]  {pull}20385[20385]
 - Fix ec2 disk and network metrics to use Sum statistic method. {pull}20680[20680]
 - Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736]
+- Update fields.yml in the azure module, missing metrics field. {pull}20918[20918]
 - The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938]
+- Disable Kafka metricsets based on Jolokia by default. They require a different configuration. {pull}20989[20989]
 
 *Packetbeat*
 
@@ -350,12 +353,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Fix invalid IP addresses in DNS query results from Sysmon data. {issue}18432[18432] {pull}18436[18436]
 - Fields from Winlogbeat modules were not being included in index templates and patterns. {pull}18983[18983]
-- Fix `event.outcome` in the security module for non-English languages. {issue}20079[20079] {pull}20564[20564]
-- Fix duplicated field error when exporting index-pattern with migration.6_to_7.enabled. {issue}20521[20521] {pull}20540[20540]
 
 *Functionbeat*
 
 - Fix timeout option of GCP functions. {issue}16282[16282] {pull}16287[16287]
+- Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072]
+- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070]
 
 ==== Added
 
@@ -407,8 +410,21 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
 - Add leader election for Kubernetes autodiscover. {pull}20281[20281]
 - Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767]
-- Added support for more message types for Cisco ASA and FTD. {pull}20565[20565]
 - Add replace_fields config option in add_host_metadata for replacing host fields. {pull}20490[20490] {issue}20464[20464]
+- Add container ECS fields in kubernetes metadata. {pull}20984[20984]
+- Add ingress controller dashboards. {pull}21052[21052]
+- Added experimental `citrix` module. {pull}20820[20820]
+- Added experimental `cyberark` module. {pull}20820[20820]
+- Added experimental `proofpoint` module. {pull}20820[20820]
+- Added experimental `snort` module. {pull}20820[20820]
+- Added experimental `symantec` module. {pull}20820[20820]
+- Added experimental dataset `barracuda/spamfirewall`. {pull}20820[20820]
+- Added experimental dataset `cisco/meraki`. {pull}20820[20820]
+- Added experimental dataset `f5/bigipafm`. {pull}20820[20820]
+- Added experimental dataset `fortinet/fortimail`. {pull}20820[20820]
+- Added experimental dataset `fortinet/fortimanager`. {pull}20820[20820]
+- Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
+- Added experimental dataset `sophos/utm`. {pull}20820[20820]
 
 *Auditbeat*
 
@@ -554,6 +570,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Convert httpjson to v2 input {pull}20226[20226]
 - Improve Zeek x509 module with `x509` ECS mappings {pull}20867[20867]
 - Improve Zeek SSL module with `x509` ECS mappings {pull}20927[20927]
+- Improve Zeek Kerberos module with `x509` ECS mappings {pull}20958[20958]
+- Improve Fortinet firewall module with `x509` ECS mappings {pull}20983[20983]
+- Improve Santa module with `x509` ECS mappings {pull}20976[20976]
+- Improve Suricata Eve module with `x509` ECS mappings {pull}20973[20973]
+- Added new module for Zoom webhooks {pull}20414[20414]
+- Add type and sub_type to panw panos fileset {pull}20912[20912]
 
 *Heartbeat*
 
@@ -669,9 +691,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add state_daemonset metricset for Kubernetes Metricbeat module {pull}20649[20649]
 - Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641]
 - Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391]
+- Add billing data collection from Cost Explorer into aws billing metricset. {pull}20527[20527] {issue}20103[20103]
+- Migrate `compute_vm` metricset to a light one, map `cloud.instance.id` field. {pull}20889[20889]
 - Request prometheus endpoints to be gzipped by default {pull}20766[20766]
 - Release all kubernetes `state` metricsets as GA {pull}20901[20901]
 - Add billing metricset into googlecloud module. {pull}20812[20812] {issue}20738[20738]
+- Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985]
+- Sanitize `event.host`. {pull}21022[21022]
 
 *Packetbeat*
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -38,6 +38,7 @@ pipeline {
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
     OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
+    PYTEST_ADDOPTS = "${params.PYTEST_ADDOPTS}"
   }
   options {
     timeout(time: 2, unit: 'HOURS')
@@ -61,6 +62,7 @@ pipeline {
     string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
     booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
     booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
+    string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`')
   }
   stages {
     /**
@@ -1125,6 +1127,7 @@ def isChangedOSSCode(patterns) {
   def allPatterns = [
     "^Jenkinsfile",
     "^go.mod",
+    "^pytest.ini",
     "^libbeat/.*",
     "^testing/.*",
     "^dev-tools/.*",
@@ -1138,6 +1141,7 @@ def isChangedXPackCode(patterns) {
   def allPatterns = [
     "^Jenkinsfile",
     "^go.mod",
+    "^pytest.ini",
     "^libbeat/.*",
     "^dev-tools/.*",
     "^testing/.*",

diff --git a/Makefile b/Makefile
@@ -167,7 +167,7 @@ notice:
 .PHONY: python-env
 python-env:
 	@test -d $(PYTHON_ENV) || ${PYTHON_EXE} -m venv $(VENV_PARAMS) $(PYTHON_ENV)
-	@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.3.5 pylint==2.4.4
+	@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.5.4 pylint==2.4.4
 	@# Work around pip bug. See: https://github.com/pypa/pip/issues/4464
 	@find $(PYTHON_ENV) -type d -name dist-packages -exec sh -c "echo dist-packages > {}.pth" ';'
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -2182,36 +2182,6 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/auto
    limitations under the License.
 
 
---------------------------------------------------------------------------------
-Dependency : github.com/Masterminds/semver
-Version: v1.4.2
-Licence type (autodetected): MIT
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/github.com/!masterminds/semver@v1.4.2/LICENSE.txt:
-
-The Masterminds
-Copyright (C) 2014-2015, Matt Butcher and Matt Farina
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
-
 --------------------------------------------------------------------------------
 Dependency : github.com/Microsoft/go-winio
 Version: v0.4.15-0.20190919025122-fc70bd9a86b5
@@ -2555,11 +2525,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/antlr/antlr4
-Version: v0.0.0-20200225173536-225249fdaef5
+Version: v0.0.0-20200820155224-be881fa6b91d
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/antlr/antlr4@v0.0.0-20200225173536-225249fdaef5/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/antlr/antlr4@v0.0.0-20200820155224-be881fa6b91d/LICENSE.txt:
 
 [The "BSD 3-clause license"]
 Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
@@ -13971,11 +13941,11 @@ Contents of probable licence file $GOMODCACHE/github.com/xdg/scram@v0.0.0-201808
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm
-Version: v1.8.1-0.20200902013556-b34fe04da73f
+Version: v1.8.1-0.20200909061013-2aef45b9cf4b
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm@v1.8.1-0.20200902013556-b34fe04da73f/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm@v1.8.1-0.20200909061013-2aef45b9cf4b/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -19451,6 +19421,36 @@ Contents of probable licence file $GOMODCACHE/github.com/!burnt!sushi/xgb@v0.0.0
 // such litigation is filed.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/Masterminds/semver
+Version: v1.4.2
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/!masterminds/semver@v1.4.2/LICENSE.txt:
+
+The Masterminds
+Copyright (C) 2014-2015, Matt Butcher and Matt Farina
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/Microsoft/hcsshim
 Version: v0.8.7

diff --git a/auditbeat/tests/system/test_file_integrity.py b/auditbeat/tests/system/test_file_integrity.py
@@ -31,7 +31,7 @@ def file_events(objs, path, expected):
     evts = set()
     for obj in objs:
         if 'file.path' in obj and 'event.action' in obj and obj['file.path'].lower() == path.lower():
-            if type(obj['event.action']) == list:
+            if isinstance(obj['event.action'], list):
                 evts = evts.union(set(obj['event.action']))
             else:
                 evts.add(obj['event.action'])

diff --git a/dev-tools/cmd/dashboards/export_5x_dashboards.py b/dev-tools/cmd/dashboards/export_5x_dashboards.py
@@ -13,7 +13,7 @@ def ExportDashboards(es, regex, kibana_index, output_directory):
 
     try:
         reg_exp = re.compile(regex, re.IGNORECASE)
-    except:
+    except BaseException:
         print("Wrong regex {}".format(regex))
         return
 

diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go
@@ -85,6 +85,25 @@ func ListDeps(pkg string) ([]string, error) {
 	return getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg))
 }
 
+// ListDepsLocation calls `go list -dep` for every package spec given.
+func ListDepsLocation(pkg string) (map[string]string, error) {
+	const tmpl = `{{if not .Standard}}{{.ImportPath}};{{.Dir}}{{end}}`
+
+	lines, err := getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg))
+	if err != nil {
+		return nil, err
+	}
+	deps := make(map[string]string, len(lines))
+	for _, l := range lines {
+		parts := strings.Split(l, ";")
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid number of parts")
+		}
+		deps[parts[0]] = parts[1]
+	}
+	return deps, nil
+}
+
 // ListTestFiles lists all go and cgo test files available in a package.
 func ListTestFiles(pkg string) ([]string, error) {
 	const tmpl = `{{ range .TestGoFiles }}{{ printf "%s\n" . }}{{ end }}` +

diff --git a/dev-tools/mage/install.go b/dev-tools/mage/install.go
@@ -37,7 +37,7 @@ func InstallVendored(importPath string) error {
 
 // InstallGoLicenser target installs go-licenser
 func InstallGoLicenser() error {
-	return gotool.Get(
-		gotool.Get.Package(GoLicenserImportPath),
+	return gotool.Install(
+		gotool.Install.Package(GoLicenserImportPath),
 	)
 }