Merge branch 'master' into fix_24268_netflow_seq_reset

.ci/packaging.groovy

@@ -434,9 +434,7 @@ def triggerE2ETests(String suite) {
     string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
   ]
   if (isPR()) {
-    def version = "pr-${env.CHANGE_ID}"
-    parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
-    parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
+    parameters.push(string(name: 'BEAT_VERSION', value: "pr-${env.CHANGE_ID}"))
   }
 
   build(job: "${e2eTestsPipeline}",

.ci/scripts/install-docker-compose.sh

@@ -15,13 +15,15 @@ fi
 
 if command -v docker-compose
 then
+    set +e
     echo "Found docker-compose. Checking version.."
     FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//)
     if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]
     then
         echo "Versions match. No need to install docker-compose. Exiting."
         exit 0
     fi
+    set -e
 fi
 
 echo "UNMET DEP: Installing docker-compose"

.ci/scripts/install-kind.sh

@@ -9,13 +9,15 @@ KIND_CMD="${HOME}/bin/kind"
 
 if command -v kind
 then
+    set +e
     echo "Found Kind. Checking version.."
     FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
     if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ]
     then
         echo "Versions match. No need to install Kind. Exiting."
         exit 0
     fi
+    set -e
 fi
 
 echo "UNMET DEP: Installing Kind"

.ci/scripts/install-kubectl.sh

@@ -9,13 +9,15 @@ KUBECTL_CMD="${HOME}/bin/kubectl"
 
 if command -v kubectl
 then
+    set +e
     echo "Found kubectl. Checking version.."
     FOUND_KUBECTL_VERSION=$(kubectl version --client --short 2>&1 >/dev/null | awk '{print $3}')
     if [ "${FOUND_KUBECTL_VERSION}" == "${K8S_VERSION}" ]
     then
         echo "Versions match. No need to install kubectl. Exiting."
         exit 0
     fi
+    set -e
 fi
 
 echo "UNMET DEP: Installing kubectl"

.ci/scripts/install-terraform.sh

@@ -9,13 +9,15 @@ TERRAFORM_CMD="${HOME}/bin/terraform"
 
 if command -v terraform
 then
+    set +e
     echo "Found Terraform. Checking version.."
     FOUND_TERRAFORM_VERSION=$(terraform --version | awk '{print $2}' | sed s/v//)
     if [ "$FOUND_TERRAFORM_VERSION" == "$TERRAFORM_VERSION" ]
     then
         echo "Versions match. No need to install Terraform. Exiting."
         exit 0
     fi
+    set -e
 fi
 
 echo "UNMET DEP: Installing Terraform"

CHANGELOG.asciidoc

@@ -3,6 +3,26 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.11.2]]
+=== Beats version 7.11.2
+https://github.com/elastic/beats/compare/v7.11.1...v7.11.2[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318]
+
+*Filebeat*
+
+- Fix Okta default date formatting. {issue}24018[24018] {pull}24025[24025]
+- Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167]
+- Add `nodes` to filebeat-kubernetes.yaml ClusterRole. {issue}24051[24051] {pull}24052[24052]
+
+*Metricbeat*
+
+- Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647]
+
 [[release-notes-7.11.1]]
 === Beats version 7.11.1
 https://github.com/elastic/beats/compare/v7.11.0...v7.11.1[View commits]

CHANGELOG.next.asciidoc

@@ -103,7 +103,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975]
 - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041]
 - Rename `s3` input to `aws-s3` input. {pull}23469[23469]
-- Add `nodes` to filebeat-kubernetes.yaml ClusterRole. {issue}24051[24051] {pull}24052[24052]
 
 *Heartbeat*
 - Adds negative body match. {pull}20728[20728]
@@ -130,7 +129,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - kubernetes.container.cpu.limit.cores and kubernetes.container.cpu.requests.cores are now floats. {issue}11975[11975]
 - Change types of numeric metrics from Kubelet summary api to double so as to cover big numbers. {pull}23335[23335]
 - Add container.image.name and containe.name ECS fields for state_container. {pull}23802[23802]
-- Add support for the MemoryPressure, DiskPressure, OutOfDisk and PIDPressure status conditions in state_node. {pull}[23905]
+- Add support for Consul 1.9. {pull}24123[24123]
+- Add support for the MemoryPressure, DiskPressure, OutOfDisk and PIDPressure status conditions in state_node. {pull}23905[23905]
 - Store `cloudfoundry.container.cpu.pct` in decimal form and as `scaled_float`. {pull}24219[24219]
 
 *Packetbeat*
@@ -160,6 +160,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- Fix templates being overwritten if there was an error when check for the template existance. {pull}24332[24332]
 - Fix Kubernetes autodiscovery provider to correctly handle pod states and avoid missing event data {pull}17223[17223]
 - Fix `add_cloud_metadata` to better support modifying sub-fields with other processors. {pull}13808[13808]
 - TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
@@ -236,6 +237,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add FAQ entry for madvdontneed variable {pull}23429[23429]
 - Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419]
 - Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484]
+- Allow configuring credential_profile_name and shared_credential_file when using role_arn. {pull}24174[24174]
 
 *Auditbeat*
 
@@ -257,6 +259,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693]
 - system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827]
 - Note incompatibility of system/socket on ARM. {pull}23381[23381]
+- system/login: Fixed offset reset on inode reuse. {pull}24414[24414]
 
 *Filebeat*
 
@@ -385,14 +388,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777]
 - Use rfc6587 framing for fortinet firewall and clientendpoint filesets when transferring over tcp. {pull}23837[23837]
 - Fix httpjson input logging so it doesn't conflict with ECS. {pull}23972[23972]
-- Fix Okta default date formatting. {issue}24018[24018] {pull}24025[24025]
 - Fix Logstash module handling of logstash.log.log_event.action field. {issue}20709[20709]
 - aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
 - Zoom module pipeline failed to ingest some chat_channel events. {pull}23904[23904]
 - Fix Netflow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110]
 - Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167]
+- in httpjson input using encode_as "application/x-www-form-urlencoded" now sets Content-Type correctly {issue}24331[24331] {pull}24336[24336]
 - Fix netflow module ignoring detect_sequence_reset flag. {issue}24268[24268] {pull}24270[24270]
 
+
 *Heartbeat*
 
 - Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639]
@@ -509,7 +513,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
 - Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286]
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
-- Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647]
 - Fix ec2 metricset fields.yml and the integration test {pull}23726[23726]
 - Unskip s3_request integration test. {pull}23887[23887]
 - Add system.hostfs configuration option for system module. {pull}23831[23831]
@@ -872,12 +875,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Upgrade elasticsearch/audit to ECS 1.8 {issue}23118[23118] {pull}24000[24000]
 - Upgrade okta to ecs 1.8.0 and move js processor to ingest pipeline {issue}23118[23118] {pull}23929[23929]
 - Update zoom module to ECS 1.8. {pull}23904[23904] {issue}23118[23118]
+- Support X-Forwarder-For in IIS logs. {pull}19142[192142]
 
 
 *Heartbeat*
 
 - Add mime type detection for http responses. {pull}22976[22976]
 - Bundle synthetics deps with heartbeat docker image. {pull}23274[23274]
+- Handle datastreams for fleet. {pull}24223[24223]
+- Add --sandbox option for browser monitor. {pull}24172[24172]
 
 *Journalbeat*
 
@@ -1005,6 +1011,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Enrich events of `state_service` metricset with kubernetes services' metadata. {pull}23730[23730]
 - Add support for Darwin/arm M1. {pull}24019[24019]
 - Check fields are documented in aws metricsets. {pull}23887[23887]
+- Add support for defining metrics_filters for prometheus module in hints. {pull}24264[24264]
+- Add support for PostgreSQL 10, 11, 12 and 13. {pull}24402[24402]
 
 *Packetbeat*
 

Jenkinsfile

@@ -507,12 +507,18 @@ def target(Map args = [:]) {
 }
 
 /**
-* This method wraps the node call with some latency to avoid the known issue with the scalabitity in gobld.
+* This method wraps the node call for two reasons:
+*  1. with some latency to avoid the known issue with the scalabitity in gobld.
+*  2. allocate a new workspace to workaround the flakiness of windows workers with deleteDir
 */
 def withNode(String label, Closure body) {
   sleep randomNumber(min: 10, max: 200)
+  // this should workaround the existing issue with reusing workers with the Gobld
+  def uuid = UUID.randomUUID().toString()
   node(label) {
-    body()
+    ws("workspace/${JOB_BASE_NAME}-${BUILD_NUMBER}-${uuid}") {
+      body()
+    }
   }
 }
 
@@ -555,7 +561,12 @@ def withBeatsEnv(Map args = [:], Closure body) {
     gox_flags = '-arch 386'
   }
 
-  deleteDir()
+  // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace.
+  //            Windows workers are ephemerals, so this should not really affect us.
+  if(isUnix()) {
+    deleteDir()
+  }
+
   unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
   // NOTE: This is required to run after the unstash
   def module = withModule ? getCommonModuleInTheChangeSet(directory) : ''
@@ -599,11 +610,24 @@ def withBeatsEnv(Map args = [:], Closure body) {
         if (archive) {
           archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
         }
-        // Tear down the setup for the permamnent workers.
-        catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
-          fixPermissions("${WORKSPACE}")
-          deleteDir()
-        }
+        tearDown()
+      }
+    }
+  }
+}
+
+/**
+* Tear down the setup for the permanent workers.
+*/
+def tearDown() {
+  catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
+    cmd(label: 'Remove the entire module cache', script: 'go clean -modcache', returnStatus: true)
+    fixPermissions("${WORKSPACE}")
+    // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace.
+    //            Windows workers are ephemerals, so this should not really affect us.
+    if (isUnix()) {
+      dir("${WORKSPACE}") {
+        deleteDir()
       }
     }
   }
@@ -618,6 +642,7 @@ def fixPermissions(location) {
   if(isUnix()) {
     sh(label: 'Fix permissions', script: """#!/usr/bin/env bash
       set +x
+      echo "Cleaning up ${location}"
       source ./dev-tools/common.bash
       docker_setup
       script/fix_permissions.sh ${location}""", returnStatus: true)

auditbeat/Jenkinsfile.yml

@@ -40,7 +40,7 @@ stages:
     macos:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.
-          - "macosx"
+          - "macosx&&x86_64"
         when:                  ## Override the top-level when.
             comments:
                 - "/test auditbeat for macos"

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -243,9 +243,13 @@ rules:
   - namespaces
   - events
   - pods
-  - secrets
   - services
   verbs: ["get", "list", "watch"]
+# Enable this rule only if planing to use Kubernetes keystore
+#- apiGroups: [""]
+#  resources:
+#  - secrets
+#  verbs: ["get"]
 - apiGroups: ["extensions"]
   resources:
   - replicasets

deploy/kubernetes/metricbeat/metricbeat-role.yaml

@@ -11,9 +11,13 @@ rules:
   - namespaces
   - events
   - pods
-  - secrets
   - services
   verbs: ["get", "list", "watch"]
+# Enable this rule only if planing to use Kubernetes keystore
+#- apiGroups: [""]
+#  resources:
+#  - secrets
+#  verbs: ["get"]
 - apiGroups: ["extensions"]
   resources:
   - replicasets

dev-tools/packaging/package_test.go

@@ -192,8 +192,8 @@ func checkDocker(t *testing.T, file string) {
 	checkDockerEntryPoint(t, p, info)
 	checkDockerLabels(t, p, info, file)
 	checkDockerUser(t, p, info, *rootUserContainer)
-	checkConfigPermissionsWithMode(t, p, os.FileMode(0640))
-	checkManifestPermissionsWithMode(t, p, os.FileMode(0640))
+	checkConfigPermissionsWithMode(t, p, os.FileMode(0644))
+	checkManifestPermissionsWithMode(t, p, os.FileMode(0644))
 	checkModulesPresent(t, "", p)
 	checkModulesDPresent(t, "", p)
 	checkLicensesPresent(t, "licenses/", p)

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -10,18 +10,18 @@ COPY beat {{ $beatHome }}
 
 RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/logs && \
     chown -R root:root {{ $beatHome }} && \
-    find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
-    find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
+    find {{ $beatHome }} -type d -exec chmod 0755 {} \; && \
+    find {{ $beatHome }} -type f -exec chmod 0644 {} \; && \
     find {{ $beatHome }}/data -type d -exec chmod 0770 {} \; && \
     find {{ $beatHome }}/data -type f -exec chmod 0660 {} \; && \
     rm {{ $beatBinary }} && \
     ln -s {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/elastic-agent {{ $beatBinary }} && \
-    chmod 0750 {{ $beatHome }}/data/elastic-agent-*/elastic-agent && \
+    chmod 0755 {{ $beatHome }}/data/elastic-agent-*/elastic-agent && \
 {{- if .linux_capabilities }}
     setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
 {{- end }}
 {{- range $i, $modulesd := .ModulesDirs }}
-    chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
+    chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
     true
 

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -10,16 +10,16 @@ COPY beat {{ $beatHome }}
 
 RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
     chown -R root:root {{ $beatHome }} && \
-    find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
-    find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
-    chmod 0750 {{ $beatBinary }} && \
+    find {{ $beatHome }} -type d -exec chmod 0755 {} \; && \
+    find {{ $beatHome }} -type f -exec chmod 0644 {} \; && \
+    chmod 0755 {{ $beatBinary }} && \
 {{- if .linux_capabilities }}
     setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
 {{- end }}
 {{- range $i, $modulesd := .ModulesDirs }}
-    chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
+    chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
-    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs
+    chmod 0775 {{ $beatHome }}/data {{ $beatHome }}/logs
 
 FROM {{ .from }}