Merge remote-tracking branch 'upstream/master' into feature/windows-10

* upstream/master:
  feat: package aliases for snapshots (elastic#21960)
  [DOC] Add firewall as possible troubleshooting issue (elastic#21743)
  [Filebeat] Add max_number_of_messages config parameter for S3 input (elastic#21993)
  [Elastic Agent] Fix missing elastic_agent event data  (elastic#21994)
  Document auditbeat system process module config (elastic#21766)
  Update links (elastic#22012)
  dynamically find librpm (elastic#21936)
  Fix Istio docs (elastic#22019)
  [beats-tester][packaging] store packages in another location (elastic#21903)
  [Kubernetes] Remove redundant dockersock volume mount (elastic#22009)
  [Ingest Manager] Always try snapshot repo for agent upgrade (elastic#21951)
  Azure storage metricset values not inside the metricset name (elastic#21845)
  fix diskio and memory bugs under windows (elastic#21992)
  Fix TestDockerStart flaky test (elastic#21681)
  filebeat: add SSL options to checkpoint module (elastic#19560)
  Stop storing stateless kubernetes keystores (elastic#21880)
  [Elastic Agent] Fix named pipe communication on Windows 7 (elastic#21931)
  [Elastic Agent] Fix index for Agent monitoring to to elastic_agent. (elastic#21932)

.ci/beats-tester.groovy

@@ -54,13 +54,15 @@ pipeline {
           options { skipDefaultCheckout() }
           when { branch 'master' }
           steps {
+            // TODO: to use the git commit that triggered the upstream build
             runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
           }
         }
         stage('Build *.x branch') {
           options { skipDefaultCheckout() }
           when { branch '*.x' }
           steps {
+            // TODO: to use the git commit that triggered the upstream build
             runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
           }
         }
@@ -84,6 +86,7 @@ pipeline {
             }
            }
           steps {
+            // TODO: to use the git commit that triggered the upstream build
             runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
           }
         }

.ci/packaging.groovy

@@ -191,10 +191,14 @@ def pushCIDockerImages(){
   }
 }
 
-def tagAndPush(name){
+def tagAndPush(beatName){
   def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+  def aliasVersion = ""
   if("${env.SNAPSHOT}" == "true"){
+    aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
+
     libbetaVer += "-SNAPSHOT"
+    aliasVersion += "-SNAPSHOT"
   }
 
   def tagName = "${libbetaVer}"
@@ -207,25 +211,37 @@ def tagAndPush(name){
   // supported image flavours
   def variants = ["", "-oss", "-ubi8"]
   variants.each { variant ->
-    def oldName = "${DOCKER_REGISTRY}/beats/${name}${variant}:${libbetaVer}"
-    def newName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${tagName}"
-    def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${env.GIT_BASE_COMMIT}"
-
-    def iterations = 0
-    retryWithSleep(retries: 3, seconds: 5, backoff: true) {
-      iterations++
-      def status = sh(label:'Change tag and push', script: """
-        docker tag ${oldName} ${newName}
-        docker push ${newName}
-        docker tag ${oldName} ${commitName}
-        docker push ${commitName}
-      """, returnStatus: true)
-
-      if ( status > 0 && iterations < 3) {
-        error('tag and push failed, retry')
-      } else if ( status > 0 ) {
-        log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
-      }
+    doTagAndPush(beatName, variant, libbetaVer, tagName)
+    doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")
+
+    if (!isPR() && aliasVersion != "") {
+      doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
+    }
+  }
+}
+
+/**
+* @param beatName name of the Beat
+* @param variant name of the variant used to build the docker image name
+* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
+* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
+*/
+def doTagAndPush(beatName, variant, sourceTag, targetTag) {
+  def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
+  def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
+
+  def iterations = 0
+  retryWithSleep(retries: 3, seconds: 5, backoff: true) {
+    iterations++
+    def status = sh(label: "Change tag and push ${targetName}", script: """
+      docker tag ${sourceName} ${targetName}
+      docker push ${targetName}
+    """, returnStatus: true)
+
+    if ( status > 0 && iterations < 3) {
+      error("tag and push failed for ${beatName}, retry")
+    } else if ( status > 0 ) {
+      log(level: 'WARN', text: "${beatName} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
     }
   }
 }
@@ -327,7 +343,16 @@ def publishPackages(baseDir){
     bucketUri = "gs://${JOB_GCS_BUCKET}/pull-requests/pr-${env.CHANGE_ID}"
   }
   def beatsFolderName = getBeatsName(baseDir)
-  googleStorageUpload(bucket: "${bucketUri}/${beatsFolderName}",
+  uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
+
+  // Copy those files to another location with the sha commit to test them
+  // aftewords.
+  bucketUri = "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}"
+  uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
+}
+
+def uploadPackages(bucketUri, baseDir){
+  googleStorageUpload(bucket: bucketUri,
     credentialsId: "${JOB_GCS_CREDENTIALS}",
     pathPrefix: "${baseDir}/build/distributions/",
     pattern: "${baseDir}/build/distributions/**/*",

CHANGELOG.next.asciidoc

@@ -375,6 +375,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix retrieving resources by ID for the azure module. {pull}21711[21711] {issue}21707[21707]
 - Use timestamp from CloudWatch API when creating events. {pull}21498[21498]
 - Report the correct windows events for system/filesystem {pull}21758[21758]
+- Fix azure storage event format. {pull}21845[21845]
+- Fix panic in kubernetes autodiscover related to keystores {issue}21843[21843] {pull}21880[21880]
+- [Kubernetes] Remove redundant dockersock volume mount {pull}22009[22009]
 
 *Packetbeat*
 
@@ -630,6 +633,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - New juniper.srx dataset for Juniper SRX logs. {pull}20017[20017]
 - Adding support for Microsoft 365 Defender (Microsoft Threat Protection) {pull}21446[21446]
 - Adding support for FIPS in s3 input {pull}21446[21446]
+- Add SSL option to checkpoint module {pull}19560[19560]
+- Add max_number_of_messages config into s3 input. {pull}21993[21993]
 
 *Heartbeat*
 

NOTICE.txt

@@ -2183,12 +2183,12 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/auto
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/Microsoft/go-winio
-Version: v0.4.15-0.20190919025122-fc70bd9a86b5
+Dependency : github.com/bi-zone/go-winio
+Version: v0.4.15
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!microsoft/go-winio@v0.4.15-0.20190919025122-fc70bd9a86b5/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/bi-zone/go-winio@v0.4.15/LICENSE:
 
 The MIT License (MIT)
 

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -189,8 +189,6 @@ spec:
         - name: modules
           mountPath: /usr/share/metricbeat/modules.d
           readOnly: true
-        - name: dockersock
-          mountPath: /var/run/docker.sock
         - name: proc
           mountPath: /hostfs/proc
           readOnly: true
@@ -204,9 +202,6 @@ spec:
       - name: cgroup
         hostPath:
           path: /sys/fs/cgroup
-      - name: dockersock
-        hostPath:
-          path: /var/run/docker.sock
       - name: config
         configMap:
           defaultMode: 0640

deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml

@@ -64,8 +64,6 @@ spec:
         - name: modules
           mountPath: /usr/share/metricbeat/modules.d
           readOnly: true
-        - name: dockersock
-          mountPath: /var/run/docker.sock
         - name: proc
           mountPath: /hostfs/proc
           readOnly: true
@@ -79,9 +77,6 @@ spec:
       - name: cgroup
         hostPath:
           path: /sys/fs/cgroup
-      - name: dockersock
-        hostPath:
-          path: /var/run/docker.sock
       - name: config
         configMap:
           defaultMode: 0640

filebeat/docs/modules/checkpoint.asciidoc

@@ -70,6 +70,18 @@ A list of tags to include in events. Including `forwarded` indicates that the
 events did not originate on this host and causes `host.name` to not be added to
 events. Defaults to `[checkpoint-firewall, forwarded]`.
 
+*`var.ssl`*::
+
+The SSL/TLS configuration for the filebeat instance. This can be used to enforce mutual TLS.
+```yaml
+ssl:
+  enabled: true
+  certificate_authorities: ["my-ca.pem"]
+  certificate: "filebeat-cert.pem"
+  key: "filebeat-key.pem"
+  client_authentication: "required"
+```
+
 [float]
 ==== Check Point devices
 

go.mod

@@ -189,6 +189,7 @@ require (
 
 replace (
 	github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.2.0+incompatible
+	github.com/Microsoft/go-winio => github.com/bi-zone/go-winio v0.4.15
 	github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec
 	github.com/cucumber/godog => github.com/cucumber/godog v0.8.1
 	github.com/docker/docker => github.com/docker/engine v0.0.0-20191113042239-ea84732a7725

go.sum

@@ -80,8 +80,6 @@ github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
-github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6trr62pF5DucadTWGdEB4mEyvzi0e2nbcmcyA=
-github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
 github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg=
 github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -132,6 +130,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bi-zone/go-winio v0.4.15 h1:viLHm+U7bzIkfVHuWgc3Wp/sT5zaLoRG7XdOEy1b12w=
+github.com/bi-zone/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
 github.com/blakerouse/service v1.1.1-0.20200924160513-057808572ffa h1:aXHPZwx8Y5z8r+1WPylnu095usTf6QSshaHs6nVMBc0=
 github.com/blakerouse/service v1.1.1-0.20200924160513-057808572ffa/go.mod h1:RrJI2xn5vve/r32U5suTbeaSGoMU6GbNPoj36CVYcHc=
 github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 h1:oMCHnXa6CCCafdPDbMh/lWRhRByN0VFLvv+g+ayx1SI=

libbeat/autodiscover/providers/docker/docker_integration_test.go

@@ -36,8 +36,6 @@ import (
 
 // Test docker start emits an autodiscover event
 func TestDockerStart(t *testing.T) {
-	t.Skip("#20360 Flaky TestDockerStart skipped")
-
 	log := logp.NewLogger("docker")
 
 	d, err := dk.NewClient()
@@ -70,15 +68,17 @@ func TestDockerStart(t *testing.T) {
 	// Start
 	cmd := []string{"echo", "Hi!"}
 	labels := map[string]string{"label": "foo", "label.child": "bar"}
-	ID, err := d.ContainerStart("busybox", cmd, labels)
+	ID, err := d.ContainerStart("busybox:latest", cmd, labels)
 	if err != nil {
 		t.Fatal(err)
 	}
-	checkEvent(t, listener, true)
+	defer d.ContainerRemove(ID)
+
+	checkEvent(t, listener, ID, true)
 
 	// Kill
 	d.ContainerKill(ID)
-	checkEvent(t, listener, false)
+	checkEvent(t, listener, ID, false)
 }
 
 func getValue(e bus.Event, key string) interface{} {
@@ -89,12 +89,13 @@ func getValue(e bus.Event, key string) interface{} {
 	return val
 }
 
-func checkEvent(t *testing.T, listener bus.Listener, start bool) {
+func checkEvent(t *testing.T, listener bus.Listener, id string, start bool) {
+	timeout := time.After(60 * time.Second)
 	for {
 		select {
 		case e := <-listener.Events():
 			// Ignore any other container
-			if getValue(e, "docker.container.image") != "busybox" {
+			if getValue(e, "container.id") != id {
 				continue
 			}
 			if start {
@@ -104,7 +105,7 @@ func checkEvent(t *testing.T, listener bus.Listener, start bool) {
 				assert.Equal(t, getValue(e, "stop"), true)
 				assert.Nil(t, getValue(e, "start"))
 			}
-			assert.Equal(t, getValue(e, "container.image.name"), "busybox")
+			assert.Equal(t, getValue(e, "container.image.name"), "busybox:latest")
 			// labels.dedot=true by default
 			assert.Equal(t,
 				common.MapStr{
@@ -122,8 +123,7 @@ func checkEvent(t *testing.T, listener bus.Listener, start bool) {
 			assert.Equal(t, getValue(e, "docker.container.name"), getValue(e, "meta.container.name"))
 			assert.Equal(t, getValue(e, "docker.container.image"), getValue(e, "meta.container.image.name"))
 			return
-
-		case <-time.After(10 * time.Second):
+		case <-timeout:
 			t.Fatal("Timeout waiting for provider events")
 			return
 		}

libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go

@@ -30,14 +30,10 @@ import (
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
-type KubernetesKeystores map[string]keystore.Keystore
-
-// KubernetesKeystoresRegistry holds KubernetesKeystores for known namespaces. Once a Keystore for one k8s namespace
-// is initialized it will be reused every time it is needed.
+// KubernetesKeystoresRegistry implements a Provider for Keystore.
 type KubernetesKeystoresRegistry struct {
-	kubernetesKeystores KubernetesKeystores
-	logger              *logp.Logger
-	client              k8s.Interface
+	logger *logp.Logger
+	client k8s.Interface
 }
 
 // KubernetesSecretsKeystore allows to retrieve passwords from Kubernetes secrets for a given namespace
@@ -56,9 +52,8 @@ func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface, logger *logp.
 // NewKubernetesKeystoresRegistry initializes a KubernetesKeystoresRegistry
 func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) keystore.Provider {
 	return &KubernetesKeystoresRegistry{
-		kubernetesKeystores: KubernetesKeystores{},
-		logger:              logger,
-		client:              client,
+		logger: logger,
+		client: client,
 	}
 }
 
@@ -75,12 +70,7 @@ func (kr *KubernetesKeystoresRegistry) GetKeystore(event bus.Event) keystore.Key
 		namespace = ns.(string)
 	}
 	if namespace != "" {
-		// either retrieve already stored keystore or create a new one for the namespace
-		if storedKeystore, ok := kr.kubernetesKeystores[namespace]; ok {
-			return storedKeystore
-		}
 		k8sKeystore, _ := Factoryk8s(namespace, kr.client, kr.logger)
-		kr.kubernetesKeystores["namespace"] = k8sKeystore
 		return k8sKeystore
 	}
 	kr.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event)

libbeat/docs/getting-started.asciidoc

@@ -13,5 +13,5 @@ Each Beat is a separately installable product. To learn how to get started, see:
 * {winlogbeat-ref}/winlogbeat-installation-configuration.html[Winlogbeat]
 
 If you're planning to use the {metrics-app} or the {logs-app} in {kib},
-also see the {metrics-guide}[Metrics Monitoring Guide]
-and the {logs-guide}[Logs Monitoring Guide].
+see {observability-guide}/analyze-metrics.html[Analyze metrics]
+and {observability-guide}/monitor-logs.html[Monitor logs].

libbeat/docs/howto/load-dashboards.asciidoc

@@ -15,8 +15,8 @@
 ifdef::has_solutions[]
 TIP: For deeper observability into your infrastructure, you can use the
 {metrics-app} and the {logs-app} in {kib}.
-For more details, see the {metrics-guide}[Metrics Monitoring Guide]
-and the {logs-guide}[Logs Monitoring Guide].
+For more details, see {observability-guide}/analyze-metrics.html[Analyze metrics]
+and {observability-guide}/monitor-logs.html[Monitor logs].
 endif::has_solutions[]
 
 {beatname_uc} comes packaged with example Kibana dashboards, visualizations,

libbeat/docs/overview.asciidoc

@@ -28,8 +28,8 @@ To get started, see <<getting-started>>.
 Want to get up and running quickly with infrastructure metrics monitoring and
 centralized log analytics?
 Try out the {metrics-app} and the {logs-app} in {kib}.
-For more details, see the {metrics-guide}[Metrics Monitoring Guide]
-and the {logs-guide}[Logs Monitoring Guide].
+For more details, see {observability-guide}/analyze-metrics.html[Analyze metrics]
+and {observability-guide}/monitor-logs.html[Monitor logs].
 
 [float]
 === Need to capture other kinds of data?

libbeat/docs/shared-faq.asciidoc

@@ -54,6 +54,27 @@ connect to the Lumberjack input plugin. To learn how to install and update
 plugins, see {logstash-ref}/working-with-plugins.html[Working with plugins].
 endif::[]
 
+ifndef::no-output-logstash[]
+[[publishing-ls-fails-connection-reset-by-peer]]
+=== Publishing to {ls} fails with "connection reset by peer" message
+
+{beatname_uc} requires a persistent TCP connection to {ls}. If a firewall interferes
+with the connection, you might see errors like this: 
+
+[source,shell]
+----------------------------------------------------------------------
+Failed to publish events caused by: write tcp ... write: connection reset by peer
+----------------------------------------------------------------------
+
+
+To solve the problem:
+
+* make sure the firewall is not closing connections between {beatname_uc} and {ls}, or
+* set the `ttl` value in the <<logstash-output,{ls} output>> to a value that's
+lower than the maximum time allowed by the firewall, and set `pipelining` to 0
+(pipelining cannot be enabled when `ttl` is used).
+endif::[]
+
 ifndef::no-output-logstash[]
 [[metadata-missing]]
 === @metadata is missing in {ls}