Merge remote-tracking branch 'upstream/master' into feature/ci-pipeli…

…ne-2.0

* upstream/master:
  [docs] Promote ingest management to beta (elastic#20295)
  Upgrade elasticsearch client library used in tests (elastic#20405)
  Disable logging when pulling on python integration tests (elastic#20397)
  Remove pillow from testing requirements.txt (elastic#20407)
  [Filebeat][ATP Module]Setting user agent field required by the API (elastic#20440)
  [Ingest Manager] Send datastreams fields (elastic#20402)
  Add event.ingested to all Filebeat modules (elastic#20386)
  [Elastic Agent] Fix agent control socket path to always be less than 107 characters (elastic#20426)
  Improve cgroup_regex docs with examples (elastic#20425)
  Makes `metrics` config option required in app_insights (elastic#20406)
  Ensure install scripts only install if needed (elastic#20349)
  Update container name for the azure filesets (elastic#19899)
  Group same timestamp metrics values in app_insights metricset (elastic#20403)
  add_process_metadata processor adds container id even if process metadata not accessible (elastic#19767)
  Support "cluster" scope in Metricbeat elasticsearch module (elastic#18547)
  [Filebeat][SophosXG Module] Renaming module and fileset (elastic#20396)
  Update Suricata dashboards (elastic#20394)
  [Elastic Agent] Improve version, restart, enroll CLI commands (elastic#20359)
  Prepare home directories for docker images in a different stage (elastic#20356)

.ci/scripts/install-docker-compose.sh

@@ -2,9 +2,23 @@
 
 set -exuo pipefail
 
-MSG="parameter missing."
+MSG="environment variable missing: DOCKER_COMPOSE_VERSION."
 DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION:?$MSG}
 HOME=${HOME:?$MSG}
+
+if command -v docker-compose
+then
+    echo "Found docker-compose. Checking version.."
+    FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//)
+    if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]
+    then
+        echo "Versions match. No need to install docker-compose. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing docker-compose"
+
 DC_CMD="${HOME}/bin/docker-compose"
 
 mkdir -p "${HOME}/bin"

.ci/scripts/install-go.sh

@@ -1,13 +1,25 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-MSG="parameter missing."
+MSG="environment variable missing"
 GO_VERSION=${GO_VERSION:?$MSG}
 PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"}
 HOME=${HOME:?$MSG}
 ARCH=$(uname -s| tr '[:upper:]' '[:lower:]')
 GVM_CMD="${HOME}/bin/gvm"
 
+if command -v go
+then
+    echo "Found Go. Checking version.."
+    FOUND_GO_VERSION=$(go version|awk '{print $3}'|sed s/go//)
+    if [ $FOUND_GO_VERSION == $GO_VERSION ]
+    then
+        echo "Versions match. No need to install Go. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing Go"
 mkdir -p "${HOME}/bin"
 
 curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64"

.ci/scripts/install-kind.sh

@@ -1,12 +1,25 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-MSG="parameter missing."
+MSG="environment variable missing."
 DEFAULT_HOME="/usr/local"
 KIND_VERSION=${KIND_VERSION:?$MSG}
 HOME=${HOME:?$DEFAULT_HOME}
 KIND_CMD="${HOME}/bin/kind"
 
+if command -v kind
+then
+    echo "Found Kind. Checking version.."
+    FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
+    if [ $FOUND_KIND_VERSION == $KIND_VERSION ]
+    then
+        echo "Versions match. No need to install Kind. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing Kind"
+
 mkdir -p "${HOME}/bin"
 
 curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"

.ci/scripts/install-terraform.sh

@@ -2,13 +2,26 @@
 
 set -exuo pipefail
 
-MSG="parameter missing."
+MSG="environment variable missing."
 TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG}
 HOME=${HOME:?$MSG}
 TERRAFORM_CMD="${HOME}/bin/terraform"
 
 OS=$(uname -s | tr '[:upper:]' '[:lower:]')
 
+if command -v terraform
+then
+    echo "Found Terraform. Checking version.."
+    FOUND_TERRAFORM_VERSION=$(terraform --version | awk '{print $2}' | sed s/v//)
+    if [ $FOUND_TERRAFORM_VERSION == $TERRAFORM_VERSION ]
+    then
+        echo "Versions match. No need to install Terraform. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing Terraform"
+
 mkdir -p "${HOME}/bin"
 
 curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip

CHANGELOG.next.asciidoc

@@ -222,6 +222,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix memory leak in tcp and unix input sources. {pull}19459[19459]
 - Add missing `default_field: false` to aws filesets fields.yml. {pull}19568[19568]
 - Fix tls mapping in suricata module {issue}19492[19492] {pull}19494[19494]
+- Update container name for the azure filesets. {pull}19899[19899]
 - Fix bug with empty filter values in system/service {pull}19812[19812]
 - Fix S3 input to trim delimiter /n from each log line. {pull}19972[19972]
 - Ignore missing in Zeek module when dropping unecessary fields. {pull}19984[19984]
@@ -302,6 +303,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix k8s scheduler compatibility issue. {pull}19699[19699]
 - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898
 - Modify doc for app_insights metricset to contain example of config. {pull}20185[20185]
+- Add required option for `metrics` in app_insights. {pull}20406[20406]
+- Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403]
 
 *Packetbeat*
 
@@ -364,6 +367,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562]
 - Add support for DNS over TLS for the dns_processor. {pull}19321[19321]
 - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
+- Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767]
 
 *Auditbeat*
 
@@ -501,6 +505,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add event.ingested for CrowdStrike module {pull}20138[20138]
 - Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138]
 - Add event.ingested for Suricata module {pull}20220[20220]
+- Add event.ingested to all Filebeat modules. {pull}20386[20386]
 
 *Heartbeat*
 
@@ -610,6 +615,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added cache and connection_errors metrics to status metricset of MySQL module {issue}16955[16955] {pull}19844[19844]
 - Update MySQL dashboard with connection errors and cache metrics {pull}19913[19913] {issue}16955[16955]
 - Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077]
+- Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547]
 
 *Packetbeat*
 

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -2,6 +2,25 @@
 {{- $beatBinary := printf "%s/%s" $beatHome .BeatName }}
 {{- $repoInfo := repo }}
 
+# Prepare home in a different stage to avoid creating additional layers on
+# the final image because of permission changes.
+FROM {{ .from }} AS home
+
+COPY beat {{ $beatHome }}
+
+RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \
+    chown -R root:root {{ $beatHome }} && \
+    find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
+    find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
+    chmod 0750 {{ $beatBinary }} && \
+{{- if .linux_capabilities }}
+    setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
+{{- end }}
+{{- range $i, $modulesd := .ModulesDirs }}
+    chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
+{{- end }}
+    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs
+
 FROM {{ .from }}
 
 RUN yum -y --setopt=tsflags=nodocs update && \
@@ -23,26 +42,13 @@ LABEL \
 ENV ELASTIC_CONTAINER "true"
 ENV PATH={{ $beatHome }}:$PATH
 
-COPY beat {{ $beatHome }}
 COPY docker-entrypoint /usr/local/bin/docker-entrypoint
 RUN chmod 755 /usr/local/bin/docker-entrypoint
 
-RUN groupadd --gid 1000 {{ .BeatName }}
-
-RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \
-    chown -R root:root {{ $beatHome }} && \
-    find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
-    find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
-    chmod 0750 {{ $beatBinary }} && \
-{{- if .linux_capabilities }}
-    setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
-{{- end }}
-{{- range $i, $modulesd := .ModulesDirs }}
-    chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
-{{- end }}
-    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs
+COPY --from=home {{ $beatHome }} {{ $beatHome }}
 
 {{- if ne .user "root" }}
+RUN groupadd --gid 1000 {{ .BeatName }}
 RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }}
 {{- end }}
 USER {{ .user }}