Merge remote-tracking branch 'upstream/master' into fix/ci-tools-inst…

…allation * upstream/master: (23 commits) [docs] Promote ingest management to beta (elastic#20295) Upgrade elasticsearch client library used in tests (elastic#20405) Disable logging when pulling on python integration tests (elastic#20397) Remove pillow from testing requirements.txt (elastic#20407) [Filebeat][ATP Module]Setting user agent field required by the API (elastic#20440) [Ingest Manager] Send datastreams fields (elastic#20402) Add event.ingested to all Filebeat modules (elastic#20386) [Elastic Agent] Fix agent control socket path to always be less than 107 characters (elastic#20426) Improve cgroup_regex docs with examples (elastic#20425) Makes `metrics` config option required in app_insights (elastic#20406) Ensure install scripts only install if needed (elastic#20349) Update container name for the azure filesets (elastic#19899) Group same timestamp metrics values in app_insights metricset (elastic#20403) add_process_metadata processor adds container id even if process metadata not accessible (elastic#19767) Support "cluster" scope in Metricbeat elasticsearch module (elastic#18547) [Filebeat][SophosXG Module] Renaming module and fileset (elastic#20396) Update Suricata dashboards (elastic#20394) [Elastic Agent] Improve version, restart, enroll CLI commands (elastic#20359) Prepare home directories for docker images in a different stage (elastic#20356) New multiline mode in Filebeat: while_pattern (elastic#19662) ...
v1v · Aug 6, 2020 · 6b99d98 · 6b99d98
2 parents df3ffa3 + 44fc8e3
commit 6b99d98
Show file tree

Hide file tree

Showing 251 changed files with 5,406 additions and 4,358 deletions.
diff --git a/.ci/scripts/install-docker-compose.sh b/.ci/scripts/install-docker-compose.sh
@@ -2,9 +2,23 @@
 
 set -exuo pipefail
 
-MSG="parameter missing."
+MSG="environment variable missing: DOCKER_COMPOSE_VERSION."
 DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION:?$MSG}
 HOME=${HOME:?$MSG}
+
+if command -v docker-compose
+then
+    echo "Found docker-compose. Checking version.."
+    FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//)
+    if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]
+    then
+        echo "Versions match. No need to install docker-compose. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing docker-compose"
+
 DC_CMD="${HOME}/bin/docker-compose"
 
 mkdir -p "${HOME}/bin"

diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh
@@ -1,13 +1,25 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-MSG="parameter missing."
+MSG="environment variable missing"
 GO_VERSION=${GO_VERSION:?$MSG}
 PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"}
 HOME=${HOME:?$MSG}
 ARCH=$(uname -s| tr '[:upper:]' '[:lower:]')
 GVM_CMD="${HOME}/bin/gvm"
 
+if command -v go
+then
+    echo "Found Go. Checking version.."
+    FOUND_GO_VERSION=$(go version|awk '{print $3}'|sed s/go//)
+    if [ $FOUND_GO_VERSION == $GO_VERSION ]
+    then
+        echo "Versions match. No need to install Go. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing Go"
 mkdir -p "${HOME}/bin"
 
 curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64"

diff --git a/.ci/scripts/install-kind.sh b/.ci/scripts/install-kind.sh
@@ -1,12 +1,25 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-MSG="parameter missing."
+MSG="environment variable missing."
 DEFAULT_HOME="/usr/local"
 KIND_VERSION=${KIND_VERSION:?$MSG}
 HOME=${HOME:?$DEFAULT_HOME}
 KIND_CMD="${HOME}/bin/kind"
 
+if command -v kind
+then
+    echo "Found Kind. Checking version.."
+    FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
+    if [ $FOUND_KIND_VERSION == $KIND_VERSION ]
+    then
+        echo "Versions match. No need to install Kind. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing Kind"
+
 mkdir -p "${HOME}/bin"
 
 curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"

diff --git a/.ci/scripts/install-terraform.sh b/.ci/scripts/install-terraform.sh
@@ -2,13 +2,26 @@
 
 set -exuo pipefail
 
-MSG="parameter missing."
+MSG="environment variable missing."
 TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG}
 HOME=${HOME:?$MSG}
 TERRAFORM_CMD="${HOME}/bin/terraform"
 
 OS=$(uname -s | tr '[:upper:]' '[:lower:]')
 
+if command -v terraform
+then
+    echo "Found Terraform. Checking version.."
+    FOUND_TERRAFORM_VERSION=$(terraform --version | awk '{print $2}' | sed s/v//)
+    if [ $FOUND_TERRAFORM_VERSION == $TERRAFORM_VERSION ]
+    then
+        echo "Versions match. No need to install Terraform. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing Terraform"
+
 mkdir -p "${HOME}/bin"
 
 curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -1028,10 +1028,11 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit
 - Add Kibana Dashboard for MISP module. {pull}14147[14147]
 - Add support for gzipped files in S3 input {pull}13980[13980]
 - Add Filebeat Azure Dashboards {pull}14127[14127]
+- Add support for space or time sync character before timestamp in syslog input. {pull}13278[13278] {issue}13269[13269]
 - Add support for thread ID in Filebeat Kafka module. {pull}19463[19463]
 
-
 *Heartbeat*
+
 - Add non-privileged icmp on linux and darwin(mac). {pull}13795[13795] {issue}11498[11498]
 - Allow `hosts` to be used to configure http monitors {pull}13703[13703]
 

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -62,6 +62,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Adds Gsuite Drive support. {pull}19704[19704]
 - Adds Gsuite Groups support. {pull}19725[19725]
 - Move file metrics to dataset endpoint {pull}19977[19977]
+- Add `while_pattern` type to multiline reader. {pull}19662[19662]
 
 *Heartbeat*
 
@@ -221,6 +222,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix memory leak in tcp and unix input sources. {pull}19459[19459]
 - Add missing `default_field: false` to aws filesets fields.yml. {pull}19568[19568]
 - Fix tls mapping in suricata module {issue}19492[19492] {pull}19494[19494]
+- Update container name for the azure filesets. {pull}19899[19899]
 - Fix bug with empty filter values in system/service {pull}19812[19812]
 - Fix S3 input to trim delimiter /n from each log line. {pull}19972[19972]
 - Ignore missing in Zeek module when dropping unecessary fields. {pull}19984[19984]
@@ -301,6 +303,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix k8s scheduler compatibility issue. {pull}19699[19699]
 - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898
 - Modify doc for app_insights metricset to contain example of config. {pull}20185[20185]
+- Add required option for `metrics` in app_insights. {pull}20406[20406]
+- Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403]
 
 *Packetbeat*
 
@@ -363,6 +367,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562]
 - Add support for DNS over TLS for the dns_processor. {pull}19321[19321]
 - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
+- Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767]
 
 *Auditbeat*
 
@@ -500,6 +505,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add event.ingested for CrowdStrike module {pull}20138[20138]
 - Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138]
 - Add event.ingested for Suricata module {pull}20220[20220]
+- Add event.ingested to all Filebeat modules. {pull}20386[20386]
 
 *Heartbeat*
 
@@ -609,6 +615,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added cache and connection_errors metrics to status metricset of MySQL module {issue}16955[16955] {pull}19844[19844]
 - Update MySQL dashboard with connection errors and cache metrics {pull}19913[19913] {issue}16955[16955]
 - Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077]
+- Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547]
 
 *Packetbeat*