Merge branch 'master' into feature-filebeat-filestream-decrease-log-l…

…evel

.ci/bump-go-release-version.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Given the Golang release version this script will bump the version.
+#
+# This script is executed by the automation we are putting in place
+# and it requires the git add/commit commands.
+#
+# Parameters:
+#	$1 -> the Golang release version to be bumped. Mandatory.
+#
+set -euo pipefail
+MSG="parameter missing."
+GO_RELEASE_VERSION=${1:?$MSG}
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+
+if [ "${OS}" == "darwin" ] ; then
+	SED="sed -i .bck"
+else
+	SED="sed -i"
+fi
+
+echo "Update go version ${GO_RELEASE_VERSION}"
+echo "${GO_RELEASE_VERSION}" > .go-version
+git add .go-version
+
+find . -maxdepth 3 -name Dockerfile -print0 |
+    while IFS= read -r -d '' line; do
+        ${SED} -E -e "s#(FROM golang):[0-9]+\.[0-9]+\.[0-9]+#\1:${GO_RELEASE_VERSION}#g" "$line"
+        ${SED} -E -e "s#(ARG GO_VERSION)=[0-9]+\.[0-9]+\.[0-9]+#\1=${GO_RELEASE_VERSION}#g" "$line"
+        git add "${line}"
+    done
+
+${SED} -E -e "s#(:go-version:) [0-9]+\.[0-9]+\.[0-9]+#\1 ${GO_RELEASE_VERSION}#g" libbeat/docs/version.asciidoc
+git add libbeat/docs/version.asciidoc
+
+git diff --staged --quiet || git commit -m "[Automation] Update go release version to ${GO_RELEASE_VERSION}"
+git --no-pager log -1
+
+echo "You can now push and create a Pull Request"

.ci/packaging.groovy

@@ -212,7 +212,6 @@ pipeline {
                   'x-pack/filebeat',
                   'x-pack/heartbeat',
                   'x-pack/metricbeat',
-                  'x-pack/osquerybeat',
                   'x-pack/packetbeat'
                 )
               }
@@ -426,11 +425,13 @@ def triggerE2ETests(String suite) {
 
   def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
   def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+  def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
 
   def parameters = [
     booleanParam(name: 'forceSkipGitChecks', value: true),
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    booleanParam(name: 'BEAT_VERSION', value: beatVersion),
     booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),

.gitignore

@@ -51,5 +51,5 @@ x-pack/elastic-agent/pkg/agent/operation/tests/scripts/serviceable-1.0-darwin-x8
 *.terraform
 *.tfstate*
 
-# Files generated with the bump elastic stack version automation
-testing/environments/*.bck
+# Files generated with the bump version automations
+*.bck

.mergify.yml

@@ -84,11 +84,25 @@ pull_request_rules:
       merge:
         method: squash
         strict: smart+fasttrack
-  - name: delete upstream branch after merging changes on testing/environments/snapshot*
+  - name: delete upstream branch after merging changes on testing/environments/snapshot* or it's closed
     conditions:
-      - merged
-      - label=automation
-      - head~=^update-stack-version
-      - files~=^testing/environments/snapshot.*\.yml$
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^update-stack-version
+        - files~=^testing/environments/snapshot.*\.yml$
+    actions:
+      delete_head_branch:
+  - name: delete upstream branch after merging changes on .go-version or it's closed
+    conditions:
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^update-go-version
+        - files~=^.go-version$
     actions:
       delete_head_branch:

CHANGELOG.next.asciidoc

@@ -276,6 +276,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix default config template values for paths on oracle module: {pull}26276[26276]
 - Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
 - Fix bug in `httpjson` that prevented `first_event` getting updated. {pull}26407[26407]
+- Fix bug in the Syslog input that misparsed rfc5424 days starting with 0. {pull}26419[26419]
 - Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411] 
 
 *Filebeat*
@@ -820,11 +821,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make `filestream` input GA. {pull}26127[26127]
 - Add new `parser` to `filestream` input: `container`. {pull}26115[26115]
 - Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564]
+- Add possibility to include headers in resulting docs and preserve the original event in http_endpoint input {pull}26279[26279]
 - Add `preserve_original_event` option to `o365audit` input. {pull}26273[26273]
 - Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267]
 - Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
 - RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293]
-
+- Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835]
 
 *Heartbeat*
 
@@ -959,6 +961,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
 - Migrate rds metricsets to use cloudwatch input. {pull}26077[26077]
 - Migrate sqs metricsets to use cloudwatch input. {pull}26117[26117]
+- Collect linked account information in AWS billing. {pull}26285[26285]
 - Add total CPU to vSphere virtual machine metrics. {pull}26167[26167]
 
 *Packetbeat*

Jenkinsfile

@@ -520,6 +520,7 @@ def e2e(Map args = [:]) {
     def goVersionForE2E = readFile('.go-version').trim()
     withEnv(["GO_VERSION=${goVersionForE2E}",
               "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
+              "BEAT_VERSION=${env.VERSION}-SNAPSHOT",
               "LOG_LEVEL=TRACE"]) {
       def status = 0
       filebeat(output: dockerLogFile){

filebeat/docs/modules/elasticsearch.asciidoc

@@ -8,8 +8,6 @@ This file is generated! See scripts/docs_collector.py
 
 == Elasticsearch module
 
-beta[]
-
 This is the elasticsearch module.
 
 include::../include/what-happens.asciidoc[]

filebeat/docs/modules/kibana.asciidoc

@@ -8,8 +8,6 @@ This file is generated! See scripts/docs_collector.py
 
 == Kibana module
 
-beta[]
-
 This is the Kibana module.
 
 include::../include/what-happens.asciidoc[]

filebeat/input/syslog/event.go

@@ -417,7 +417,7 @@ func bytesToInt(b []byte) int {
 
 func skipLeadZero(b []byte) []byte {
 	if len(b) > 1 && b[0] == '0' {
-		return b[1:len(b)]
+		return b[1:]
 	}
 	return b
 }

filebeat/input/syslog/parser/syslog_rfc5424.rl

@@ -18,7 +18,7 @@
   # timestamp
   DATE_FULLYEAR   = digit{4}>tok %year;
   DATE_MONTH      = (("0"[1-9]) | ("1"[0-2]))>tok %month_numeric;
-  DATE_MDAY       = (([12][0-9]) | ("3"[01]))>tok %day;
+  DATE_MDAY       = (("0"[1-9]) | ([12][0-9]) | ("3"[01]))>tok %day;
   FULL_DATE       = DATE_FULLYEAR "-" DATE_MONTH "-" DATE_MDAY;
 
   TIME_HOUR       = ([01][0-9] | "2"[0-3])>tok %hour;