Skip to content

Commit

Permalink
Merge branch 'main' into fix-ilm-docs
Browse files Browse the repository at this point in the history
  • Loading branch information
belimawr committed Feb 12, 2024
2 parents a0d232a + e5000b6 commit b6b3e37
Show file tree
Hide file tree
Showing 26 changed files with 2,174 additions and 76 deletions.
3 changes: 1 addition & 2 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
fi
fi


if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" ]]; then
source .buildkite/scripts/setenv.sh
if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
echo "Skipped pre-command when running the Upload pipeline"
Expand Down
4 changes: 0 additions & 4 deletions .buildkite/libbeat/pipeline.libbeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,3 @@ steps:
- label: ":linux: Load dynamic Libbeat pipeline"
key: "libbeat-pipeline"
command: ".buildkite/scripts/generate_libbeat_pipeline.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
5 changes: 0 additions & 5 deletions .buildkite/metricbeat/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ env:
IMAGE_WIN_2019: "family/core-windows-2019"
IMAGE_WIN_2022: "family/core-windows-2022"
IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
GO_AGENT_IMAGE: "golang:${GO_VERSION}"
BEATS_PROJECT_NAME: "metricbeat"
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
Expand Down Expand Up @@ -46,7 +45,3 @@ steps:
- label: ":linux: Load dynamic metricbeat pipeline"
key: "metricbeat-pipeline"
command: ".buildkite/scripts/generate_metricbeat_pipeline.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
3 changes: 0 additions & 3 deletions .buildkite/packetbeat/pipeline.packetbeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,3 @@ steps:
- label: ":linux: Load dynamic packetbeat pipeline"
key: "packetbeat-pipeline"
command: ".buildkite/scripts/generate_packetbeat_pipeline.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
46 changes: 25 additions & 21 deletions .buildkite/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@ platform_type="$(uname)"
platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]')
arch_type="$(uname -m)"
GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""}
ONLY_DOCS=${ONLY_DOCS:-"true"}

[ -z "${runLibbeat+x}" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
[ -z "${runMetricbeat+x}" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})"
[ -z "${runPacketbeat+x}" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})"
[ -z "${runWinlogbeat+x}" ] && runWinlogbeat="$(buildkite-agent meta-data get runWinlogbeat --default ${runWinlogbeat:-"false"})"
[ -z "${runLibBeatArmTest+x}" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
[ -z "${runPacketbeatArmTest+x}" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})"
[ -z "${runMetricbeatMacOsTests+x}" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})"
Expand All @@ -29,6 +30,10 @@ packetbeat_changeset=(
"^packetbeat/.*"
)

winlogbeat_changeset=(
"^winlogbeat/.*"
)

oss_changeset=(
"^go.mod"
"^pytest.ini"
Expand Down Expand Up @@ -231,18 +236,23 @@ are_changed_only_paths() {

are_conditions_met_mandatory_tests() {
if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then
return 0
fi
return 0
fi
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" ]]; then
if are_paths_changed "${winlogbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test winlogbeat" || "${GITHUB_PR_LABELS}" =~ Winlogbeat || "${runWinlogbeat}" == "true" ]]; then
return 0
fi
fi
return 1
Expand Down Expand Up @@ -280,14 +290,8 @@ are_conditions_met_macos_tests() {

are_conditions_met_packaging() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then
return 0
fi
if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then
return 0
fi
fi
return 1
Expand Down
109 changes: 109 additions & 0 deletions .buildkite/scripts/generate_winlogbeat_pipeline.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
#!/usr/bin/env bash

source .buildkite/scripts/common.sh

set -euo pipefail

pipelineName="pipeline.winlogbeat-dynamic.yml"

echo "Add the mandatory and extended tests without additional conditions into the pipeline"
if are_conditions_met_mandatory_tests; then
cat > $pipelineName <<- YAML
steps:
- group: "Mandatory Tests"
key: "mandatory-tests"
steps:
- label: ":negative_squared_cross_mark: Cross compile"
key: "mandatory-cross-compile"
command: ".buildkite/scripts/crosscompile.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
- label: ":windows: Windows 2016/2019/2022 Unit Tests - {{matrix.image}}"
command: ".buildkite/scripts/win_unit_tests.ps1"
key: "mandatory-win-unit-tests"
agents:
provider: "gcp"
image: "{{matrix.image}}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
matrix:
setup:
image:
- "${IMAGE_WIN_2016}"
- "${IMAGE_WIN_2019}"
- "${IMAGE_WIN_2022}"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
# echo "Add the extended windows tests into the pipeline"
# TODO: ADD conditions from the main pipeline
- group: "Extended Windows Tests"
key: "extended-win-tests"
steps:
- label: ":windows: Windows 10 Unit Tests"
key: "extended-win-10-unit-tests"
command: ".buildkite/scripts/win_unit_tests.ps1"
agents:
provider: "gcp"
image: "${IMAGE_WIN_10}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
- label: ":windows: Windows 11 Unit Tests"
key: "extended-win-11-unit-tests"
command: ".buildkite/scripts/win_unit_tests.ps1"
agents:
provider: "gcp"
image: "${IMAGE_WIN_11}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
YAML
else
echo "The conditions don't match to requirements for generating pipeline steps."
exit 0
fi

echo "Check and add the Packaging into the pipeline"
if are_conditions_met_packaging; then
cat >> $pipelineName <<- YAML
- wait: ~
depends_on:
- step: "mandatory-tests"
allow_failure: false
- group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589
key: "packaging"
steps:
- label: ":linux: Packaging Linux"
key: "packaging-linux"
command: ".buildkite/scripts/packaging.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
env:
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
YAML
fi

echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public
cat $pipelineName

echo "--- Loading dynamic steps"
buildkite-agent pipeline upload $pipelineName
9 changes: 6 additions & 3 deletions .buildkite/scripts/win_unit_tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@ function withChoco {
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
}
function withGolang($version) {
Write-Host "-- Install golang $version --"
choco install -y golang --version=$version
refreshenv
$downloadPath = Join-Path $env:TEMP "go_installer.msi"
$goInstallerUrl = "https://golang.org/dl/go$version.windows-amd64.msi"
Invoke-WebRequest -Uri $goInstallerUrl -OutFile $downloadPath
Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $downloadPath /quiet" -Wait
$goBinPath = "${env:ProgramFiles}\Go\bin"
$env:Path += ";$goBinPath"
go version
}
function withPython($version) {
Expand Down
34 changes: 32 additions & 2 deletions .buildkite/winlogbeat/pipeline.winlogbeat.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,35 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
env:
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
IMAGE_WIN_10: "family/general-windows-10"
IMAGE_WIN_11: "family/general-windows-11"
IMAGE_WIN_2016: "family/core-windows-2016"
IMAGE_WIN_2019: "family/core-windows-2019"
IMAGE_WIN_2022: "family/core-windows-2022"
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
GCP_WIN_MACHINE_TYPE: "n2-standard-8"
BEATS_PROJECT_NAME: "winlogbeat"

steps:
- label: "Example test"
command: echo "Hello!"

- input: "Input Parameters"
key: "input-run-all-stages"
fields:
- select: "Packetbeat - runWinlogbeat"
key: "runWinlogbeat"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
if: "build.source == 'ui'"

- wait: ~
if: "build.source == 'ui'"
allow_dependency_failure: false

- label: ":linux: Load dynamic winlogbeat pipeline"
key: "winlogbeat-pipeline"
command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh"
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ CHANGELOG*
/x-pack/filebeat/input/lumberjack/ @elastic/security-service-integrations
/x-pack/filebeat/input/netflow/ @elastic/sec-deployment-and-devices
/x-pack/filebeat/input/o365audit/ @elastic/security-service-integrations
/x-pack/filebeat/input/websocket/ @elastic/security-service-integrations
/x-pack/filebeat/module/activemq @elastic/obs-infraobs-integrations
/x-pack/filebeat/module/aws @elastic/obs-cloud-monitoring
/x-pack/filebeat/module/awsfargate @elastic/obs-cloud-monitoring
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d
- Relax TCP/UDP metric polling expectations to improve metric collection. {pull}37714[37714]
- Add support for PEM-based Okta auth in HTTPJSON. {pull}37772[37772]
- Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836]
- Added experimental version of the Websocket Input. {pull}37774[37774]
- Add support for PEM-based Okta auth in CEL. {pull}37813[37813]

*Auditbeat*
Expand Down
64 changes: 32 additions & 32 deletions NOTICE.txt
Original file line number Diff line number Diff line change
Expand Up @@ -18447,6 +18447,38 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


--------------------------------------------------------------------------------
Dependency : github.com/gorilla/websocket
Version: v1.4.2
Licence type (autodetected): BSD-2-Clause
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/gorilla/websocket@v1.4.2/LICENSE:

Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.

Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


--------------------------------------------------------------------------------
Dependency : github.com/h2non/filetype
Version: v1.1.1
Expand Down Expand Up @@ -41594,38 +41626,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


--------------------------------------------------------------------------------
Dependency : github.com/gorilla/websocket
Version: v1.4.2
Licence type (autodetected): BSD-2-Clause
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/gorilla/websocket@v1.4.2/LICENSE:

Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.

Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


--------------------------------------------------------------------------------
Dependency : github.com/hashicorp/cronexpr
Version: v1.1.0
Expand Down
3 changes: 3 additions & 0 deletions filebeat/docs/filebeat-options.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ You can configure {beatname_uc} to use the following inputs:
* <<{beatname_lc}-input-tcp>>
* <<{beatname_lc}-input-udp>>
* <<{beatname_lc}-input-gcs>>
* <<{beatname_lc}-input-websocket>>

include::multiline.asciidoc[]

Expand Down Expand Up @@ -145,3 +146,5 @@ include::inputs/input-udp.asciidoc[]
include::inputs/input-unix.asciidoc[]

include::../../x-pack/filebeat/docs/inputs/input-gcs.asciidoc[]

include::../../x-pack/filebeat/docs/inputs/input-websocket.asciidoc[]
Loading

0 comments on commit b6b3e37

Please sign in to comment.