From 8df2f0dd5b12f338c3d45532341d30f8dc76f4bd Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Tue, 12 Dec 2023 11:22:03 -0500 Subject: [PATCH 1/4] add config to enable apm-server --- internal/profile/_static/config.yml.example | 4 ++++ internal/profile/_testdata/config.yml | 1 + internal/profile/config_test.go | 5 +++++ .../_static/docker-compose-stack.yml.tmpl | 4 ++++ internal/stack/_static/kibana.yml.tmpl | 18 ++++++++++++++++++ internal/stack/resources.go | 5 +++-- 6 files changed, 35 insertions(+), 2 deletions(-) diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index 76a28a71bf..232318d548 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -10,6 +10,10 @@ # Region where the Serverless project is going to be created # stack.serverless.region: aws-us-east-1 +## Enable apm-server +# Flag to enable apm-server in elastic-package stack profile config +# stack.apm_server_enabled: true + ## Enable logstash for testing # Flag to enable logstash in elastic-package stack profile config # stack.logstash_enabled: true diff --git a/internal/profile/_testdata/config.yml b/internal/profile/_testdata/config.yml index 0eb5991f74..c5b366a10e 100644 --- a/internal/profile/_testdata/config.yml +++ b/internal/profile/_testdata/config.yml @@ -1,5 +1,6 @@ # An expected setting. stack.geoip_dir: "/home/foo/Documents/ingest-geoip" +stack.apm_server_enabled: true stack.logstash_enabled: true # An empty string, should exist, but return empty. diff --git a/internal/profile/config_test.go b/internal/profile/config_test.go index 5f7644d3d1..ed41bdcb62 100644 --- a/internal/profile/config_test.go +++ b/internal/profile/config_test.go @@ -47,6 +47,11 @@ func TestLoadProfileConfig(t *testing.T) { expected: "false", found: true, }, + { + name: "stack.apm_server_enabled", + expected: "true", + found: true, + }, { name: "stack.logstash_enabled", expected: "true", diff --git a/internal/stack/_static/docker-compose-stack.yml.tmpl b/internal/stack/_static/docker-compose-stack.yml.tmpl index 7b4e72c10e..5a6486dae2 100644 --- a/internal/stack/_static/docker-compose-stack.yml.tmpl +++ b/internal/stack/_static/docker-compose-stack.yml.tmpl @@ -112,6 +112,10 @@ services: - "../certs/fleet-server:/etc/ssl/elastic-agent" ports: - "127.0.0.1:8220:8220" + {{ $apm_server_enabled := fact "apm_server_enabled" }} + {{ if eq $apm_server_enabled "true" }} + - "127.0.0.1:8200:8200" + {{ end }} fleet-server_is_ready: image: tianon/true diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index 739711eea8..9415a32118 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -48,6 +48,11 @@ xpack.fleet.packages: version: latest - name: fleet_server version: latest + {{ $apm_server_enabled := fact "apm_server_enabled" }} + {{ if eq $apm_server_enabled "true" }} + - name: apm + version: latest + {{ end }} xpack.fleet.agentPolicies: - name: Elastic-Agent (elastic-package) id: elastic-agent-managed-ep @@ -72,6 +77,19 @@ xpack.fleet.agentPolicies: id: default-fleet-server package: name: fleet_server + {{ $apm_server_enabled := fact "apm_server_enabled" }} + {{ if eq $apm_server_enabled "true" }} + - name: apm-1 + package: + name: apm + inputs: + - type: apm + vars: + - name: host + value: "0.0.0.0:8200" + - name: secret_token + value: "" + {{ end }} xpack.fleet.outputs: - id: fleet-default-output name: default diff --git a/internal/stack/resources.go b/internal/stack/resources.go index 75a4db4546..c34411b6a5 100644 --- a/internal/stack/resources.go +++ b/internal/stack/resources.go @@ -129,8 +129,9 @@ func applyResources(profile *profile.Profile, stackVersion string) error { "username": elasticsearchUsername, "password": elasticsearchPassword, - "geoip_dir": profile.Config("stack.geoip_dir", "./ingest-geoip"), - "logstash_enabled": profile.Config("stack.logstash_enabled", "false"), + "geoip_dir": profile.Config("stack.geoip_dir", "./ingest-geoip"), + "apm_server_enabled": profile.Config("stack.apm_server_enabled", "false"), + "logstash_enabled": profile.Config("stack.logstash_enabled", "false"), }) os.MkdirAll(stackDir, 0755) From 1180fdda46f0aa3bd51e2c5f9746f882bbf63f5f Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Tue, 12 Dec 2023 17:33:16 -0500 Subject: [PATCH 2/4] add integration test --- Makefile | 5 ++++- scripts/test-stack-command.sh | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8ef78ae89c..7e55bce5c2 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,10 @@ test-stack-command-86: test-stack-command-8x: ./scripts/test-stack-command.sh 8.11.0-SNAPSHOT -test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x +test-stack-command-with-apm-server: + APM_SERVER_ENABLED=true ./scripts/test-stack-command.sh + +test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x test-stack-command-with-apm-server test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash diff --git a/scripts/test-stack-command.sh b/scripts/test-stack-command.sh index 132593ae33..4938f71225 100755 --- a/scripts/test-stack-command.sh +++ b/scripts/test-stack-command.sh @@ -3,6 +3,7 @@ set -euxo pipefail VERSION=${1:-default} +APM_SERVER_ENABLED=${APM_SERVER_ENABLED:-false} cleanup() { r=$? @@ -13,6 +14,11 @@ cleanup() { # Take down the stack elastic-package stack down -v + if [ "${APM_SERVER_ENABLED}" = true ]; then + # Create an apm-server profile and use it + elastic-package profiles delete with-apm-server + fi + exit $r } @@ -34,7 +40,22 @@ if [ "${VERSION}" != "default" ]; then EXPECTED_VERSION=${VERSION} fi +if [ "${APM_SERVER_ENABLED}" = true ]; then + # Create an apm-server profile and use it + profile=with-apm-server + elastic-package profiles create -v ${profile} + elastic-package profiles use ${profile} + + # Create the config and enable apm-server + cat ~/.elastic-package/profiles/${profile}/config.yml.example - < ~/.elastic-package/profiles/${profile}/config.yml +stack.apm_server_enabled: true +EOF +fi + OUTPUT_PATH_STATUS="build/elastic-stack-status/${VERSION}" +if [ "${APM_SERVER_ENABLED}" = true ]; then + OUTPUT_PATH_STATUS="build/elastic-stack-status/${VERSION}_with_apm_server" +fi mkdir -p ${OUTPUT_PATH_STATUS} # Initial status empty @@ -71,4 +92,8 @@ elastic-package stack status -v 2> ${OUTPUT_PATH_STATUS}/running.txt clean_status_output "${OUTPUT_PATH_STATUS}/expected_running.txt" > ${OUTPUT_PATH_STATUS}/expected_no_spaces.txt clean_status_output "${OUTPUT_PATH_STATUS}/running.txt" > ${OUTPUT_PATH_STATUS}/running_no_spaces.txt +if [ "${APM_SERVER_ENABLED}" = true ]; then + curl --fail-with-body http://localhost:8200/ +fi + diff -q ${OUTPUT_PATH_STATUS}/running_no_spaces.txt ${OUTPUT_PATH_STATUS}/expected_no_spaces.txt From bc284dfb08d93a90984faeba7ce18b1076b6b46e Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Tue, 12 Dec 2023 17:35:40 -0500 Subject: [PATCH 3/4] trigger integration test from CI --- .buildkite/pipeline.trigger.integration.tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 23e76440c4..5a27da9986 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -16,6 +16,7 @@ STACK_COMMAND_TESTS=( test-stack-command-7x test-stack-command-86 test-stack-command-8x + test-stack-command-with-apm-server ) for test in ${STACK_COMMAND_TESTS[@]}; do From 47d9a8c1c6092fa05977ad1ae86eaf1a70886a71 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Tue, 12 Dec 2023 18:00:39 -0500 Subject: [PATCH 4/4] accommodate old curl --- scripts/test-stack-command.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-stack-command.sh b/scripts/test-stack-command.sh index 4938f71225..6fcf86f684 100755 --- a/scripts/test-stack-command.sh +++ b/scripts/test-stack-command.sh @@ -93,7 +93,7 @@ clean_status_output "${OUTPUT_PATH_STATUS}/expected_running.txt" > ${OUTPUT_PATH clean_status_output "${OUTPUT_PATH_STATUS}/running.txt" > ${OUTPUT_PATH_STATUS}/running_no_spaces.txt if [ "${APM_SERVER_ENABLED}" = true ]; then - curl --fail-with-body http://localhost:8200/ + curl http://localhost:8200/ fi diff -q ${OUTPUT_PATH_STATUS}/running_no_spaces.txt ${OUTPUT_PATH_STATUS}/expected_no_spaces.txt