From 81c59d4b8bcacc5723ca9cc1aaba685ba8397d59 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 May 2023 16:49:55 +0200 Subject: [PATCH 1/8] Add missing env. vars --- internal/stack/logs.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/stack/logs.go b/internal/stack/logs.go index 961350bc22..3f07b9eb2c 100644 --- a/internal/stack/logs.go +++ b/internal/stack/logs.go @@ -15,6 +15,11 @@ import ( ) func dockerComposeLogs(serviceName string, snapshotFile string) ([]byte, error) { + appConfig, err := install.Configuration() + if err != nil { + return nil, errors.Wrap(err, "can't read application configuration") + } + p, err := compose.NewProject(DockerComposeProjectName, snapshotFile) if err != nil { return nil, errors.Wrap(err, "could not create docker compose project") @@ -22,6 +27,7 @@ func dockerComposeLogs(serviceName string, snapshotFile string) ([]byte, error) opts := compose.CommandOptions{ Env: newEnvBuilder(). + withEnvs(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv()). withEnv(stackVariantAsEnv(install.DefaultStackVersion)). build(), Services: []string{serviceName}, From e03501dbbeb0147813f0a6f988286c8f26a2eb95 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 May 2023 16:50:16 +0200 Subject: [PATCH 2/8] Remove default folder from internal logs path --- internal/stack/logs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/stack/logs.go b/internal/stack/logs.go index 3f07b9eb2c..834e49f7f6 100644 --- a/internal/stack/logs.go +++ b/internal/stack/logs.go @@ -54,7 +54,7 @@ func copyDockerInternalLogs(serviceName, outputPath string) error { outputPath = filepath.Join(outputPath, serviceName+"-internal") serviceContainer := p.ContainerName(serviceName) - err = docker.Copy(serviceContainer, "/usr/share/elastic-agent/state/data/logs/default", outputPath) + err = docker.Copy(serviceContainer, "/usr/share/elastic-agent/state/data/logs/", outputPath) if err != nil { return errors.Wrap(err, "docker copy failed") } From bdf5a39b4beb97c453e8aadf7bc171e83f73d276 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 May 2023 17:11:15 +0200 Subject: [PATCH 3/8] Update artifact paths --- .buildkite/pipeline.trigger.integration.tests.sh | 6 +++--- .buildkite/scripts/integration_tests.sh | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index fcfb55a783..157bbd30eb 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -25,7 +25,7 @@ for test in ${STACK_COMMAND_TESTS[@]}; do echo " provider: \"gcp\"" echo " artifact_paths:" echo " - build/elastic-stack-dump/stack/*/logs/*.log" - echo " - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*.log" + echo " - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*" echo " - build/elastic-stack-status/*/*" done @@ -43,7 +43,7 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/elastic-stack-dump/stack/check-*/logs/*.log" - echo " - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log" + echo " - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*" echo " - build/elastic-stack-status/*/*" if [[ $test =~ with-kind$ ]]; then echo " - build/kubectl-dump.txt" @@ -63,7 +63,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/elastic-stack-dump/stack/check-*/logs/*.log" - echo " - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/*.log" + echo " - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*" done popd > /dev/null diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index eedb363b13..612db4a164 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -121,9 +121,15 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ - "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*" \ + "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*.*" \ "insecure-logs/${PACKAGE}/" + # required for <8.6.0 + upload_safe_logs \ + "${JOB_GCS_BUCKET_INTERNAL}" \ + "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/default/*" \ + "insecure-logs/${PACKAGE}/default/" + upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ "build/container-logs/*.log" \ From a7c35c788f9a7b30a05a2ca65aaecc242095a9eb Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 May 2023 19:32:03 +0200 Subject: [PATCH 4/8] Add PROFILE_NAME env var --- internal/stack/dump.go | 2 +- internal/stack/logs.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/stack/dump.go b/internal/stack/dump.go index 7a80a27077..c9e0bf6682 100644 --- a/internal/stack/dump.go +++ b/internal/stack/dump.go @@ -57,7 +57,7 @@ func dumpStackLogs(options DumpOptions) error { for _, serviceName := range observedServices { logger.Debugf("Dump stack logs for %s", serviceName) - content, err := dockerComposeLogs(serviceName, snapshotPath) + content, err := dockerComposeLogs(serviceName, snapshotPath, options.Profile) if err != nil { logger.Errorf("can't fetch service logs (service: %s): %v", serviceName, err) } else { diff --git a/internal/stack/logs.go b/internal/stack/logs.go index 834e49f7f6..eb9eb834cd 100644 --- a/internal/stack/logs.go +++ b/internal/stack/logs.go @@ -12,9 +12,10 @@ import ( "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/install" + "github.com/elastic/elastic-package/internal/profile" ) -func dockerComposeLogs(serviceName string, snapshotFile string) ([]byte, error) { +func dockerComposeLogs(serviceName string, snapshotFile string, profile *profile.Profile) ([]byte, error) { appConfig, err := install.Configuration() if err != nil { return nil, errors.Wrap(err, "can't read application configuration") @@ -29,6 +30,7 @@ func dockerComposeLogs(serviceName string, snapshotFile string) ([]byte, error) Env: newEnvBuilder(). withEnvs(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv()). withEnv(stackVariantAsEnv(install.DefaultStackVersion)). + withEnvs(profile.ComposeEnvVars()). build(), Services: []string{serviceName}, } From 1df7808b013eb29a258de5981c8bd5867c95b9e5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 May 2023 19:39:34 +0200 Subject: [PATCH 5/8] Check if profile is nil --- internal/stack/logs.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/stack/logs.go b/internal/stack/logs.go index eb9eb834cd..eb7a99f2ef 100644 --- a/internal/stack/logs.go +++ b/internal/stack/logs.go @@ -26,12 +26,16 @@ func dockerComposeLogs(serviceName string, snapshotFile string, profile *profile return nil, errors.Wrap(err, "could not create docker compose project") } + envBuilder := newEnvBuilder(). + withEnvs(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv()). + withEnv(stackVariantAsEnv(install.DefaultStackVersion)) + + if profile != nil { + envBuilder = envBuilder.withEnvs(profile.ComposeEnvVars()) + } + opts := compose.CommandOptions{ - Env: newEnvBuilder(). - withEnvs(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv()). - withEnv(stackVariantAsEnv(install.DefaultStackVersion)). - withEnvs(profile.ComposeEnvVars()). - build(), + Env: envBuilder.build(), Services: []string{serviceName}, } From 67a1fba9a7f29c0ea2816f2bc3c3b561dea194af Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 May 2023 20:00:27 +0200 Subject: [PATCH 6/8] Get snapshot file from profile inside dockerComposeLogs --- internal/stack/dump.go | 4 +--- internal/stack/logs.go | 18 ++++++++---------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/internal/stack/dump.go b/internal/stack/dump.go index c9e0bf6682..abe088a806 100644 --- a/internal/stack/dump.go +++ b/internal/stack/dump.go @@ -52,12 +52,10 @@ func dumpStackLogs(options DumpOptions) error { return errors.Wrapf(err, "can't create output location (path: %s)", logsPath) } - snapshotPath := options.Profile.Path(profileStackPath, SnapshotFile) - for _, serviceName := range observedServices { logger.Debugf("Dump stack logs for %s", serviceName) - content, err := dockerComposeLogs(serviceName, snapshotPath, options.Profile) + content, err := dockerComposeLogs(serviceName, options.Profile) if err != nil { logger.Errorf("can't fetch service logs (service: %s): %v", serviceName, err) } else { diff --git a/internal/stack/logs.go b/internal/stack/logs.go index eb7a99f2ef..5911b5095a 100644 --- a/internal/stack/logs.go +++ b/internal/stack/logs.go @@ -15,27 +15,25 @@ import ( "github.com/elastic/elastic-package/internal/profile" ) -func dockerComposeLogs(serviceName string, snapshotFile string, profile *profile.Profile) ([]byte, error) { +func dockerComposeLogs(serviceName string, profile *profile.Profile) ([]byte, error) { appConfig, err := install.Configuration() if err != nil { return nil, errors.Wrap(err, "can't read application configuration") } + snapshotFile := options.Profile.Path(profileStackPath, SnapshotFile) + p, err := compose.NewProject(DockerComposeProjectName, snapshotFile) if err != nil { return nil, errors.Wrap(err, "could not create docker compose project") } - envBuilder := newEnvBuilder(). - withEnvs(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv()). - withEnv(stackVariantAsEnv(install.DefaultStackVersion)) - - if profile != nil { - envBuilder = envBuilder.withEnvs(profile.ComposeEnvVars()) - } - opts := compose.CommandOptions{ - Env: envBuilder.build(), + Env: newEnvBuilder(). + withEnvs(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv()). + withEnv(stackVariantAsEnv(install.DefaultStackVersion)). + withEnvs(profile.ComposeEnvVars()). + build(), Services: []string{serviceName}, } From 07532d258303143f2b7adacb3ce7b5b791c25c39 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 May 2023 20:02:12 +0200 Subject: [PATCH 7/8] Fix profile usage --- internal/stack/logs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/stack/logs.go b/internal/stack/logs.go index 5911b5095a..41f10c4561 100644 --- a/internal/stack/logs.go +++ b/internal/stack/logs.go @@ -21,7 +21,7 @@ func dockerComposeLogs(serviceName string, profile *profile.Profile) ([]byte, er return nil, errors.Wrap(err, "can't read application configuration") } - snapshotFile := options.Profile.Path(profileStackPath, SnapshotFile) + snapshotFile := profile.Path(profileStackPath, SnapshotFile) p, err := compose.NewProject(DockerComposeProjectName, snapshotFile) if err != nil { From 76e31e450cea922389cbf15dc8bb57e5fa546647 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 8 May 2023 11:54:34 +0200 Subject: [PATCH 8/8] Update bucket paths --- .buildkite/scripts/integration_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index 612db4a164..0436950e74 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -122,13 +122,13 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*.*" \ - "insecure-logs/${PACKAGE}/" + "insecure-logs/${PACKAGE}/elastic-agent-logs/" # required for <8.6.0 upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/default/*" \ - "insecure-logs/${PACKAGE}/default/" + "insecure-logs/${PACKAGE}/elastic-agent-logs/default/" upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \