From a63751681487e19d6a9292ff29ae12c88edb804f Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 14 Feb 2022 10:17:12 +0100 Subject: [PATCH 1/6] Use FLEET_TOKEN_POLICY_NAME --- internal/profile/_static/docker-compose-stack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/profile/_static/docker-compose-stack.yml index 313e4098db..7bced74388 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/profile/_static/docker-compose-stack.yml @@ -107,6 +107,7 @@ services: - "FLEET_ENROLL=1" - "FLEET_INSECURE=1" - "FLEET_URL=http://fleet-server:8220" + - "FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package)" volumes: - type: bind source: ../../../tmp/service_logs/ From 2f252585eb8fdfb8b2b9fc4a5880e74f84f1c995 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 14 Feb 2022 11:03:44 +0100 Subject: [PATCH 2/6] Use separate env file for Elastic Agent --- .../profile/_static/docker-compose-stack.yml | 6 +---- internal/profile/_static/elastic-agent_8x.env | 4 +++ .../profile/_static/elastic-agent_default.env | 3 +++ internal/profile/profile.go | 2 ++ internal/profile/static.go | 27 +++++++++++++++++++ 5 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 internal/profile/_static/elastic-agent_8x.env create mode 100644 internal/profile/_static/elastic-agent_default.env diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/profile/_static/docker-compose-stack.yml index 7bced74388..9d3c114acb 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/profile/_static/docker-compose-stack.yml @@ -103,11 +103,7 @@ services: retries: 180 interval: 1s hostname: docker-fleet-agent - environment: - - "FLEET_ENROLL=1" - - "FLEET_INSECURE=1" - - "FLEET_URL=http://fleet-server:8220" - - "FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package)" + env_file: "./elastic-agent_${STACK_VERSION_VARIANT}.env" volumes: - type: bind source: ../../../tmp/service_logs/ diff --git a/internal/profile/_static/elastic-agent_8x.env b/internal/profile/_static/elastic-agent_8x.env new file mode 100644 index 0000000000..2d35422b79 --- /dev/null +++ b/internal/profile/_static/elastic-agent_8x.env @@ -0,0 +1,4 @@ +FLEET_ENROLL=1 +FLEET_INSECURE=1 +FLEET_URL=http://fleet-server:8220 +FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package) \ No newline at end of file diff --git a/internal/profile/_static/elastic-agent_default.env b/internal/profile/_static/elastic-agent_default.env new file mode 100644 index 0000000000..bb341c3952 --- /dev/null +++ b/internal/profile/_static/elastic-agent_default.env @@ -0,0 +1,3 @@ +FLEET_ENROLL=1 +FLEET_INSECURE=1 +FLEET_URL=http://fleet-server:8220 \ No newline at end of file diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 15298fea42..1c5262ff7e 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -36,6 +36,8 @@ type configFile string // managedProfileFiles is the list of all files managed in a profile // If you create a new file that's managed by a profile, it needs to go in this list var managedProfileFiles = map[configFile]NewConfig{ + ElasticAgentDefaultEnvFile: newElasticAgentDefaultEnv, + ElasticAgent8xEnvFile: newElasticAgent8xEnv, ElasticsearchConfigDefaultFile: newElasticsearchConfigDefault, ElasticsearchConfig8xFile: newElasticsearchConfig8x, KibanaConfigDefaultFile: newKibanaConfigDefault, diff --git a/internal/profile/static.go b/internal/profile/static.go index 48ca985738..3bd2a73761 100644 --- a/internal/profile/static.go +++ b/internal/profile/static.go @@ -115,5 +115,32 @@ func newPackageRegistryDockerfile(_ string, profilePath string) (*simpleFile, er path: filepath.Join(profilePath, profileStackPath, string(PackageRegistryDockerfileFile)), body: packageRegistryDockerfile, }, nil +} + +// ElasticAgent8xEnvFile is the .env for the 8x stack. +const ElasticAgent8xEnvFile configFile = "elastic-agent_8x.env" + +//go:embed _static/elastic-agent_8x.env +var elasticAgent8xEnv string +func newElasticAgent8xEnv(_ string, profilePath string) (*simpleFile, error) { + return &simpleFile{ + name: string(ElasticAgent8xEnvFile), + path: filepath.Join(profilePath, profileStackPath, string(ElasticAgent8xEnvFile)), + body: elasticAgent8xEnv, + }, nil +} + +// ElasticAgentDefaultEnvFile is the default .env file. +const ElasticAgentDefaultEnvFile configFile = "elastic-agent_default.env" + +//go:embed _static/elastic-agent_default.env +var elasticAgentDefaultEnv string + +func newElasticAgentDefaultEnv(_ string, profilePath string) (*simpleFile, error) { + return &simpleFile{ + name: string(ElasticAgentDefaultEnvFile), + path: filepath.Join(profilePath, profileStackPath, string(ElasticAgentDefaultEnvFile)), + body: elasticAgentDefaultEnv, + }, nil } From cf22431fc589120c6ef8e552eddda340ff5a2e63 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 14 Feb 2022 11:32:42 +0100 Subject: [PATCH 3/6] Fix: missing envs --- internal/profile/static.go | 4 ++-- internal/stack/logs.go | 4 ++++ internal/stack/shellinit.go | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/profile/static.go b/internal/profile/static.go index 3bd2a73761..ff2ed10d6f 100644 --- a/internal/profile/static.go +++ b/internal/profile/static.go @@ -118,7 +118,7 @@ func newPackageRegistryDockerfile(_ string, profilePath string) (*simpleFile, er } // ElasticAgent8xEnvFile is the .env for the 8x stack. -const ElasticAgent8xEnvFile configFile = "elastic-agent_8x.env" +const ElasticAgent8xEnvFile configFile = "elastic-agent.8x.env" //go:embed _static/elastic-agent_8x.env var elasticAgent8xEnv string @@ -132,7 +132,7 @@ func newElasticAgent8xEnv(_ string, profilePath string) (*simpleFile, error) { } // ElasticAgentDefaultEnvFile is the default .env file. -const ElasticAgentDefaultEnvFile configFile = "elastic-agent_default.env" +const ElasticAgentDefaultEnvFile configFile = "elastic-agent.default.env" //go:embed _static/elastic-agent_default.env var elasticAgentDefaultEnv string diff --git a/internal/stack/logs.go b/internal/stack/logs.go index 896b201ada..961350bc22 100644 --- a/internal/stack/logs.go +++ b/internal/stack/logs.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/docker" + "github.com/elastic/elastic-package/internal/install" ) func dockerComposeLogs(serviceName string, snapshotFile string) ([]byte, error) { @@ -20,6 +21,9 @@ func dockerComposeLogs(serviceName string, snapshotFile string) ([]byte, error) } opts := compose.CommandOptions{ + Env: newEnvBuilder(). + withEnv(stackVariantAsEnv(install.DefaultStackVersion)). + build(), Services: []string{serviceName}, } diff --git a/internal/stack/shellinit.go b/internal/stack/shellinit.go index 029976ae20..5166c588cd 100644 --- a/internal/stack/shellinit.go +++ b/internal/stack/shellinit.go @@ -63,7 +63,11 @@ func ShellInit(elasticStackProfile *profile.Profile) (string, error) { } serviceComposeConfig, err := p.Config(compose.CommandOptions{ - Env: append(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv(), elasticStackProfile.ComposeEnvVars()...), + Env: newEnvBuilder(). + withEnvs(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv()). + withEnvs(elasticStackProfile.ComposeEnvVars()). + withEnv(stackVariantAsEnv(install.DefaultStackVersion)). + build(), }) if err != nil { return "", errors.Wrap(err, "could not get Docker Compose configuration for service") From 637f4bee70e528b146c4c94f0282311445a4349c Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 14 Feb 2022 11:34:14 +0100 Subject: [PATCH 4/6] Fix --- internal/profile/_static/docker-compose-stack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/profile/_static/docker-compose-stack.yml index 9d3c114acb..41142a6774 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/profile/_static/docker-compose-stack.yml @@ -103,7 +103,7 @@ services: retries: 180 interval: 1s hostname: docker-fleet-agent - env_file: "./elastic-agent_${STACK_VERSION_VARIANT}.env" + env_file: "./elastic-agent.${STACK_VERSION_VARIANT}.env" volumes: - type: bind source: ../../../tmp/service_logs/ From 18b8f4df560ee85b8930bb9005fbfb0c47c1bd4a Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 14 Feb 2022 15:00:06 +0100 Subject: [PATCH 5/6] Fix: Kubernetes agent token --- .../elastic-agent-managed.yaml.tmpl | 2 ++ .../runners/system/servicedeployer/kubernetes.go | 15 +++++++++++++-- test/packages/with-kind/kubernetes/manifest.yml | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/elastic-agent-managed.yaml.tmpl b/internal/testrunner/runners/system/servicedeployer/elastic-agent-managed.yaml.tmpl index 9dc962776d..6270c3464a 100644 --- a/internal/testrunner/runners/system/servicedeployer/elastic-agent-managed.yaml.tmpl +++ b/internal/testrunner/runners/system/servicedeployer/elastic-agent-managed.yaml.tmpl @@ -35,6 +35,8 @@ spec: # If left empty KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed - name: FLEET_ENROLLMENT_TOKEN value: "" + - name: FLEET_TOKEN_POLICY_NAME + value: "{{ .elasticAgentTokenPolicyName }}" - name: KIBANA_HOST value: "http://kibana:5601" - name: KIBANA_FLEET_USERNAME diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index 3ef1bde9fd..7b09a4398f 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -8,6 +8,7 @@ import ( "bytes" _ "embed" "path/filepath" + "strings" "text/template" "github.com/pkg/errors" @@ -179,8 +180,9 @@ func getElasticAgentYAML(stackVersion string) ([]byte, error) { var elasticAgentYaml bytes.Buffer err = tmpl.Execute(&elasticAgentYaml, map[string]string{ - "fleetURL": "http://fleet-server:8220", - "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, + "fleetURL": "http://fleet-server:8220", + "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, + "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion), }) if err != nil { return nil, errors.Wrap(err, "can't generate elastic agent manifest") @@ -188,3 +190,12 @@ func getElasticAgentYAML(stackVersion string) ([]byte, error) { return elasticAgentYaml.Bytes(), nil } + +// getTokenPolicyName function returns the preconfigured policy name for the 8.x Elastic stack. +// The logic is not present in older stacks. +func getTokenPolicyName(stackVersion string) string { + if strings.HasPrefix(stackVersion, "8.") { + return "Elastic-Agent (elastic-package)" + } + return "" +} diff --git a/test/packages/with-kind/kubernetes/manifest.yml b/test/packages/with-kind/kubernetes/manifest.yml index 990bb54d36..e72a1928b5 100644 --- a/test/packages/with-kind/kubernetes/manifest.yml +++ b/test/packages/with-kind/kubernetes/manifest.yml @@ -10,7 +10,7 @@ categories: - kubernetes release: ga conditions: - kibana.version: "^7.16.0 || ^8.0.0" + kibana.version: "^8.0.0" screenshots: - src: /img/metricbeat_kubernetes_overview.png title: Metricbeat Kubernetes Overview From 8cccd35b2f6a90aedd6a80eb7800d96353a1c2ce Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 15 Feb 2022 10:45:59 +0100 Subject: [PATCH 6/6] Fix: comment --- .../testrunner/runners/system/servicedeployer/kubernetes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index 7b09a4398f..a1c12e118d 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -191,8 +191,8 @@ func getElasticAgentYAML(stackVersion string) ([]byte, error) { return elasticAgentYaml.Bytes(), nil } -// getTokenPolicyName function returns the preconfigured policy name for the 8.x Elastic stack. -// The logic is not present in older stacks. +// getTokenPolicyName function returns the policy name for the 8.x Elastic stack. The agent's policy +// is predefined in the Kibana configuration file. The logic is not present in older stacks. func getTokenPolicyName(stackVersion string) string { if strings.HasPrefix(stackVersion, "8.") { return "Elastic-Agent (elastic-package)"