From a4fe7b1bf384ab243576153b3392ce72b59ccb0a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 8 Mar 2024 13:49:18 +0100 Subject: [PATCH 001/114] Moved k8s elastic agent template to _static folder --- .../{ => _static}/elastic-agent-managed.yaml.tmpl | 2 +- internal/servicedeployer/kubernetes.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename internal/servicedeployer/{ => _static}/elastic-agent-managed.yaml.tmpl (99%) diff --git a/internal/servicedeployer/elastic-agent-managed.yaml.tmpl b/internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl similarity index 99% rename from internal/servicedeployer/elastic-agent-managed.yaml.tmpl rename to internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl index f621e8986e..b3e728ad08 100644 --- a/internal/servicedeployer/elastic-agent-managed.yaml.tmpl +++ b/internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl @@ -278,4 +278,4 @@ metadata: namespace: kube-system labels: k8s-app: elastic-agent ---- \ No newline at end of file +--- diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index f5cf927a57..1a090712f8 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -192,7 +192,7 @@ func installElasticAgentInCluster(profile *profile.Profile, stackVersion string) return nil } -//go:embed elastic-agent-managed.yaml.tmpl +//go:embed _static/elastic-agent-managed.yaml.tmpl var elasticAgentManagedYamlTmpl string func getElasticAgentYAML(profile *profile.Profile, stackVersion string) ([]byte, error) { From c505e522c97b253894eab685b9109c70b6f6d28a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 8 Mar 2024 16:02:28 +0100 Subject: [PATCH 002/114] Fix lint warnings --- internal/servicedeployer/terraform.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/servicedeployer/terraform.go b/internal/servicedeployer/terraform.go index 8bbb0c65d4..381439133f 100644 --- a/internal/servicedeployer/terraform.go +++ b/internal/servicedeployer/terraform.go @@ -26,7 +26,7 @@ const ( terraformDeployerDockerfile = "Dockerfile" terraformDeployerRun = "run.sh" terraformOutputPrefix = "TF_OUTPUT_" - terraformOutputJsonFile = "tfOutputValues.json" + terraformOutputJSONFile = "tfOutputValues.json" ) //go:embed _static/terraform_deployer.yml @@ -48,7 +48,7 @@ type TerraformServiceDeployer struct { // like `{{TF_OUTPUT_queue_url}}` where `queue_url` is the output configured func addTerraformOutputs(outCtxt ServiceContext) error { // Read the `output.json` file where terraform outputs are generated - outputFile := filepath.Join(outCtxt.OutputDir, terraformOutputJsonFile) + outputFile := filepath.Join(outCtxt.OutputDir, terraformOutputJSONFile) content, err := os.ReadFile(outputFile) if err != nil { return fmt.Errorf("failed to read terraform output file: %w", err) @@ -210,8 +210,8 @@ func (tsd TerraformServiceDeployer) installDockerfile() (string, error) { return tfDir, nil } -func CreateOutputDir(locationManager *locations.LocationManager, runId string) (string, error) { - outputDir := filepath.Join(locationManager.ServiceOutputDir(), runId) +func CreateOutputDir(locationManager *locations.LocationManager, runID string) (string, error) { + outputDir := filepath.Join(locationManager.ServiceOutputDir(), runID) if err := os.MkdirAll(outputDir, 0755); err != nil { return "", fmt.Errorf("failed to create output directory: %w", err) } From 4e69a8a3cf5f54d2b037ef86d5628df423386442 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 8 Mar 2024 16:31:59 +0100 Subject: [PATCH 003/114] Remove unused parameter --- internal/testrunner/runners/system/runner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 447791a995..a833c48f2a 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -200,7 +200,7 @@ func (r *runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, e } serviceOptions := r.createServiceOptions(variant) - ctxt, err := r.createServiceContext(serviceOptions) + ctxt, err := r.createServiceContext() if err != nil { return result.WithError(err) } @@ -276,7 +276,7 @@ func (r *runner) createServiceOptions(variantName string) servicedeployer.Factor } } -func (r *runner) createServiceContext(serviceOptions servicedeployer.FactoryOptions) (servicedeployer.ServiceContext, error) { +func (r *runner) createServiceContext() (servicedeployer.ServiceContext, error) { var ctxt servicedeployer.ServiceContext ctxt.Name = r.options.TestFolder.Package ctxt.Logs.Folder.Local = r.locationManager.ServiceLogDir() @@ -472,7 +472,7 @@ func (r *runner) run() (results []testrunner.TestResult, err error) { func (r *runner) runTestPerVariant(result *testrunner.ResultComposer, cfgFile, variantName string) ([]testrunner.TestResult, error) { serviceOptions := r.createServiceOptions(variantName) - ctxt, err := r.createServiceContext(serviceOptions) + ctxt, err := r.createServiceContext() if err != nil { return result.WithError(err) } From 33c46194f3cf0a23031fdb7c75d9b52f57ea71c2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 12 Mar 2024 12:12:00 +0100 Subject: [PATCH 004/114] WIP --- internal/testrunner/runners/system/runner.go | 83 ++++++++++++++++++-- 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index a833c48f2a..71bfbe7c7f 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -20,6 +20,7 @@ import ( "github.com/Masterminds/semver/v3" "gopkg.in/yaml.v3" + "github.com/elastic/elastic-package/internal/agentdeployer" "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/elasticsearch" @@ -114,6 +115,7 @@ type runner struct { resetAgentPolicyHandler func() error resetAgentLogLevelHandler func() error shutdownServiceHandler func() error + shutdownAgentHandler func() error wipeDataStreamHandler func() error } @@ -261,6 +263,23 @@ func (r *runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, e return result.WithSuccess() } +func (r *runner) createAgentOptions(scenario scenarioTest) agentdeployer.FactoryOptions { + return agentdeployer.FactoryOptions{ + Profile: r.options.Profile, + API: r.options.API, + PackageRootPath: r.options.PackageRootPath, + DataStreamRootPath: r.dataStreamPath, + DevDeployDir: DevDeployDir, + Type: agentdeployer.TypeTest, + StackVersion: r.stackVersion.Version(), + PackageName: scenario.pkgManifest.Name, + DataStream: scenario.dataStreamManifest.Name, + RunTearDown: r.options.RunTearDown, + RunTestsOnly: r.options.RunTestsOnly, + RunSetup: r.options.RunSetup, + } +} + func (r *runner) createServiceOptions(variantName string) servicedeployer.FactoryOptions { return servicedeployer.FactoryOptions{ Profile: r.options.Profile, @@ -276,6 +295,17 @@ func (r *runner) createServiceOptions(variantName string) servicedeployer.Factor } } +func (r *runner) createAgentInfo() (agentdeployer.AgentInfo, error) { + var info agentdeployer.AgentInfo + + info.Name = r.options.TestFolder.Package + info.Logs.Folder.Local = r.locationManager.ServiceLogDir() + info.Logs.Folder.Agent = ServiceLogsAgentDir + info.Test.RunID = createTestRunID() + + return info, nil +} + func (r *runner) createServiceContext() (servicedeployer.ServiceContext, error) { var ctxt servicedeployer.ServiceContext ctxt.Name = r.options.TestFolder.Package @@ -338,6 +368,13 @@ func (r *runner) tearDownTest() error { r.shutdownServiceHandler = nil } + if r.shutdownAgentHandler != nil { + if err := r.shutdownAgentHandler(); err != nil { + return err + } + r.shutdownAgentHandler = nil + } + if r.wipeDataStreamHandler != nil { if err := r.wipeDataStreamHandler(); err != nil { return err @@ -842,11 +879,38 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic } } + // Setup agent + logger.Debug("setting up agent...") + agentOptions := r.createAgentOptions(scenario) + agentInfo, err := r.createAgentInfo() + if err != nil { + return nil, err + } + + agentDeployer, err := agentdeployer.Factory(agentOptions) + if err != nil { + return nil, fmt.Errorf("could not create agent runner: %w", err) + } + agentDeployed, err := agentDeployer.SetUp(agentInfo) + if err != nil { + return nil, fmt.Errorf("could not setup agent: %w", err) + } + r.shutdownAgentHandler = func() error { + logger.Debug("tearing down agent...") + if err := agentDeployed.TearDown(); err != nil { + return fmt.Errorf("error tearing down agent: %w", err) + } + + return nil + } + var origPolicy kibana.Policy - agents, err := checkEnrolledAgents(r.options.KibanaClient, ctxt) + agents, err := checkEnrolledAgents(r.options.KibanaClient, ctxt, agentInfo) if err != nil { return nil, fmt.Errorf("can't check enrolled agents: %w", err) } + data, _ := json.Marshal(agents) + logger.Debugf("JSON Agents:\n%s", string(data)) agent := agents[0] if r.options.RunTearDown || r.options.RunTestsOnly { @@ -1002,7 +1066,7 @@ func (r *runner) removeServiceStateFile() error { func (r *runner) createServiceStateDir() error { dirPath := filepath.Dir(r.serviceStateFilePath) - err := os.MkdirAll(dirPath, 0755) + err := os.MkdirAll(dirPath, 0o755) if err != nil { return fmt.Errorf("mkdir failed (path: %s): %w", dirPath, err) } @@ -1044,7 +1108,7 @@ func (r *runner) writeScenarioState(currentPolicy, origPolicy *kibana.Policy, co return fmt.Errorf("failed to marshall service setup data: %w", err) } - err = os.WriteFile(r.serviceStateFilePath, dataBytes, 0644) + err = os.WriteFile(r.serviceStateFilePath, dataBytes, 0o644) if err != nil { return fmt.Errorf("failed to write service setup JSON: %w", err) } @@ -1182,7 +1246,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return r.validateTestScenario(result, scenario, config, serviceOptions) } -func checkEnrolledAgents(client *kibana.Client, ctxt servicedeployer.ServiceContext) ([]kibana.Agent, error) { +func checkEnrolledAgents(client *kibana.Client, ctxt servicedeployer.ServiceContext, agentInfo agentdeployer.AgentInfo) ([]kibana.Agent, error) { var agents []kibana.Agent enrolled, err := waitUntilTrue(func() (bool, error) { if signal.SIGINT() { @@ -1194,7 +1258,7 @@ func checkEnrolledAgents(client *kibana.Client, ctxt servicedeployer.ServiceCont return false, fmt.Errorf("could not list agents: %w", err) } - agents = filterAgents(allAgents, ctxt) + agents = filterAgents(allAgents, ctxt, agentInfo) logger.Debugf("found %d enrolled agent(s)", len(agents)) if len(agents) == 0 { return false, nil // selected agents are unavailable yet @@ -1618,7 +1682,7 @@ func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) } } -func filterAgents(allAgents []kibana.Agent, ctx servicedeployer.ServiceContext) []kibana.Agent { +func filterAgents(allAgents []kibana.Agent, ctx servicedeployer.ServiceContext, agentInfo agentdeployer.AgentInfo) []kibana.Agent { if ctx.Agent.Host.NamePrefix != "" { logger.Debugf("filter agents using criteria: NamePrefix=%s", ctx.Agent.Host.NamePrefix) } @@ -1629,7 +1693,10 @@ func filterAgents(allAgents []kibana.Agent, ctx servicedeployer.ServiceContext) continue // For some reason Kibana doesn't always return a valid policy revision (eventually it will be present and valid) } - if ctx.Agent.Host.NamePrefix != "" && !strings.HasPrefix(agent.LocalMetadata.Host.Name, ctx.Agent.Host.NamePrefix) { + serviceHasPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, ctx.Agent.Host.NamePrefix) + agentHasPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) + + if ctx.Agent.Host.NamePrefix != "" && !serviceHasPrefix && !agentHasPrefix { continue } filtered = append(filtered, agent) @@ -1644,7 +1711,7 @@ func writeSampleEvent(path string, doc common.MapStr, specVersion semver.Version return fmt.Errorf("marshalling sample event failed: %w", err) } - err = os.WriteFile(filepath.Join(path, "sample_event.json"), body, 0644) + err = os.WriteFile(filepath.Join(path, "sample_event.json"), body, 0o644) if err != nil { return fmt.Errorf("writing sample event failed: %w", err) } From 5e88e06616a0855dfcdc35e074cbce5896223a7d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 12 Mar 2024 15:56:59 +0100 Subject: [PATCH 005/114] Add agent per test - PoC --- .../agentdeployer/_static/custom-agent.yml | 8 + .../_static/docker-agent-base.yml | 18 ++ internal/agentdeployer/agent.go | 250 +++++++++++++++++ internal/agentdeployer/compose.go | 264 ++++++++++++++++++ internal/agentdeployer/deployed_agent.go | 27 ++ internal/agentdeployer/deployer.go | 13 + internal/agentdeployer/factory.go | 122 ++++++++ internal/agentdeployer/info.go | 73 +++++ internal/agentdeployer/variants.go | 98 +++++++ 9 files changed, 873 insertions(+) create mode 100644 internal/agentdeployer/_static/custom-agent.yml create mode 100644 internal/agentdeployer/_static/docker-agent-base.yml create mode 100644 internal/agentdeployer/agent.go create mode 100644 internal/agentdeployer/compose.go create mode 100644 internal/agentdeployer/deployed_agent.go create mode 100644 internal/agentdeployer/deployer.go create mode 100644 internal/agentdeployer/factory.go create mode 100644 internal/agentdeployer/info.go create mode 100644 internal/agentdeployer/variants.go diff --git a/internal/agentdeployer/_static/custom-agent.yml b/internal/agentdeployer/_static/custom-agent.yml new file mode 100644 index 0000000000..531c24ced9 --- /dev/null +++ b/internal/agentdeployer/_static/custom-agent.yml @@ -0,0 +1,8 @@ +version: "2.3" +services: + docker-custom-agent: + pid: host + cap_add: + - AUDIT_CONTROL + - AUDIT_READ + user: root diff --git a/internal/agentdeployer/_static/docker-agent-base.yml b/internal/agentdeployer/_static/docker-agent-base.yml new file mode 100644 index 0000000000..5dcd21bd72 --- /dev/null +++ b/internal/agentdeployer/_static/docker-agent-base.yml @@ -0,0 +1,18 @@ +version: "2.3" +services: + docker-custom-agent: + image: "${ELASTIC_AGENT_IMAGE_REF}" + healthcheck: + test: "elastic-agent status" + retries: 180 + interval: 1s + hostname: docker-custom-agent + environment: + - FLEET_ENROLL=1 + - FLEET_URL=https://fleet-server:8220 + - KIBANA_HOST=https://kibana:5601 + - FLEET_TOKEN_POLICY_NAME=${FLEET_TOKEN_POLICY_NAME} + volumes: + - ${SERVICE_LOGS_DIR}:/tmp/service_logs/ + - ${LOCAL_CA_CERT}:/etc/ssl/certs/elastic-package.pem + diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go new file mode 100644 index 0000000000..1612284a33 --- /dev/null +++ b/internal/agentdeployer/agent.go @@ -0,0 +1,250 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package agentdeployer + +import ( + _ "embed" + "encoding/base64" + "fmt" + "os" + "path/filepath" + + "github.com/elastic/elastic-package/internal/compose" + "github.com/elastic/elastic-package/internal/docker" + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/files" + "github.com/elastic/elastic-package/internal/install" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/stack" +) + +const ( + dockerCustomAgentName = "docker-custom-agent" + dockerCustomAgentDir = "docker_custom_agent" + dockerCustomAgentDockerCompose = "docker-agent-base.yml" + defaultAgentPolicyName = "Elastic-Agent (elastic-package)" +) + +//go:embed _static/docker-agent-base.yml +var dockerAgentDockerComposeContent []byte + +// CustomAgentDeployer knows how to deploy a custom elastic-agent defined via +// a Docker Compose file. +type CustomAgentDeployer struct { + profile *profile.Profile + dockerComposeFile string + stackVersion string + + variant AgentVariant + + API *elasticsearch.API + + packageName string + dataStream string + + runTearDown bool + runTestsOnly bool +} + +type CustomAgentDeployerOptions struct { + Profile *profile.Profile + DockerComposeFile string + StackVersion string + Variant AgentVariant + + API *elasticsearch.API + + PackageName string + DataStream string + + RunTearDown bool + RunTestsOnly bool +} + +// NewCustomAgentDeployer returns a new instance of a deployedCustomAgent. +func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDeployer, error) { + return &CustomAgentDeployer{ + profile: options.Profile, + dockerComposeFile: options.DockerComposeFile, + stackVersion: options.StackVersion, + packageName: options.PackageName, + dataStream: options.DataStream, + API: options.API, + variant: options.Variant, + runTearDown: options.RunTearDown, + runTestsOnly: options.RunTestsOnly, + }, nil +} + +func readCACertBase64(profile *profile.Profile) (string, error) { + caCertPath, err := stack.FindCACertificate(profile) + if err != nil { + return "", fmt.Errorf("can't locate CA certificate: %w", err) + } + + d, err := os.ReadFile(caCertPath) + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(d), nil +} + +// SetUp sets up the service and returns any relevant information. +func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { + logger.Debug("setting up service using Docker Compose agent deployer") + + appConfig, err := install.Configuration() + if err != nil { + return nil, fmt.Errorf("can't read application configuration: %w", err) + } + + caCertPath, err := stack.FindCACertificate(d.profile) + if err != nil { + return nil, fmt.Errorf("can't locate CA certificate: %w", err) + } + + env := append( + appConfig.StackImageRefs(d.stackVersion).AsEnv(), + fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local), + fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), + fmt.Sprintf("%s=%s", fleetPolicyEnv, defaultAgentPolicyName), + ) + + configDir, err := d.installDockerfile() + if err != nil { + return nil, fmt.Errorf("could not create resources for custom agent: %w", err) + } + + ymlPaths := []string{ + filepath.Join(configDir, dockerCustomAgentDockerCompose), + } + if d.dockerComposeFile != "" { + ymlPaths = []string{ + d.dockerComposeFile, + filepath.Join(configDir, dockerCustomAgentDockerCompose), + } + } + + composeProjectName := fmt.Sprintf("elastic-package-agent-%s", d.agentName()) + + service := dockerComposeDeployedAgent{ + ymlPaths: ymlPaths, + project: composeProjectName, + variant: AgentVariant{ + Name: dockerCustomAgentName, + Env: env, + }, + } + + outCtxt := inCtxt + outCtxt.ConfigDir = configDir + + p, err := compose.NewProject(service.project, service.ymlPaths...) + if err != nil { + return nil, fmt.Errorf("could not create Docker Compose project for agent: %w", err) + } + + // Verify the Elastic stack network + err = stack.EnsureStackNetworkUp(d.profile) + if err != nil { + return nil, fmt.Errorf("stack network is not ready: %w", err) + } + + // Clean service logs + if d.runTestsOnly { + // service logs folder must no be deleted to avoid breaking log files written + // by the service. If this is required, those files should be rotated or truncated + // so the service can still write to them. + logger.Debug("Skipping removing service logs folder folder %s", outCtxt.Logs.Folder.Local) + } else { + err = files.RemoveContent(outCtxt.Logs.Folder.Local) + if err != nil { + return nil, fmt.Errorf("removing service logs failed: %w", err) + } + } + + // Service name defined in the docker-compose file + inCtxt.Name = dockerCustomAgentName + serviceName := inCtxt.Name + + opts := compose.CommandOptions{ + Env: env, + ExtraArgs: []string{"--build", "-d"}, + } + + if d.runTestsOnly || d.runTearDown { + logger.Debug("Skipping bringing up docker-compose project and connect container to network (non setup steps)") + } else { + err = p.Up(opts) + if err != nil { + return nil, fmt.Errorf("could not boot up agent using Docker Compose: %w", err) + } + // Connect service network with stack network (for the purpose of metrics collection) + err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network(d.profile)) + if err != nil { + return nil, fmt.Errorf("can't attach service container to the stack network: %w", err) + } + } + + // requires to be connected the service to the stack network + err = p.WaitForHealthy(opts) + if err != nil { + processAgentContainerLogs(p, compose.CommandOptions{ + Env: opts.Env, + }, outCtxt.Name) + return nil, fmt.Errorf("service is unhealthy: %w", err) + } + + // Build service container name + outCtxt.Hostname = p.ContainerName(serviceName) + + logger.Debugf("adding service container %s internal ports to context", p.ContainerName(serviceName)) + serviceComposeConfig, err := p.Config(compose.CommandOptions{Env: env}) + if err != nil { + return nil, fmt.Errorf("could not get Docker Compose configuration for service: %w", err) + } + + s := serviceComposeConfig.Services[serviceName] + outCtxt.Ports = make([]int, len(s.Ports)) + for idx, port := range s.Ports { + outCtxt.Ports[idx] = port.InternalPort + } + + // Shortcut to first port for convenience + if len(outCtxt.Ports) > 0 { + outCtxt.Port = outCtxt.Ports[0] + } + + outCtxt.Agent.Host.NamePrefix = inCtxt.Name + service.agentInfo = outCtxt + return &service, nil +} + +func (d *CustomAgentDeployer) agentName() string { + if d.variant.Name != "" { + return fmt.Sprintf("%s-%s-%s", d.packageName, d.variant.Name, d.dataStream) + } + return fmt.Sprintf("%s-%s", d.packageName, d.dataStream) +} + +// installDockerfile creates the files needed to run the custom elastic agent and returns +// the directory with these files. +func (d *CustomAgentDeployer) installDockerfile() (string, error) { + customAgentDir := filepath.Join(d.profile.ProfilePath, fmt.Sprintf("agent-%s", d.agentName())) + err := os.MkdirAll(customAgentDir, 0o755) + if err != nil { + return "", fmt.Errorf("failed to create directory for custom agent files: %w", err) + } + + customAgentDockerfile := filepath.Join(customAgentDir, dockerCustomAgentDockerCompose) + err = os.WriteFile(customAgentDockerfile, dockerAgentDockerComposeContent, 0o644) + if err != nil { + return "", fmt.Errorf("failed to create docker compose file for custom agent: %w", err) + } + + return customAgentDir, nil +} diff --git a/internal/agentdeployer/compose.go b/internal/agentdeployer/compose.go new file mode 100644 index 0000000000..31870b3b68 --- /dev/null +++ b/internal/agentdeployer/compose.go @@ -0,0 +1,264 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. +package agentdeployer + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/elastic/elastic-package/internal/builder" + "github.com/elastic/elastic-package/internal/compose" + "github.com/elastic/elastic-package/internal/docker" + "github.com/elastic/elastic-package/internal/files" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/stack" +) + +// DockerComposeServiceDeployer knows how to deploy a service defined via +// a Docker Compose file. +type DockerComposeAgentDeployer struct { + profile *profile.Profile + ymlPaths []string + variant AgentVariant +} + +type dockerComposeDeployedAgent struct { + agentInfo AgentInfo + + ymlPaths []string + project string + variant AgentVariant + env []string +} + +// NewDockerComposeAgentDeployer returns a new instance of a DockerComposeAgentDeployer. +func NewDockerComposeAgentDeployer(profile *profile.Profile, ymlPaths []string) (*DockerComposeAgentDeployer, error) { + return &DockerComposeAgentDeployer{ + profile: profile, + ymlPaths: ymlPaths, + }, nil +} + +// SetUp sets up the service and returns any relevant information. +func (d *DockerComposeAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { + logger.Debug("setting up agent using Docker Compose agent deployer") + agent := dockerComposeDeployedAgent{ + ymlPaths: d.ymlPaths, + project: "elastic-package-agent", + variant: d.variant, + env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local)}, + } + outCtxt := inCtxt + + p, err := compose.NewProject(agent.project, agent.ymlPaths...) + if err != nil { + return nil, fmt.Errorf("could not create Docker Compose project for service: %w", err) + } + + // Verify the Elastic stack network + err = stack.EnsureStackNetworkUp(d.profile) + if err != nil { + return nil, fmt.Errorf("stack network is not ready: %w", err) + } + + // Clean service logs + err = files.RemoveContent(outCtxt.Logs.Folder.Local) + if err != nil { + return nil, fmt.Errorf("removing service logs failed: %w", err) + } + + // Boot up service + if d.variant.active() { + logger.Infof("Using service variant: %s", d.variant.String()) + } + + agentName := inCtxt.Name + opts := compose.CommandOptions{ + Env: append( + agent.env, + d.variant.Env..., + ), + ExtraArgs: []string{"--build", "-d"}, + } + err = p.Up(opts) + if err != nil { + return nil, fmt.Errorf("could not boot up service using Docker Compose: %w", err) + } + + err = p.WaitForHealthy(opts) + if err != nil { + processAgentContainerLogs(p, compose.CommandOptions{ + Env: opts.Env, + }, outCtxt.Name) + return nil, fmt.Errorf("service is unhealthy: %w", err) + } + + // Build service container name + outCtxt.Hostname = p.ContainerName(agentName) + + // Connect service network with stack network (for the purpose of metrics collection) + err = docker.ConnectToNetwork(p.ContainerName(agentName), stack.Network(d.profile)) + if err != nil { + return nil, fmt.Errorf("can't attach service container to the stack network: %w", err) + } + + logger.Debugf("adding agent container %s internal ports to context", p.ContainerName(agentName)) + agentComposeConfig, err := p.Config(compose.CommandOptions{ + Env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, outCtxt.Logs.Folder.Local)}, + }) + if err != nil { + return nil, fmt.Errorf("could not get Docker Compose configuration for agent: %w", err) + } + + s := agentComposeConfig.Services[agentName] + outCtxt.Ports = make([]int, len(s.Ports)) + for idx, port := range s.Ports { + outCtxt.Ports[idx] = port.InternalPort + } + + // Shortcut to first port for convenience + if len(outCtxt.Ports) > 0 { + outCtxt.Port = outCtxt.Ports[0] + } + + outCtxt.Agent.Host.NamePrefix = "docker-custom-agent" + agent.agentInfo = outCtxt + return &agent, nil +} + +// Signal sends a signal to the agent. +func (s *dockerComposeDeployedAgent) Signal(signal string) error { + p, err := compose.NewProject(s.project, s.ymlPaths...) + if err != nil { + return fmt.Errorf("could not create Docker Compose project for service: %w", err) + } + + opts := compose.CommandOptions{ + Env: append( + s.env, + s.variant.Env..., + ), + ExtraArgs: []string{"-s", signal}, + } + if s.agentInfo.Name != "" { + opts.Services = append(opts.Services, s.agentInfo.Name) + } + + err = p.Kill(opts) + if err != nil { + return fmt.Errorf("could not send %q signal: %w", signal, err) + } + return nil +} + +// ExitCode returns true if the agent is exited and its exit code. +func (s *dockerComposeDeployedAgent) ExitCode(agent string) (bool, int, error) { + p, err := compose.NewProject(s.project, s.ymlPaths...) + if err != nil { + return false, -1, fmt.Errorf("could not create Docker Compose project for agent: %w", err) + } + + opts := compose.CommandOptions{ + Env: append( + s.env, + s.variant.Env..., + ), + } + + return p.ServiceExitCode(agent, opts) +} + +// TearDown tears down the agent. +func (s *dockerComposeDeployedAgent) TearDown() error { + logger.Debugf("tearing down agent using Docker Compose runner") + defer func() { + err := files.RemoveContent(s.agentInfo.Logs.Folder.Local) + if err != nil { + logger.Errorf("could not remove the agent logs (path: %s)", s.agentInfo.Logs.Folder.Local) + } + // Remove the outputs generated by the service container + if err = os.RemoveAll(s.agentInfo.OutputDir); err != nil { + logger.Errorf("could not remove the temporary output files %w", err) + } + + // Remove the configuration dir (e.g. compose scenario files) + if err = os.RemoveAll(s.agentInfo.ConfigDir); err != nil { + logger.Errorf("could not remove the agent configuration directory %w", err) + } + }() + + p, err := compose.NewProject(s.project, s.ymlPaths...) + if err != nil { + return fmt.Errorf("could not create Docker Compose project for service: %w", err) + } + + opts := compose.CommandOptions{ + Env: append( + s.env, + s.variant.Env..., + ), + } + processAgentContainerLogs(p, opts, s.agentInfo.Name) + + if err := p.Down(compose.CommandOptions{ + Env: opts.Env, + ExtraArgs: []string{"--volumes"}, // Remove associated volumes. + }); err != nil { + return fmt.Errorf("could not shut down agent using Docker Compose: %w", err) + } + return nil +} + +// Context returns the current context for the agent. +func (s *dockerComposeDeployedAgent) Context() AgentInfo { + return s.agentInfo +} + +// SetContext sets the current context for the agent. +func (s *dockerComposeDeployedAgent) SetContext(ctxt AgentInfo) error { + s.agentInfo = ctxt + return nil +} + +func processAgentContainerLogs(p *compose.Project, opts compose.CommandOptions, agentName string) { + content, err := p.Logs(opts) + if err != nil { + logger.Errorf("can't export service logs: %v", err) + return + } + + if len(content) == 0 { + logger.Info("service container hasn't written anything logs.") + return + } + + err = writeAgentContainerLogs(agentName, content) + if err != nil { + logger.Errorf("can't write service container logs: %v", err) + } +} + +func writeAgentContainerLogs(agentName string, content []byte) error { + buildDir, err := builder.BuildDirectory() + if err != nil { + return fmt.Errorf("locating build directory failed: %w", err) + } + + containerLogsDir := filepath.Join(buildDir, "container-logs") + err = os.MkdirAll(containerLogsDir, 0o755) + if err != nil { + return fmt.Errorf("can't create directory for agent container logs (path: %s): %w", containerLogsDir, err) + } + + containerLogsFilepath := filepath.Join(containerLogsDir, fmt.Sprintf("%s-%d.log", agentName, time.Now().UnixNano())) + logger.Infof("Write container logs to file: %s", containerLogsFilepath) + err = os.WriteFile(containerLogsFilepath, content, 0o644) + if err != nil { + return fmt.Errorf("can't write container logs to file (path: %s): %w", containerLogsFilepath, err) + } + return nil +} diff --git a/internal/agentdeployer/deployed_agent.go b/internal/agentdeployer/deployed_agent.go new file mode 100644 index 0000000000..006284c72a --- /dev/null +++ b/internal/agentdeployer/deployed_agent.go @@ -0,0 +1,27 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package agentdeployer + +import "errors" + +var ErrNotSupported error = errors.New("not supported") + +// DeployedAgent defines the interface for interacting with a service that has been deployed. +type DeployedAgent interface { + // TearDown implements the logic for tearing down a service. + TearDown() error + + // Signal sends a signal to the service. + Signal(signal string) error + + // Context returns the current context from the service. + Context() AgentInfo + + // SetContext sets the current context for the service. + SetContext(str AgentInfo) error + + // ExitCode returns true if the service is exited and its exit code. + ExitCode(service string) (bool, int, error) +} diff --git a/internal/agentdeployer/deployer.go b/internal/agentdeployer/deployer.go new file mode 100644 index 0000000000..d90f5deb60 --- /dev/null +++ b/internal/agentdeployer/deployer.go @@ -0,0 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package agentdeployer + +// AgentDeployer defines the interface for deploying an agent. It defines methods for +// controlling the lifecycle of an agent. +type AgentDeployer interface { + // SetUp implements the logic for setting up an agent. It takes a context and returns a + // AgentHandler. + SetUp(ctxt AgentInfo) (DeployedAgent, error) +} diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go new file mode 100644 index 0000000000..4de6ff6fb1 --- /dev/null +++ b/internal/agentdeployer/factory.go @@ -0,0 +1,122 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package agentdeployer + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/profile" +) + +const ( + TypeTest = "test" + TypeBench = "bench" +) + +// FactoryOptions defines options used to create an instance of a service deployer. +type FactoryOptions struct { + Profile *profile.Profile + + API *elasticsearch.API + + PackageRootPath string + DataStreamRootPath string + DevDeployDir string + Type string + StackVersion string + + PackageName string + DataStream string + + Variant string + + RunTearDown bool + RunTestsOnly bool + RunSetup bool +} + +// Factory chooses the appropriate service runner for the given data stream, depending +// on service configuration files defined in the package or data stream. +func Factory(options FactoryOptions) (AgentDeployer, error) { + // devDeployPath, err := FindDevDeployPath(options) + // if err != nil { + // return nil, fmt.Errorf("can't find \"%s\" directory: %w", options.DevDeployDir, err) + // } + + // agentDeployerName, err := findAgentDeployer(devDeployPath) + // if err != nil { + // return nil, fmt.Errorf("can't find any valid agent deployer: %w", err) + // } + + // agentDeployerPath := filepath.Join(devDeployPath, agentDeployerName) + agentDeployerName := "agent" + + switch agentDeployerName { + case "agent": + if options.Type != TypeTest { + return nil, fmt.Errorf("agent deployer is not supported for type %s", options.Type) + } + // FIXME + // customAgentCfgYMLPath := filepath.Join(agentDeployerPath, "custom-agent.yml") + // if _, err := os.Stat(customAgentCfgYMLPath); err != nil { + // return nil, fmt.Errorf("can't find expected file custom-agent.yml: %w", err) + // } + customAgentCfgYMLPath := "" + opts := CustomAgentDeployerOptions{ + Profile: options.Profile, + API: options.API, + DockerComposeFile: customAgentCfgYMLPath, + StackVersion: options.StackVersion, + PackageName: options.PackageName, + DataStream: options.DataStream, + RunTearDown: options.RunTearDown, + RunTestsOnly: options.RunTestsOnly, + } + return NewCustomAgentDeployer(opts) + } + return nil, fmt.Errorf("unsupported agent deployer (name: %s)", agentDeployerName) +} + +// FindDevDeployPath function returns a path reference to the "_dev/deploy" directory. +func FindDevDeployPath(options FactoryOptions) (string, error) { + dataStreamDevDeployPath := filepath.Join(options.DataStreamRootPath, options.DevDeployDir) + if _, err := os.Stat(dataStreamDevDeployPath); err == nil { + return dataStreamDevDeployPath, nil + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("stat failed for data stream (path: %s): %w", dataStreamDevDeployPath, err) + } + + packageDevDeployPath := filepath.Join(options.PackageRootPath, options.DevDeployDir) + if _, err := os.Stat(packageDevDeployPath); err == nil { + return packageDevDeployPath, nil + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("stat failed for package (path: %s): %w", packageDevDeployPath, err) + } + + return "", fmt.Errorf("\"%s\" directory doesn't exist", options.DevDeployDir) +} + +func findAgentDeployer(devDeployPath string) (string, error) { + fis, err := os.ReadDir(devDeployPath) + if err != nil { + return "", fmt.Errorf("can't read directory (path: %s): %w", devDeployPath, err) + } + + var folders []os.DirEntry + for _, fi := range fis { + if fi.IsDir() { + folders = append(folders, fi) + } + } + + if len(folders) != 1 { + return "", fmt.Errorf("expected to find only one agent deployer in \"%s\"", devDeployPath) + } + return folders[0].Name(), nil +} diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go new file mode 100644 index 0000000000..4a1bdcb2d6 --- /dev/null +++ b/internal/agentdeployer/info.go @@ -0,0 +1,73 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package agentdeployer + +const ( + localCACertEnv = "LOCAL_CA_CERT" + serviceLogsDirEnv = "SERVICE_LOGS_DIR" + testRunIDEnv = "TEST_RUN_ID" + fleetPolicyEnv = "FLEET_TOKEN_POLICY_NAME" +) + +// AgentInfo encapsulates context that is both available to a AgentDeployer and +// populated by a DeployedAgent. The fields in AgentInfo may be used in handlebars +// templates in system test configuration files, for example: {{ Hostname }}. +type AgentInfo struct { + // Name is the name of the service. + Name string + + // Hostname is the host name of the service, as addressable from + // the Agent container. + Hostname string + + // Ports is a list of ports that the service listens on, as addressable + // from the Agent container. + Ports []int + + // Port points to the first port in the list of ports. It's provided as + // a convenient shortcut as most services tend to listen on a single port. + Port int + + // Logs contains folder paths for log files produced by the service. + Logs struct { + Folder struct { + // Local contains the folder path where log files produced by + // the service are stored on the local filesystem, i.e. where + // elastic-package is running. + Local string + + // Agent contains the folder path where log files produced by + // the service are stored on the Agent container's filesystem. + Agent string + } + } + + // Test related properties. + Test struct { + // RunID identifies the current test run. + RunID string + } + + // Agent related properties. + Agent struct { + // Host describes the machine which is running the agent. + Host struct { + // Name prefix for the host's name + NamePrefix string + } + } + + // CustomProperties store additional data used to boot up the service, e.g. AWS credentials. + CustomProperties map[string]interface{} + + // Directory to store any outputs generated + OutputDir string + + // Directory to store agent configuration files + ConfigDir string + + // Service token + Token string +} diff --git a/internal/agentdeployer/variants.go b/internal/agentdeployer/variants.go new file mode 100644 index 0000000000..3326c7e8b7 --- /dev/null +++ b/internal/agentdeployer/variants.go @@ -0,0 +1,98 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package agentdeployer + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// VariantsFile describes different variants of the service under test. +type VariantsFile struct { + Default string `yaml:"default"` + Variants map[string]Environment +} + +// Environment is a key-value map storing environment variables. +type Environment map[string]string + +// AgentVariant describes a variant of the service using Environment variables. +type AgentVariant struct { + Name string + Env []string // Environment variables in format of pairs: key=value +} + +// String method returns a string representation of the service variant. +func (sv *AgentVariant) String() string { + return fmt.Sprintf("AgentVariant{Name: %s, Env: %s}", sv.Name, strings.Join(sv.Env, ",")) +} + +func (sv *AgentVariant) active() bool { + return sv.Name != "" +} + +// ReadVariantsFile function reads available service variants. +func ReadVariantsFile(devDeployPath string) (*VariantsFile, error) { + variantsYmlPath := filepath.Join(devDeployPath, "variants.yml") + _, err := os.Stat(variantsYmlPath) + if errors.Is(err, os.ErrNotExist) { + return nil, os.ErrNotExist + } + if err != nil { + return nil, fmt.Errorf("can't stat variants file: %w", err) + } + + content, err := os.ReadFile(variantsYmlPath) + if err != nil { + return nil, fmt.Errorf("can't read variants file: %w", err) + } + + var f VariantsFile + err = yaml.Unmarshal(content, &f) + if err != nil { + return nil, fmt.Errorf("can't unmarshal variants file: %w", err) + } + return &f, nil +} + +func useAgentVariant(devDeployPath, selected string) (AgentVariant, error) { + f, err := ReadVariantsFile(devDeployPath) + if errors.Is(err, os.ErrNotExist) { + return AgentVariant{}, nil // no "variants.yml" present + } else if err != nil { + return AgentVariant{}, err + } + + if selected == "" { + selected = f.Default + } + + if f.Default == "" { + return AgentVariant{}, errors.New("default variant is undefined") + } + + env, ok := f.Variants[selected] + if !ok { + return AgentVariant{}, fmt.Errorf(`variant "%s" is missing`, selected) + } + + return AgentVariant{ + Name: selected, + Env: asEnvVarPairs(env), + }, nil +} + +func asEnvVarPairs(env Environment) []string { + var pairs []string + for k, v := range env { + pairs = append(pairs, fmt.Sprintf("%s=%s", k, v)) + } + return pairs +} From bb0f72e74b5d12ff48f4afd5fb108df0aa5a0fa4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 12 Mar 2024 15:57:11 +0100 Subject: [PATCH 006/114] Filter agents by status too --- internal/kibana/agents.go | 1 + internal/testrunner/runners/system/runner.go | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/internal/kibana/agents.go b/internal/kibana/agents.go index f9add95512..acc169eb8b 100644 --- a/internal/kibana/agents.go +++ b/internal/kibana/agents.go @@ -35,6 +35,7 @@ type Agent struct { } `json:"agent"` } `json:"elastic"` } `json:"local_metadata"` + Status string `json:"status"` } // String method returns string representation of an agent. diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 71bfbe7c7f..11aa3eebda 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1693,10 +1693,21 @@ func filterAgents(allAgents []kibana.Agent, ctx servicedeployer.ServiceContext, continue // For some reason Kibana doesn't always return a valid policy revision (eventually it will be present and valid) } + if agent.PolicyID != "elastic-agent-managed-ep" { + logger.Debugf("filted agent %q", agent.ID) + continue + } + + if agent.Status != "online" { + logger.Debugf("filted agent %q", agent.ID) + continue + } + serviceHasPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, ctx.Agent.Host.NamePrefix) agentHasPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) if ctx.Agent.Host.NamePrefix != "" && !serviceHasPrefix && !agentHasPrefix { + logger.Debugf("filtered agent %q", agent.ID) continue } filtered = append(filtered, agent) From 8ccfdcd25ffa3b819ba9cf3ab5c3c0606b89d7df Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 12 Mar 2024 17:55:24 +0100 Subject: [PATCH 007/114] Use different folder for logs in each agent --- internal/agentdeployer/agent.go | 10 +++ internal/configuration/locations/locations.go | 5 ++ internal/testrunner/runners/system/runner.go | 66 ++++++++++--------- 3 files changed, 51 insertions(+), 30 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 1612284a33..73ed42f613 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -12,6 +12,7 @@ import ( "path/filepath" "github.com/elastic/elastic-package/internal/compose" + "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/files" @@ -248,3 +249,12 @@ func (d *CustomAgentDeployer) installDockerfile() (string, error) { return customAgentDir, nil } + +func CreateServiceLogsDir(elasticPackagePath *locations.LocationManager, name string) (string, error) { + dirPath := elasticPackagePath.ServiceLogDirPerAgent(name) + err := os.MkdirAll(dirPath, 0755) + if err != nil { + return "", fmt.Errorf("mkdir failed (path: %s): %w", dirPath, err) + } + return dirPath, nil +} diff --git a/internal/configuration/locations/locations.go b/internal/configuration/locations/locations.go index 3f4fba0fbd..bd93d4bb4c 100644 --- a/internal/configuration/locations/locations.go +++ b/internal/configuration/locations/locations.go @@ -98,6 +98,11 @@ func (loc LocationManager) ServiceLogDir() string { return filepath.Join(loc.stackPath, serviceLogsDir) } +// ServiceLogDirPerAgent returns the log directory assigned to an specific agent +func (loc LocationManager) ServiceLogDirPerAgent(name string) string { + return filepath.Join(loc.stackPath, serviceLogsDir, name) +} + // ServiceOutputDir returns the output directory func (loc LocationManager) ServiceOutputDir() string { return filepath.Join(loc.stackPath, serviceOutputDir) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 11aa3eebda..59920154c2 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -295,11 +295,15 @@ func (r *runner) createServiceOptions(variantName string) servicedeployer.Factor } } -func (r *runner) createAgentInfo() (agentdeployer.AgentInfo, error) { +func (r *runner) createAgentInfo(scenario scenarioTest) (agentdeployer.AgentInfo, error) { var info agentdeployer.AgentInfo + dirPath, err := agentdeployer.CreateServiceLogsDir(r.locationManager, fmt.Sprintf("agent-%s-%s", scenario.pkgManifest.Name, scenario.dataStreamManifest.Name)) + if err != nil { + return agentdeployer.AgentInfo{}, fmt.Errorf("failed to create service logs dir: %w", err) + } info.Name = r.options.TestFolder.Package - info.Logs.Folder.Local = r.locationManager.ServiceLogDir() + info.Logs.Folder.Local = dirPath info.Logs.Folder.Agent = ServiceLogsAgentDir info.Test.RunID = createTestRunID() @@ -720,6 +724,30 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic if err != nil { return nil, fmt.Errorf("failed to find the selected policy_template: %w", err) } + // Setup agent + logger.Debug("setting up agent...") + agentOptions := r.createAgentOptions(scenario) + agentInfo, err := r.createAgentInfo(scenario) + if err != nil { + return nil, err + } + + agentDeployer, err := agentdeployer.Factory(agentOptions) + if err != nil { + return nil, fmt.Errorf("could not create agent runner: %w", err) + } + agentDeployed, err := agentDeployer.SetUp(agentInfo) + if err != nil { + return nil, fmt.Errorf("could not setup agent: %w", err) + } + r.shutdownAgentHandler = func() error { + logger.Debug("tearing down agent...") + if err := agentDeployed.TearDown(); err != nil { + return fmt.Errorf("error tearing down agent: %w", err) + } + + return nil + } // Setup service. logger.Debug("setting up service...") @@ -728,6 +756,7 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic return nil, fmt.Errorf("could not create service runner: %w", err) } + ctxt.Logs.Folder.Local = agentInfo.Logs.Folder.Local if config.Service != "" { ctxt.Name = config.Service } @@ -879,31 +908,6 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic } } - // Setup agent - logger.Debug("setting up agent...") - agentOptions := r.createAgentOptions(scenario) - agentInfo, err := r.createAgentInfo() - if err != nil { - return nil, err - } - - agentDeployer, err := agentdeployer.Factory(agentOptions) - if err != nil { - return nil, fmt.Errorf("could not create agent runner: %w", err) - } - agentDeployed, err := agentDeployer.SetUp(agentInfo) - if err != nil { - return nil, fmt.Errorf("could not setup agent: %w", err) - } - r.shutdownAgentHandler = func() error { - logger.Debug("tearing down agent...") - if err := agentDeployed.TearDown(); err != nil { - return fmt.Errorf("error tearing down agent: %w", err) - } - - return nil - } - var origPolicy kibana.Policy agents, err := checkEnrolledAgents(r.options.KibanaClient, ctxt, agentInfo) if err != nil { @@ -1687,19 +1691,21 @@ func filterAgents(allAgents []kibana.Agent, ctx servicedeployer.ServiceContext, logger.Debugf("filter agents using criteria: NamePrefix=%s", ctx.Agent.Host.NamePrefix) } + // filtered list of agents must contain all agents started by the stack + // they could be assigned the default policy (elastic-agent-managed-ep) or the test policy (ep-test-system-*) var filtered []kibana.Agent for _, agent := range allAgents { if agent.PolicyRevision == 0 { continue // For some reason Kibana doesn't always return a valid policy revision (eventually it will be present and valid) } - if agent.PolicyID != "elastic-agent-managed-ep" { - logger.Debugf("filted agent %q", agent.ID) + if agent.PolicyID == "fleet-server-policy" { + logger.Debugf("filtered agent %q", agent.ID) continue } if agent.Status != "online" { - logger.Debugf("filted agent %q", agent.ID) + logger.Debugf("filtered agent %q", agent.ID) continue } From 1cec83f6e9b9ccc94e81e62697fbafeb4bd9c0b0 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 12 Mar 2024 17:58:45 +0100 Subject: [PATCH 008/114] Remove unused elasticsearch API --- internal/agentdeployer/agent.go | 6 ------ internal/agentdeployer/factory.go | 4 ---- internal/testrunner/runners/system/runner.go | 1 - 3 files changed, 11 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 73ed42f613..2f504da8a9 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -14,7 +14,6 @@ import ( "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/docker" - "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" @@ -41,8 +40,6 @@ type CustomAgentDeployer struct { variant AgentVariant - API *elasticsearch.API - packageName string dataStream string @@ -56,8 +53,6 @@ type CustomAgentDeployerOptions struct { StackVersion string Variant AgentVariant - API *elasticsearch.API - PackageName string DataStream string @@ -73,7 +68,6 @@ func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDep stackVersion: options.StackVersion, packageName: options.PackageName, dataStream: options.DataStream, - API: options.API, variant: options.Variant, runTearDown: options.RunTearDown, runTestsOnly: options.RunTestsOnly, diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go index 4de6ff6fb1..014b024fb4 100644 --- a/internal/agentdeployer/factory.go +++ b/internal/agentdeployer/factory.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" - "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/profile" ) @@ -23,8 +22,6 @@ const ( type FactoryOptions struct { Profile *profile.Profile - API *elasticsearch.API - PackageRootPath string DataStreamRootPath string DevDeployDir string @@ -70,7 +67,6 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { customAgentCfgYMLPath := "" opts := CustomAgentDeployerOptions{ Profile: options.Profile, - API: options.API, DockerComposeFile: customAgentCfgYMLPath, StackVersion: options.StackVersion, PackageName: options.PackageName, diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 59920154c2..ac48825de5 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -266,7 +266,6 @@ func (r *runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, e func (r *runner) createAgentOptions(scenario scenarioTest) agentdeployer.FactoryOptions { return agentdeployer.FactoryOptions{ Profile: r.options.Profile, - API: r.options.API, PackageRootPath: r.options.PackageRootPath, DataStreamRootPath: r.dataStreamPath, DevDeployDir: DevDeployDir, From 107d10252694c7b67ffcc6f490d3375883506392 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 12 Mar 2024 18:30:53 +0100 Subject: [PATCH 009/114] Set different hostnames for agents --- internal/agentdeployer/_static/docker-agent-base.yml | 2 +- internal/agentdeployer/agent.go | 1 + internal/agentdeployer/info.go | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/agentdeployer/_static/docker-agent-base.yml b/internal/agentdeployer/_static/docker-agent-base.yml index 5dcd21bd72..0554f79bb3 100644 --- a/internal/agentdeployer/_static/docker-agent-base.yml +++ b/internal/agentdeployer/_static/docker-agent-base.yml @@ -1,12 +1,12 @@ version: "2.3" services: docker-custom-agent: + hostname: ${AGENT_HOSTNAME} image: "${ELASTIC_AGENT_IMAGE_REF}" healthcheck: test: "elastic-agent status" retries: 180 interval: 1s - hostname: docker-custom-agent environment: - FLEET_ENROLL=1 - FLEET_URL=https://fleet-server:8220 diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 2f504da8a9..f3593e229d 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -107,6 +107,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), fmt.Sprintf("%s=%s", fleetPolicyEnv, defaultAgentPolicyName), + fmt.Sprintf("%s=docker-custom-agent-%s", agentHosnameEnv, d.agentName()), ) configDir, err := d.installDockerfile() diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go index 4a1bdcb2d6..ad0fda6076 100644 --- a/internal/agentdeployer/info.go +++ b/internal/agentdeployer/info.go @@ -9,6 +9,7 @@ const ( serviceLogsDirEnv = "SERVICE_LOGS_DIR" testRunIDEnv = "TEST_RUN_ID" fleetPolicyEnv = "FLEET_TOKEN_POLICY_NAME" + agentHosnameEnv = "AGENT_HOSTNAME" ) // AgentInfo encapsulates context that is both available to a AgentDeployer and From 0d04fe6b0bf7b25f3aa3e322844f5b9632832b1d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 12 Mar 2024 19:14:45 +0100 Subject: [PATCH 010/114] Add interface checks --- internal/servicedeployer/compose.go | 4 ++++ internal/servicedeployer/custom_agent.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/internal/servicedeployer/compose.go b/internal/servicedeployer/compose.go index 175c93dc4c..360a46ef2d 100644 --- a/internal/servicedeployer/compose.go +++ b/internal/servicedeployer/compose.go @@ -48,6 +48,8 @@ type dockerComposeDeployedService struct { env []string } +var _ ServiceDeployer = new(DockerComposeServiceDeployer) + // NewDockerComposeServiceDeployer returns a new instance of a DockerComposeServiceDeployer. func NewDockerComposeServiceDeployer(options DockerComposeServiceDeployerOptions) (*DockerComposeServiceDeployer, error) { return &DockerComposeServiceDeployer{ @@ -254,6 +256,8 @@ func (s *dockerComposeDeployedService) SetContext(ctxt ServiceContext) error { return nil } +var _ DeployedService = new(dockerComposeDeployedService) + func processServiceContainerLogs(p *compose.Project, opts compose.CommandOptions, serviceName string) { content, err := p.Logs(opts) if err != nil { diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index b0a0c0fedc..05f243136d 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -49,6 +49,8 @@ type CustomAgentDeployerOptions struct { RunTestsOnly bool } +var _ ServiceDeployer = new(CustomAgentDeployer) + // NewCustomAgentDeployer returns a new instance of a deployedCustomAgent. func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDeployer, error) { return &CustomAgentDeployer{ From 35d38695492386f7cc39391ce458cdad94cb47e7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 12 Mar 2024 19:15:02 +0100 Subject: [PATCH 011/114] Remove unused function and add interface checks --- internal/agentdeployer/agent.go | 17 ++--------------- internal/agentdeployer/compose.go | 4 ++++ 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index f3593e229d..10aa30aed7 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -6,7 +6,6 @@ package agentdeployer import ( _ "embed" - "encoding/base64" "fmt" "os" "path/filepath" @@ -60,6 +59,8 @@ type CustomAgentDeployerOptions struct { RunTestsOnly bool } +var _ AgentDeployer = new(CustomAgentDeployer) + // NewCustomAgentDeployer returns a new instance of a deployedCustomAgent. func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDeployer, error) { return &CustomAgentDeployer{ @@ -74,20 +75,6 @@ func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDep }, nil } -func readCACertBase64(profile *profile.Profile) (string, error) { - caCertPath, err := stack.FindCACertificate(profile) - if err != nil { - return "", fmt.Errorf("can't locate CA certificate: %w", err) - } - - d, err := os.ReadFile(caCertPath) - if err != nil { - return "", err - } - - return base64.StdEncoding.EncodeToString(d), nil -} - // SetUp sets up the service and returns any relevant information. func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { logger.Debug("setting up service using Docker Compose agent deployer") diff --git a/internal/agentdeployer/compose.go b/internal/agentdeployer/compose.go index 31870b3b68..0fec1de505 100644 --- a/internal/agentdeployer/compose.go +++ b/internal/agentdeployer/compose.go @@ -43,6 +43,8 @@ func NewDockerComposeAgentDeployer(profile *profile.Profile, ymlPaths []string) }, nil } +var _ AgentDeployer = new(DockerComposeAgentDeployer) + // SetUp sets up the service and returns any relevant information. func (d *DockerComposeAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { logger.Debug("setting up agent using Docker Compose agent deployer") @@ -224,6 +226,8 @@ func (s *dockerComposeDeployedAgent) SetContext(ctxt AgentInfo) error { return nil } +var _ DeployedAgent = new(dockerComposeDeployedAgent) + func processAgentContainerLogs(p *compose.Project, opts compose.CommandOptions, agentName string) { content, err := p.Logs(opts) if err != nil { From d7fd5ddffca4542d82e31aee2b172683c57f0763 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 12:09:18 +0100 Subject: [PATCH 012/114] Add kubernetes agent deployer Added maxUnavailable and maxSurge keys into elastic-agent template to allow wait for the pods created by the daemonset. --- .../_static/elastic-agent-managed.yaml.tmpl | 286 ++++++++++++++++++ internal/agentdeployer/factory.go | 57 +++- internal/agentdeployer/kubernetes.go | 209 +++++++++++++ internal/kubectl/kubectl.go | 20 +- internal/kubectl/kubectl_delete.go | 9 + 5 files changed, 564 insertions(+), 17 deletions(-) create mode 100644 internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl create mode 100644 internal/agentdeployer/kubernetes.go diff --git a/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl b/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl new file mode 100644 index 0000000000..1c0726b90b --- /dev/null +++ b/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl @@ -0,0 +1,286 @@ +apiVersion: v1 +kind: Secret +metadata: + name: elastic-package-certs + namespace: kube-system +data: + ca-cert.pem: {{ .caCertPem }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: elastic-agent + namespace: kube-system + labels: + app: elastic-agent +spec: + selector: + matchLabels: + app: elastic-agent + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: elastic-agent + spec: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: elastic-agent + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: elastic-agent + image: {{ .elasticAgentImage }} + env: + - name: FLEET_ENROLL + value: "1" + # The ip:port pair of fleet server + - name: FLEET_URL + value: {{ .fleetURL }} + # If left empty KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed + - name: FLEET_ENROLLMENT_TOKEN + value: "" + - name: FLEET_TOKEN_POLICY_NAME + value: "{{ .elasticAgentTokenPolicyName }}" + - name: KIBANA_HOST + value: {{ .kibanaURL }} + - name: KIBANA_FLEET_USERNAME + value: "elastic" + - name: KIBANA_FLEET_PASSWORD + value: "changeme" + - name: SSL_CERT_DIR + value: "/etc/ssl/certs:/etc/ssl/elastic-package" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + securityContext: + runAsUser: 0 + resources: + limits: + memory: 500Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: elastic-package-ca + mountPath: /etc/ssl/elastic-package + readOnly: true + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: etc-kubernetes + mountPath: /hostfs/etc/kubernetes + - name: var-lib + mountPath: /hostfs/var/lib + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true + - name: passwd + mountPath: /hostfs/etc/passwd + readOnly: true + - name: group + mountPath: /hostfs/etc/group + readOnly: true + - name: etcsysmd + mountPath: /hostfs/etc/systemd + readOnly: true + volumes: + - name: elastic-package-ca + secret: + secretName: elastic-package-certs + - name: proc + hostPath: + path: /proc + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: var-lib + hostPath: + path: /var/lib + - name: passwd + hostPath: + path: /etc/passwd + - name: group + hostPath: + path: /etc/group + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlog + hostPath: + path: /var/log + - name: etcsysmd + hostPath: + path: /etc/systemd +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: elastic-agent +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: ClusterRole + name: elastic-agent + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: kube-system + name: elastic-agent +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: Role + name: elastic-agent + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: elastic-agent-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: Role + name: elastic-agent-kubeadm-config + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: elastic-agent + labels: + k8s-app: elastic-agent +rules: + - apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - services + - configmaps + - serviceaccounts + - persistentvolumes + - persistentvolumeclaims + verbs: ["get", "list", "watch"] + # Enable this rule only if planing to use kubernetes_secrets provider + #- apiGroups: [""] + # resources: + # - secrets + # verbs: ["get"] + - apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + - daemonsets + verbs: ["get", "list", "watch"] + - apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get + - apiGroups: [ "batch" ] + resources: + - jobs + - cronjobs + verbs: [ "get", "list", "watch" ] + # required for apiserver + - nonResourceURLs: + - "/metrics" + verbs: + - get + - apiGroups: ["rbac.authorization.k8s.io"] + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: + - ingressclasses + - ingresses + verbs: ["get", "list", "watch"] + - apiGroups: ["policy"] + resources: + - podsecuritypolicies + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: elastic-agent + # should be the namespace where elastic-agent is running + namespace: kube-system + labels: + k8s-app: elastic-agent +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: elastic-agent-kubeadm-config + namespace: kube-system + labels: + k8s-app: elastic-agent +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elastic-agent + namespace: kube-system + labels: + k8s-app: elastic-agent +--- diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go index 014b024fb4..9036c4ad6b 100644 --- a/internal/agentdeployer/factory.go +++ b/internal/agentdeployer/factory.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" + "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" ) @@ -41,30 +42,42 @@ type FactoryOptions struct { // Factory chooses the appropriate service runner for the given data stream, depending // on service configuration files defined in the package or data stream. func Factory(options FactoryOptions) (AgentDeployer, error) { - // devDeployPath, err := FindDevDeployPath(options) - // if err != nil { - // return nil, fmt.Errorf("can't find \"%s\" directory: %w", options.DevDeployDir, err) - // } + devDeployPath, err := FindDevDeployPath(options) + if err != nil { + return nil, fmt.Errorf("can't find \"%s\" directory: %w", options.DevDeployDir, err) + } - // agentDeployerName, err := findAgentDeployer(devDeployPath) - // if err != nil { - // return nil, fmt.Errorf("can't find any valid agent deployer: %w", err) - // } + agentDeployerName, err := findAgentDeployer(devDeployPath) + if err != nil { + logger.Debugf("Not found any agent deployer, using default one") + agentDeployerName = "agent" + } - // agentDeployerPath := filepath.Join(devDeployPath, agentDeployerName) - agentDeployerName := "agent" + agentDeployerPath := filepath.Join(devDeployPath, agentDeployerName) switch agentDeployerName { + case "default": + if options.Type != TypeTest { + return nil, fmt.Errorf("agent deployer is not supported for type %s", options.Type) + } + opts := CustomAgentDeployerOptions{ + Profile: options.Profile, + DockerComposeFile: "", + StackVersion: options.StackVersion, + PackageName: options.PackageName, + DataStream: options.DataStream, + RunTearDown: options.RunTearDown, + RunTestsOnly: options.RunTestsOnly, + } + return NewCustomAgentDeployer(opts) case "agent": if options.Type != TypeTest { return nil, fmt.Errorf("agent deployer is not supported for type %s", options.Type) } - // FIXME - // customAgentCfgYMLPath := filepath.Join(agentDeployerPath, "custom-agent.yml") - // if _, err := os.Stat(customAgentCfgYMLPath); err != nil { - // return nil, fmt.Errorf("can't find expected file custom-agent.yml: %w", err) - // } - customAgentCfgYMLPath := "" + customAgentCfgYMLPath := filepath.Join(agentDeployerPath, "custom-agent.yml") + if _, err := os.Stat(customAgentCfgYMLPath); err != nil { + return nil, fmt.Errorf("can't find expected file custom-agent.yml: %w", err) + } opts := CustomAgentDeployerOptions{ Profile: options.Profile, DockerComposeFile: customAgentCfgYMLPath, @@ -75,6 +88,18 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { RunTestsOnly: options.RunTestsOnly, } return NewCustomAgentDeployer(opts) + case "k8s": + if _, err := os.Stat(agentDeployerPath); err == nil { + opts := KubernetesAgentDeployerOptions{ + Profile: options.Profile, + DefinitionsDir: agentDeployerPath, + StackVersion: options.StackVersion, + RunSetup: options.RunSetup, + RunTestsOnly: options.RunTestsOnly, + RunTearDown: options.RunTearDown, + } + return NewKubernetesAgentDeployer(opts) + } } return nil, fmt.Errorf("unsupported agent deployer (name: %s)", agentDeployerName) } diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go new file mode 100644 index 0000000000..e5aea8d493 --- /dev/null +++ b/internal/agentdeployer/kubernetes.go @@ -0,0 +1,209 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package agentdeployer + +import ( + "bytes" + _ "embed" + "encoding/base64" + "fmt" + "os" + "strings" + "text/template" + + "github.com/elastic/elastic-package/internal/install" + "github.com/elastic/elastic-package/internal/kind" + "github.com/elastic/elastic-package/internal/kubectl" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/stack" +) + +// KubernetesAgentDeployer is responsible for deploying resources in the Kubernetes cluster. +type KubernetesAgentDeployer struct { + profile *profile.Profile + definitionsDir string + stackVersion string + + runSetup bool + runTestsOnly bool + runTearDown bool +} + +type KubernetesAgentDeployerOptions struct { + Profile *profile.Profile + DefinitionsDir string + StackVersion string + + RunSetup bool + RunTestsOnly bool + RunTearDown bool +} + +type kubernetesDeployedAgent struct { + ctxt AgentInfo + profile *profile.Profile + stackVersion string + + definitionsDir string +} + +func (s kubernetesDeployedAgent) TearDown() error { + elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion) + if err != nil { + return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) + } + err = kubectl.DeleteStdin(elasticAgentManagedYaml) + if err != nil { + return fmt.Errorf("can't uninstall Kubernetes resources (path: %s): %w", s.definitionsDir, err) + } + return nil +} + +func (s kubernetesDeployedAgent) Signal(_ string) error { + return ErrNotSupported +} + +func (s kubernetesDeployedAgent) ExitCode(_ string) (bool, int, error) { + return false, -1, ErrNotSupported +} + +func (s kubernetesDeployedAgent) Context() AgentInfo { + return s.ctxt +} + +func (s *kubernetesDeployedAgent) SetContext(sc AgentInfo) error { + s.ctxt = sc + return nil +} + +var _ DeployedAgent = new(kubernetesDeployedAgent) + +// NewKubernetesAgentDeployer function creates a new instance of KubernetesAgentDeployer. +func NewKubernetesAgentDeployer(opts KubernetesAgentDeployerOptions) (*KubernetesAgentDeployer, error) { + return &KubernetesAgentDeployer{ + profile: opts.Profile, + definitionsDir: opts.DefinitionsDir, + stackVersion: opts.StackVersion, + runSetup: opts.RunSetup, + runTestsOnly: opts.RunTestsOnly, + runTearDown: opts.RunTearDown, + }, nil +} + +// SetUp function links the kind container with elastic-package-stack network, installs Elastic-Agent and optionally +// custom YAML definitions. +func (ksd KubernetesAgentDeployer) SetUp(ctxt AgentInfo) (DeployedAgent, error) { + err := kind.VerifyContext() + if err != nil { + return nil, fmt.Errorf("kind context verification failed: %w", err) + } + + if ksd.runTearDown || ksd.runTestsOnly { + logger.Debug("Skip connect kind to Elastic stack network") + } else { + err = kind.ConnectToElasticStackNetwork(ksd.profile) + if err != nil { + return nil, fmt.Errorf("can't connect control plane to Elastic stack network: %w", err) + } + } + + if ksd.runTearDown || ksd.runTestsOnly { + logger.Debug("Skip install Elastic Agent in cluster") + } else { + err = installElasticAgentInCluster(ksd.profile, ksd.stackVersion) + if err != nil { + return nil, fmt.Errorf("can't install Elastic-Agent in the Kubernetes cluster: %w", err) + } + } + + ctxt.Name = kind.ControlPlaneContainerName + ctxt.Hostname = kind.ControlPlaneContainerName + // kind-control-plane is the name of the kind host where Pod is running since we use hostNetwork setting + // to deploy Agent Pod. Because of this, hostname inside pod will be equal to the name of the k8s host. + ctxt.Agent.Host.NamePrefix = "kind-control-plane" + return &kubernetesDeployedAgent{ + ctxt: ctxt, + definitionsDir: ksd.definitionsDir, + profile: ksd.profile, + stackVersion: ksd.stackVersion, + }, nil +} + +var _ AgentDeployer = new(KubernetesAgentDeployer) + +func installElasticAgentInCluster(profile *profile.Profile, stackVersion string) error { + logger.Debug("install Elastic Agent in the Kubernetes cluster") + + elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion) + if err != nil { + return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) + } + + err = kubectl.ApplyStdin(elasticAgentManagedYaml) + if err != nil { + return fmt.Errorf("can't install Elastic-Agent in Kubernetes cluster: %w", err) + } + + // DEBUG DaemonSet is not ready: kube-system/elastic-agent. 0 out of 1 expected pods have been scheduled + + return nil +} + +//go:embed _static/elastic-agent-managed.yaml.tmpl +var elasticAgentManagedYamlTmpl string + +func getElasticAgentYAML(profile *profile.Profile, stackVersion string) ([]byte, error) { + logger.Debugf("Prepare YAML definition for Elastic Agent running in stack v%s", stackVersion) + + appConfig, err := install.Configuration() + if err != nil { + return nil, fmt.Errorf("can't read application configuration: %w", err) + } + + caCert, err := readCACertBase64(profile) + if err != nil { + return nil, fmt.Errorf("can't read certificate authority file: %w", err) + } + + tmpl := template.Must(template.New("elastic-agent.yml").Parse(elasticAgentManagedYamlTmpl)) + + var elasticAgentYaml bytes.Buffer + err = tmpl.Execute(&elasticAgentYaml, map[string]string{ + "fleetURL": "https://fleet-server:8220", + "kibanaURL": "https://kibana:5601", + "caCertPem": caCert, + "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, + "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion), + }) + if err != nil { + return nil, fmt.Errorf("can't generate elastic agent manifest: %w", err) + } + + return elasticAgentYaml.Bytes(), nil +} + +func readCACertBase64(profile *profile.Profile) (string, error) { + caCertPath, err := stack.FindCACertificate(profile) + if err != nil { + return "", fmt.Errorf("can't locate CA certificate: %w", err) + } + + d, err := os.ReadFile(caCertPath) + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(d), nil +} + +// getTokenPolicyName function returns the policy name for the 8.x Elastic stack. The agent's policy +// is predefined in the Kibana configuration file. The logic is not present in older stacks. +func getTokenPolicyName(stackVersion string) string { + if strings.HasPrefix(stackVersion, "8.") { + return "Elastic-Agent (elastic-package)" + } + return "" +} diff --git a/internal/kubectl/kubectl.go b/internal/kubectl/kubectl.go index f3277e766e..3c13c8388d 100644 --- a/internal/kubectl/kubectl.go +++ b/internal/kubectl/kubectl.go @@ -61,7 +61,7 @@ func modifyKubernetesResources(action string, definitionPaths []string) ([]byte, func applyKubernetesResourcesStdin(input []byte) ([]byte, error) { // create kubectl apply command kubectlCmd := exec.Command("kubectl", "apply", "-f", "-", "-o", "yaml") - //Stdin of kubectl command is the manifest provided + // Stdin of kubectl command is the manifest provided kubectlCmd.Stdin = bytes.NewReader(input) errOutput := new(bytes.Buffer) kubectlCmd.Stderr = errOutput @@ -73,3 +73,21 @@ func applyKubernetesResourcesStdin(input []byte) ([]byte, error) { } return output, nil } + +// deleteKubernetesResourcesStdin deletes a Kubernetes manifest provided as stdin. +// It returns the resources deleted as output and an error +func deleteKubernetesResourcesStdin(input []byte) ([]byte, error) { + // create kubectl apply command + kubectlCmd := exec.Command("kubectl", "delete", "-f", "-") + // Stdin of kubectl command is the manifest provided + kubectlCmd.Stdin = bytes.NewReader(input) + errOutput := new(bytes.Buffer) + kubectlCmd.Stderr = errOutput + + logger.Debugf("run command: %s", kubectlCmd) + output, err := kubectlCmd.Output() + if err != nil { + return nil, fmt.Errorf("kubectl delete failed (stderr=%q): %w", errOutput.String(), err) + } + return output, nil +} diff --git a/internal/kubectl/kubectl_delete.go b/internal/kubectl/kubectl_delete.go index 2116d78cb1..dc999216ce 100644 --- a/internal/kubectl/kubectl_delete.go +++ b/internal/kubectl/kubectl_delete.go @@ -4,8 +4,17 @@ package kubectl +import "github.com/elastic/elastic-package/internal/logger" + // Delete function removes resources from the Kubernetes cluster based on provided definitions. func Delete(definitionsPath []string) error { _, err := modifyKubernetesResources("delete", definitionsPath) return err } + +// DeleteStdin function removes resources from the Kubernetes cluster based on provided definitions. +func DeleteStdin(out []byte) error { + logger.Debugf("Delete Kubernetes stdin") + _, err := deleteKubernetesResourcesStdin(out) + return err +} From 11630da011dc25d29ffc310075c787ad7b074b71 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 12:12:03 +0100 Subject: [PATCH 013/114] Do not install kubernetes agent in servicedeployer --- .../_static/elastic-agent-managed.yaml.tmpl | 281 ------------------ internal/servicedeployer/kubernetes.go | 64 +--- 2 files changed, 2 insertions(+), 343 deletions(-) delete mode 100644 internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl diff --git a/internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl b/internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl deleted file mode 100644 index b3e728ad08..0000000000 --- a/internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl +++ /dev/null @@ -1,281 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: elastic-package-certs - namespace: kube-system -data: - ca-cert.pem: {{ .caCertPem }} ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: elastic-agent - namespace: kube-system - labels: - app: elastic-agent -spec: - selector: - matchLabels: - app: elastic-agent - template: - metadata: - labels: - app: elastic-agent - spec: - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: elastic-agent - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: elastic-agent - image: {{ .elasticAgentImage }} - env: - - name: FLEET_ENROLL - value: "1" - # The ip:port pair of fleet server - - name: FLEET_URL - value: {{ .fleetURL }} - # If left empty KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed - - name: FLEET_ENROLLMENT_TOKEN - value: "" - - name: FLEET_TOKEN_POLICY_NAME - value: "{{ .elasticAgentTokenPolicyName }}" - - name: KIBANA_HOST - value: {{ .kibanaURL }} - - name: KIBANA_FLEET_USERNAME - value: "elastic" - - name: KIBANA_FLEET_PASSWORD - value: "changeme" - - name: SSL_CERT_DIR - value: "/etc/ssl/certs:/etc/ssl/elastic-package" - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - securityContext: - runAsUser: 0 - resources: - limits: - memory: 500Mi - requests: - cpu: 100m - memory: 200Mi - volumeMounts: - - name: elastic-package-ca - mountPath: /etc/ssl/elastic-package - readOnly: true - - name: proc - mountPath: /hostfs/proc - readOnly: true - - name: etc-kubernetes - mountPath: /hostfs/etc/kubernetes - - name: var-lib - mountPath: /hostfs/var/lib - readOnly: true - - name: cgroup - mountPath: /hostfs/sys/fs/cgroup - readOnly: true - - name: varlibdockercontainers - mountPath: /var/lib/docker/containers - readOnly: true - - name: varlog - mountPath: /var/log - readOnly: true - - name: passwd - mountPath: /hostfs/etc/passwd - readOnly: true - - name: group - mountPath: /hostfs/etc/group - readOnly: true - - name: etcsysmd - mountPath: /hostfs/etc/systemd - readOnly: true - volumes: - - name: elastic-package-ca - secret: - secretName: elastic-package-certs - - name: proc - hostPath: - path: /proc - - name: etc-kubernetes - hostPath: - path: /etc/kubernetes - - name: var-lib - hostPath: - path: /var/lib - - name: passwd - hostPath: - path: /etc/passwd - - name: group - hostPath: - path: /etc/group - - name: cgroup - hostPath: - path: /sys/fs/cgroup - - name: varlibdockercontainers - hostPath: - path: /var/lib/docker/containers - - name: varlog - hostPath: - path: /var/log - - name: etcsysmd - hostPath: - path: /etc/systemd ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: elastic-agent -subjects: - - kind: ServiceAccount - name: elastic-agent - namespace: kube-system -roleRef: - kind: ClusterRole - name: elastic-agent - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - namespace: kube-system - name: elastic-agent -subjects: - - kind: ServiceAccount - name: elastic-agent - namespace: kube-system -roleRef: - kind: Role - name: elastic-agent - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: elastic-agent-kubeadm-config - namespace: kube-system -subjects: - - kind: ServiceAccount - name: elastic-agent - namespace: kube-system -roleRef: - kind: Role - name: elastic-agent-kubeadm-config - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: elastic-agent - labels: - k8s-app: elastic-agent -rules: - - apiGroups: [""] - resources: - - nodes - - namespaces - - events - - pods - - services - - configmaps - - serviceaccounts - - persistentvolumes - - persistentvolumeclaims - verbs: ["get", "list", "watch"] - # Enable this rule only if planing to use kubernetes_secrets provider - #- apiGroups: [""] - # resources: - # - secrets - # verbs: ["get"] - - apiGroups: ["extensions"] - resources: - - replicasets - verbs: ["get", "list", "watch"] - - apiGroups: ["apps"] - resources: - - statefulsets - - deployments - - replicasets - - daemonsets - verbs: ["get", "list", "watch"] - - apiGroups: - - "" - resources: - - nodes/stats - verbs: - - get - - apiGroups: [ "batch" ] - resources: - - jobs - - cronjobs - verbs: [ "get", "list", "watch" ] - # required for apiserver - - nonResourceURLs: - - "/metrics" - verbs: - - get - - apiGroups: ["rbac.authorization.k8s.io"] - resources: - - clusterrolebindings - - clusterroles - - rolebindings - - roles - verbs: ["get", "list", "watch"] - - apiGroups: ["networking.k8s.io"] - resources: - - ingressclasses - - ingresses - verbs: ["get", "list", "watch"] - - apiGroups: ["policy"] - resources: - - podsecuritypolicies - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: - - storageclasses - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: elastic-agent - # should be the namespace where elastic-agent is running - namespace: kube-system - labels: - k8s-app: elastic-agent -rules: - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: ["get", "create", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: elastic-agent-kubeadm-config - namespace: kube-system - labels: - k8s-app: elastic-agent -rules: - - apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubeadm-config - verbs: ["get"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: elastic-agent - namespace: kube-system - labels: - k8s-app: elastic-agent ---- diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index 1a090712f8..39a6f4719b 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -5,16 +5,13 @@ package servicedeployer import ( - "bytes" _ "embed" "encoding/base64" "fmt" "os" "path/filepath" "strings" - "text/template" - "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kind" "github.com/elastic/elastic-package/internal/kubectl" "github.com/elastic/elastic-package/internal/logger" @@ -100,8 +97,8 @@ func NewKubernetesServiceDeployer(opts KubernetesServiceDeployerOptions) (*Kuber }, nil } -// SetUp function links the kind container with elastic-package-stack network, installs Elastic-Agent and optionally -// custom YAML definitions. +// SetUp function links the kind container with elastic-package-stack network, installs +// custom YAML definitions if any. func (ksd KubernetesServiceDeployer) SetUp(ctxt ServiceContext) (DeployedService, error) { err := kind.VerifyContext() if err != nil { @@ -117,15 +114,6 @@ func (ksd KubernetesServiceDeployer) SetUp(ctxt ServiceContext) (DeployedService } } - if ksd.runTearDown || ksd.runTestsOnly { - logger.Debug("Skip install Elastic Agent in cluster") - } else { - err = installElasticAgentInCluster(ksd.profile, ksd.stackVersion) - if err != nil { - return nil, fmt.Errorf("can't install Elastic-Agent in the Kubernetes cluster: %w", err) - } - } - if !ksd.runTearDown { err = ksd.installCustomDefinitions() if err != nil { @@ -177,54 +165,6 @@ func findKubernetesDefinitions(definitionsDir string) ([]string, error) { return definitionPaths, nil } -func installElasticAgentInCluster(profile *profile.Profile, stackVersion string) error { - logger.Debug("install Elastic Agent in the Kubernetes cluster") - - elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion) - if err != nil { - return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) - } - - err = kubectl.ApplyStdin(elasticAgentManagedYaml) - if err != nil { - return fmt.Errorf("can't install Elastic-Agent in Kubernetes cluster: %w", err) - } - return nil -} - -//go:embed _static/elastic-agent-managed.yaml.tmpl -var elasticAgentManagedYamlTmpl string - -func getElasticAgentYAML(profile *profile.Profile, stackVersion string) ([]byte, error) { - logger.Debugf("Prepare YAML definition for Elastic Agent running in stack v%s", stackVersion) - - appConfig, err := install.Configuration() - if err != nil { - return nil, fmt.Errorf("can't read application configuration: %w", err) - } - - caCert, err := readCACertBase64(profile) - if err != nil { - return nil, fmt.Errorf("can't read certificate authority file: %w", err) - } - - tmpl := template.Must(template.New("elastic-agent.yml").Parse(elasticAgentManagedYamlTmpl)) - - var elasticAgentYaml bytes.Buffer - err = tmpl.Execute(&elasticAgentYaml, map[string]string{ - "fleetURL": "https://fleet-server:8220", - "kibanaURL": "https://kibana:5601", - "caCertPem": caCert, - "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, - "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion), - }) - if err != nil { - return nil, fmt.Errorf("can't generate elastic agent manifest: %w", err) - } - - return elasticAgentYaml.Bytes(), nil -} - func readCACertBase64(profile *profile.Profile) (string, error) { caCertPath, err := stack.FindCACertificate(profile) if err != nil { From 432ca625dd79b2150eda772e27c579ba474bcf05 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 12:14:32 +0100 Subject: [PATCH 014/114] Wait for agent hosts set in agentInfo --- internal/testrunner/runners/system/runner.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index ac48825de5..1ba10138f1 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1261,7 +1261,7 @@ func checkEnrolledAgents(client *kibana.Client, ctxt servicedeployer.ServiceCont return false, fmt.Errorf("could not list agents: %w", err) } - agents = filterAgents(allAgents, ctxt, agentInfo) + agents = filterAgents(allAgents, agentInfo) logger.Debugf("found %d enrolled agent(s)", len(agents)) if len(agents) == 0 { return false, nil // selected agents are unavailable yet @@ -1685,9 +1685,9 @@ func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) } } -func filterAgents(allAgents []kibana.Agent, ctx servicedeployer.ServiceContext, agentInfo agentdeployer.AgentInfo) []kibana.Agent { - if ctx.Agent.Host.NamePrefix != "" { - logger.Debugf("filter agents using criteria: NamePrefix=%s", ctx.Agent.Host.NamePrefix) +func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo) []kibana.Agent { + if agentInfo.Agent.Host.NamePrefix != "" { + logger.Debugf("filter agents using criteria: NamePrefix=%s", agentInfo.Agent.Host.NamePrefix) } // filtered list of agents must contain all agents started by the stack @@ -1708,10 +1708,9 @@ func filterAgents(allAgents []kibana.Agent, ctx servicedeployer.ServiceContext, continue } - serviceHasPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, ctx.Agent.Host.NamePrefix) - agentHasPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) + hasAgentPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) - if ctx.Agent.Host.NamePrefix != "" && !serviceHasPrefix && !agentHasPrefix { + if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix { logger.Debugf("filtered agent %q", agent.ID) continue } From 49d296f65ddaf5964fd5a2df509227880b0d53e6 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 13:11:18 +0100 Subject: [PATCH 015/114] Add checks to get the specific agent --- .../_static/elastic-agent-managed.yaml.tmpl | 7 ++++++ internal/kibana/agents.go | 3 ++- internal/testrunner/runners/system/runner.go | 22 +++++++++++++------ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl b/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl index 1c0726b90b..c2ce38713a 100644 --- a/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl +++ b/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl @@ -71,6 +71,13 @@ spec: requests: cpu: 100m memory: 200Mi + livenessProbe: + exec: + command: + - elastic-agent + - status + initialDelaySeconds: 10 + periodSeconds: 10 volumeMounts: - name: elastic-package-ca mountPath: /etc/ssl/elastic-package diff --git a/internal/kibana/agents.go b/internal/kibana/agents.go index acc169eb8b..521c0fe76f 100644 --- a/internal/kibana/agents.go +++ b/internal/kibana/agents.go @@ -35,7 +35,8 @@ type Agent struct { } `json:"agent"` } `json:"elastic"` } `json:"local_metadata"` - Status string `json:"status"` + Status string `json:"status"` + EnrolledAt time.Time `json:"enrolled_at"` } // String method returns string representation of an agent. diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 1ba10138f1..7f47fa9543 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -731,6 +731,7 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic return nil, err } + enrollingTime := time.Now() agentDeployer, err := agentdeployer.Factory(agentOptions) if err != nil { return nil, fmt.Errorf("could not create agent runner: %w", err) @@ -908,7 +909,7 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic } var origPolicy kibana.Policy - agents, err := checkEnrolledAgents(r.options.KibanaClient, ctxt, agentInfo) + agents, err := checkEnrolledAgents(r.options.KibanaClient, agentInfo, enrollingTime) if err != nil { return nil, fmt.Errorf("can't check enrolled agents: %w", err) } @@ -1249,8 +1250,9 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return r.validateTestScenario(result, scenario, config, serviceOptions) } -func checkEnrolledAgents(client *kibana.Client, ctxt servicedeployer.ServiceContext, agentInfo agentdeployer.AgentInfo) ([]kibana.Agent, error) { +func checkEnrolledAgents(client *kibana.Client, agentInfo agentdeployer.AgentInfo, threshold time.Time) ([]kibana.Agent, error) { var agents []kibana.Agent + enrolled, err := waitUntilTrue(func() (bool, error) { if signal.SIGINT() { return false, errors.New("SIGINT: cancel checking enrolled agents") @@ -1261,7 +1263,7 @@ func checkEnrolledAgents(client *kibana.Client, ctxt servicedeployer.ServiceCont return false, fmt.Errorf("could not list agents: %w", err) } - agents = filterAgents(allAgents, agentInfo) + agents = filterAgents(allAgents, agentInfo, threshold) logger.Debugf("found %d enrolled agent(s)", len(agents)) if len(agents) == 0 { return false, nil // selected agents are unavailable yet @@ -1685,7 +1687,7 @@ func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) } } -func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo) []kibana.Agent { +func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, threshold time.Time) []kibana.Agent { if agentInfo.Agent.Host.NamePrefix != "" { logger.Debugf("filter agents using criteria: NamePrefix=%s", agentInfo.Agent.Host.NamePrefix) } @@ -1699,19 +1701,25 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo) [ } if agent.PolicyID == "fleet-server-policy" { - logger.Debugf("filtered agent %q", agent.ID) + logger.Debugf("filtered agent %q", agent.ID) // TODO: remove continue } if agent.Status != "online" { - logger.Debugf("filtered agent %q", agent.ID) + logger.Debugf("filtered agent (not online) %q", agent.ID) // TODO: remove + continue + } + + if agent.EnrolledAt.Before(threshold) { + logger.Debugf("filtered agent (enrolling time) %q", agent.ID) // TODO: remove continue } + // FIXME: check for package and data stream name too ? hasAgentPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix { - logger.Debugf("filtered agent %q", agent.ID) + logger.Debugf("filtered agent %q", agent.ID) // TODO: remove continue } filtered = append(filtered, agent) From 2a96e939f874e557d3b58837c008ac5a957e3486 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 13:17:50 +0100 Subject: [PATCH 016/114] Set default agent deployer when docker folder is defined --- internal/agentdeployer/factory.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go index 9036c4ad6b..20dd174bcd 100644 --- a/internal/agentdeployer/factory.go +++ b/internal/agentdeployer/factory.go @@ -52,6 +52,11 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { logger.Debugf("Not found any agent deployer, using default one") agentDeployerName = "agent" } + // if package defines `_dev/deploy/docker` folder to start their services, it should be + // using the default agent deployer` + if agentDeployerName == "docker" { + agentDeployerName = "default" + } agentDeployerPath := filepath.Join(devDeployPath, agentDeployerName) From 0719a325a313a0618baeadcdd76450fa68bf6dd5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 15:07:11 +0100 Subject: [PATCH 017/114] Add note about possible error if tests are parallelized --- internal/testrunner/runners/system/runner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 7f47fa9543..2a8424ff86 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1716,6 +1716,7 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t } // FIXME: check for package and data stream name too ? + // Current verson could be returning an unexpected agent if tests are parallelized hasAgentPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix { From 61fbc4e3578f4c23a7ff3d1d74802fa696d8da05 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 15:07:36 +0100 Subject: [PATCH 018/114] Consider terraform scenario as default one too (for agents) --- internal/agentdeployer/factory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go index 20dd174bcd..9189f9fac0 100644 --- a/internal/agentdeployer/factory.go +++ b/internal/agentdeployer/factory.go @@ -54,7 +54,7 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { } // if package defines `_dev/deploy/docker` folder to start their services, it should be // using the default agent deployer` - if agentDeployerName == "docker" { + if agentDeployerName == "docker" || agentDeployerName == "tf" { agentDeployerName = "default" } From c49a10fbcddf8b2cacc7a2afe2ecd7de5e3220fd Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 16:41:21 +0100 Subject: [PATCH 019/114] Add variants --- internal/agentdeployer/factory.go | 11 +++++++++++ internal/testrunner/runners/system/runner.go | 12 ++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go index 9189f9fac0..421c14ff07 100644 --- a/internal/agentdeployer/factory.go +++ b/internal/agentdeployer/factory.go @@ -65,9 +65,14 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { if options.Type != TypeTest { return nil, fmt.Errorf("agent deployer is not supported for type %s", options.Type) } + sv, err := useAgentVariant(devDeployPath, options.Variant) + if err != nil { + return nil, fmt.Errorf("can't use service variant: %w", err) + } opts := CustomAgentDeployerOptions{ Profile: options.Profile, DockerComposeFile: "", + Variant: sv, StackVersion: options.StackVersion, PackageName: options.PackageName, DataStream: options.DataStream, @@ -76,6 +81,7 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { } return NewCustomAgentDeployer(opts) case "agent": + // FIXME: should this be just carried out by service deployer? if options.Type != TypeTest { return nil, fmt.Errorf("agent deployer is not supported for type %s", options.Type) } @@ -83,10 +89,15 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { if _, err := os.Stat(customAgentCfgYMLPath); err != nil { return nil, fmt.Errorf("can't find expected file custom-agent.yml: %w", err) } + sv, err := useAgentVariant(devDeployPath, options.Variant) + if err != nil { + return nil, fmt.Errorf("can't use service variant: %w", err) + } opts := CustomAgentDeployerOptions{ Profile: options.Profile, DockerComposeFile: customAgentCfgYMLPath, StackVersion: options.StackVersion, + Variant: sv, PackageName: options.PackageName, DataStream: options.DataStream, RunTearDown: options.RunTearDown, diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 2a8424ff86..1b17e137ef 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -731,6 +731,8 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic return nil, err } + // FIXME: this should be stored in service state or not used in + // --no-provision and --tear-down stages enrollingTime := time.Now() agentDeployer, err := agentdeployer.Factory(agentOptions) if err != nil { @@ -1700,8 +1702,10 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t continue // For some reason Kibana doesn't always return a valid policy revision (eventually it will be present and valid) } + // It cannot filtered by "elastic-agent-managed-ep" , since this is the default + // policy assigned to the agents when they first enroll if agent.PolicyID == "fleet-server-policy" { - logger.Debugf("filtered agent %q", agent.ID) // TODO: remove + logger.Debugf("filtered agent (policy id) %q", agent.ID) // TODO: remove continue } @@ -1718,9 +1722,13 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t // FIXME: check for package and data stream name too ? // Current verson could be returning an unexpected agent if tests are parallelized hasAgentPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) + // hasServicePrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, ctxt.Agent.Host.NamePrefix) + // TODO: required for custom agents triggered from service deployers + + // if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix && !hasServicePrefix { if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix { - logger.Debugf("filtered agent %q", agent.ID) // TODO: remove + logger.Debugf("filtered agent (prefix) %q", agent.ID) // TODO: remove continue } filtered = append(filtered, agent) From 68887843ec54374490ccf3624981e307911af324 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 16:45:58 +0100 Subject: [PATCH 020/114] Add note for running with stages --- internal/testrunner/runners/system/runner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 1b17e137ef..8fcab874ab 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -910,6 +910,7 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic } } + // FIXME: running per stages does not work when multiple agents are created var origPolicy kibana.Policy agents, err := checkEnrolledAgents(r.options.KibanaClient, agentInfo, enrollingTime) if err != nil { From a7043d50d613dd7b644cc50805c6737de1ada612 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 17:11:40 +0100 Subject: [PATCH 021/114] Keep managing previous custom-agent using servicedeployer --- internal/agentdeployer/factory.go | 48 ++++++++++---------- internal/servicedeployer/factory.go | 1 + internal/testrunner/runners/system/runner.go | 12 +++-- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go index 421c14ff07..f52dcb662f 100644 --- a/internal/agentdeployer/factory.go +++ b/internal/agentdeployer/factory.go @@ -50,7 +50,7 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { agentDeployerName, err := findAgentDeployer(devDeployPath) if err != nil { logger.Debugf("Not found any agent deployer, using default one") - agentDeployerName = "agent" + agentDeployerName = "default" } // if package defines `_dev/deploy/docker` folder to start their services, it should be // using the default agent deployer` @@ -82,28 +82,30 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { return NewCustomAgentDeployer(opts) case "agent": // FIXME: should this be just carried out by service deployer? - if options.Type != TypeTest { - return nil, fmt.Errorf("agent deployer is not supported for type %s", options.Type) - } - customAgentCfgYMLPath := filepath.Join(agentDeployerPath, "custom-agent.yml") - if _, err := os.Stat(customAgentCfgYMLPath); err != nil { - return nil, fmt.Errorf("can't find expected file custom-agent.yml: %w", err) - } - sv, err := useAgentVariant(devDeployPath, options.Variant) - if err != nil { - return nil, fmt.Errorf("can't use service variant: %w", err) - } - opts := CustomAgentDeployerOptions{ - Profile: options.Profile, - DockerComposeFile: customAgentCfgYMLPath, - StackVersion: options.StackVersion, - Variant: sv, - PackageName: options.PackageName, - DataStream: options.DataStream, - RunTearDown: options.RunTearDown, - RunTestsOnly: options.RunTestsOnly, - } - return NewCustomAgentDeployer(opts) + // FIXME: this docker-compose scenario contains both agent and service + return nil, nil + // if options.Type != TypeTest { + // return nil, fmt.Errorf("agent deployer is not supported for type %s", options.Type) + // } + // customAgentCfgYMLPath := filepath.Join(agentDeployerPath, "custom-agent.yml") + // if _, err := os.Stat(customAgentCfgYMLPath); err != nil { + // return nil, fmt.Errorf("can't find expected file custom-agent.yml: %w", err) + // } + // sv, err := useAgentVariant(devDeployPath, options.Variant) + // if err != nil { + // return nil, fmt.Errorf("can't use service variant: %w", err) + // } + // opts := CustomAgentDeployerOptions{ + // Profile: options.Profile, + // DockerComposeFile: customAgentCfgYMLPath, + // StackVersion: options.StackVersion, + // Variant: sv, + // PackageName: options.PackageName, + // DataStream: options.DataStream, + // RunTearDown: options.RunTearDown, + // RunTestsOnly: options.RunTestsOnly, + // } + // return NewCustomAgentDeployer(opts) case "k8s": if _, err := os.Stat(agentDeployerPath); err == nil { opts := KubernetesAgentDeployerOptions{ diff --git a/internal/servicedeployer/factory.go b/internal/servicedeployer/factory.go index 4fe8680901..0a34f5a758 100644 --- a/internal/servicedeployer/factory.go +++ b/internal/servicedeployer/factory.go @@ -80,6 +80,7 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { return NewDockerComposeServiceDeployer(opts) } case "agent": + // FIXME: This docker-compose scenario contains also the definition of the elastic-agent container if options.Type != TypeTest { return nil, fmt.Errorf("agent deployer is not supported for type %s", options.Type) } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 8fcab874ab..ec450d1868 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -738,11 +738,17 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic if err != nil { return nil, fmt.Errorf("could not create agent runner: %w", err) } - agentDeployed, err := agentDeployer.SetUp(agentInfo) - if err != nil { - return nil, fmt.Errorf("could not setup agent: %w", err) + var agentDeployed agentdeployer.DeployedAgent + if agentDeployer != nil { + agentDeployed, err = agentDeployer.SetUp(agentInfo) + if err != nil { + return nil, fmt.Errorf("could not setup agent: %w", err) + } } r.shutdownAgentHandler = func() error { + if agentDeployer == nil { + return nil + } logger.Debug("tearing down agent...") if err := agentDeployed.TearDown(); err != nil { return fmt.Errorf("error tearing down agent: %w", err) From fa5cca00df71d3ca64d92d406b70efb8c64229ee Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 18:10:41 +0100 Subject: [PATCH 022/114] Skip temporarily some packages --- .buildkite/pipeline.trigger.integration.tests.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index afd61045e4..7c292a96b4 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -3,6 +3,9 @@ # exit immediately on failure, or if an undefined variable is used set -eu +echoerr() { + echo "$@" 1>&2 +} # begin the pipeline.yml file echo "steps:" @@ -73,6 +76,10 @@ done pushd test/packages/parallel > /dev/null for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do package_name=$(basename "${package}") + if [[ "$package_name" == "aws" || "$package_name" == "aws_logs" || "$package_name" == "gcp" ]] ; then + echoerr "Skip temporarily ${package_name}" + continue + fi echo " - label: \":go: Running integration test: ${package_name}\"" echo " key: \"integration-parallel-${package_name}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" From 7623c61cc775bf853a51fdc87c14718916481845 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 18:17:40 +0100 Subject: [PATCH 023/114] Update hostname in custom-agent (service deployer) --- .../_static/docker-custom-agent-base.yml | 2 +- internal/servicedeployer/context.go | 1 + internal/servicedeployer/custom_agent.go | 17 +++++++++++++++++ internal/servicedeployer/factory.go | 15 +++++++++++++-- internal/testrunner/runners/system/runner.go | 5 +++++ 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/internal/servicedeployer/_static/docker-custom-agent-base.yml b/internal/servicedeployer/_static/docker-custom-agent-base.yml index 610ef273df..039bb49e3e 100644 --- a/internal/servicedeployer/_static/docker-custom-agent-base.yml +++ b/internal/servicedeployer/_static/docker-custom-agent-base.yml @@ -6,7 +6,7 @@ services: test: "elastic-agent status" retries: 180 interval: 1s - hostname: docker-custom-agent + hostname: ${AGENT_HOSTNAME} environment: - FLEET_ENROLL=1 - FLEET_URL=https://fleet-server:8220 diff --git a/internal/servicedeployer/context.go b/internal/servicedeployer/context.go index 8ef382cb60..c01e9dacef 100644 --- a/internal/servicedeployer/context.go +++ b/internal/servicedeployer/context.go @@ -8,6 +8,7 @@ const ( localCACertEnv = "LOCAL_CA_CERT" serviceLogsDirEnv = "SERVICE_LOGS_DIR" testRunIDEnv = "TEST_RUN_ID" + agentHosnameEnv = "AGENT_HOSTNAME" ) // ServiceContext encapsulates context that is both available to a ServiceDeployer and diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index 05f243136d..06e48c75da 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -35,6 +35,9 @@ type CustomAgentDeployer struct { profile *profile.Profile dockerComposeFile string stackVersion string + variant ServiceVariant + packageName string + dataStream string runTearDown bool runTestsOnly bool @@ -44,6 +47,9 @@ type CustomAgentDeployerOptions struct { Profile *profile.Profile DockerComposeFile string StackVersion string + Variant ServiceVariant + PackageName string + DataStream string RunTearDown bool RunTestsOnly bool @@ -57,6 +63,9 @@ func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDep profile: options.Profile, dockerComposeFile: options.DockerComposeFile, stackVersion: options.StackVersion, + variant: options.Variant, + packageName: options.PackageName, + dataStream: options.DataStream, runTearDown: options.RunTearDown, runTestsOnly: options.RunTestsOnly, }, nil @@ -80,6 +89,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err appConfig.StackImageRefs(d.stackVersion).AsEnv(), fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), + fmt.Sprintf("%s=%s-%s", agentHosnameEnv, dockerCustomAgentName, d.agentName()), ) configDir, err := d.installDockerfile() @@ -183,6 +193,13 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err return &service, nil } +func (d *CustomAgentDeployer) agentName() string { + if d.variant.Name != "" { + return fmt.Sprintf("%s-%s-%s", d.packageName, d.variant.Name, d.dataStream) + } + return fmt.Sprintf("%s-%s", d.packageName, d.dataStream) +} + // installDockerfile creates the files needed to run the custom elastic agent and returns // the directory with these files. func (d *CustomAgentDeployer) installDockerfile() (string, error) { diff --git a/internal/servicedeployer/factory.go b/internal/servicedeployer/factory.go index 0a34f5a758..3b336798cd 100644 --- a/internal/servicedeployer/factory.go +++ b/internal/servicedeployer/factory.go @@ -28,6 +28,9 @@ type FactoryOptions struct { Type string StackVersion string + PackageName string + DataStream string + Variant string RunTearDown bool @@ -88,12 +91,20 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { if _, err := os.Stat(customAgentCfgYMLPath); err != nil { return nil, fmt.Errorf("can't find expected file custom-agent.yml: %w", err) } + sv, err := useServiceVariant(devDeployPath, options.Variant) + if err != nil { + return nil, fmt.Errorf("can't use service variant: %w", err) + } opts := CustomAgentDeployerOptions{ Profile: options.Profile, DockerComposeFile: customAgentCfgYMLPath, StackVersion: options.StackVersion, - RunTearDown: options.RunTearDown, - RunTestsOnly: options.RunTestsOnly, + Variant: sv, + PackageName: options.PackageName, + DataStream: options.DataStream, + + RunTearDown: options.RunTearDown, + RunTestsOnly: options.RunTestsOnly, } return NewCustomAgentDeployer(opts) case "tf": diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index ec450d1868..1509c0365a 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -288,6 +288,8 @@ func (r *runner) createServiceOptions(variantName string) servicedeployer.Factor Variant: variantName, Type: servicedeployer.TypeTest, StackVersion: r.stackVersion.Version(), + PackageName: "", // to be filled in prepareScenario + DataStream: "", // to be filled in prepareScenario RunTearDown: r.options.RunTearDown, RunTestsOnly: r.options.RunTestsOnly, RunSetup: r.options.RunSetup, @@ -759,6 +761,9 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic // Setup service. logger.Debug("setting up service...") + serviceOptions.PackageName = scenario.pkgManifest.Name + serviceOptions.DataStream = scenario.dataStreamManifest.Name + serviceDeployer, err := servicedeployer.Factory(serviceOptions) if err != nil { return nil, fmt.Errorf("could not create service runner: %w", err) From fb6f13e439fd44d759f81f33911fb3bcc09cc908 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 18:30:04 +0100 Subject: [PATCH 024/114] Remove unused functions from servicedeployer --- internal/servicedeployer/kubernetes.go | 27 -------------------------- 1 file changed, 27 deletions(-) diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index 39a6f4719b..55a749db35 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -6,17 +6,13 @@ package servicedeployer import ( _ "embed" - "encoding/base64" "fmt" - "os" "path/filepath" - "strings" "github.com/elastic/elastic-package/internal/kind" "github.com/elastic/elastic-package/internal/kubectl" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" - "github.com/elastic/elastic-package/internal/stack" ) // KubernetesServiceDeployer is responsible for deploying resources in the Kubernetes cluster. @@ -164,26 +160,3 @@ func findKubernetesDefinitions(definitionsDir string) ([]string, error) { definitionPaths = append(definitionPaths, files...) return definitionPaths, nil } - -func readCACertBase64(profile *profile.Profile) (string, error) { - caCertPath, err := stack.FindCACertificate(profile) - if err != nil { - return "", fmt.Errorf("can't locate CA certificate: %w", err) - } - - d, err := os.ReadFile(caCertPath) - if err != nil { - return "", err - } - - return base64.StdEncoding.EncodeToString(d), nil -} - -// getTokenPolicyName function returns the policy name for the 8.x Elastic stack. The agent's policy -// is predefined in the Kibana configuration file. The logic is not present in older stacks. -func getTokenPolicyName(stackVersion string) string { - if strings.HasPrefix(stackVersion, "8.") { - return "Elastic-Agent (elastic-package)" - } - return "" -} From 50ad13dc66e18af7ee094a0be38d5aed74349da3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 13 Mar 2024 19:46:10 +0100 Subject: [PATCH 025/114] Check datastream values for agent name --- internal/agentdeployer/agent.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 10aa30aed7..9c04a1cf2c 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -208,23 +208,27 @@ func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { } func (d *CustomAgentDeployer) agentName() string { + name := d.packageName if d.variant.Name != "" { - return fmt.Sprintf("%s-%s-%s", d.packageName, d.variant.Name, d.dataStream) + name = fmt.Sprintf("%s-%s", name, d.variant.Name) } - return fmt.Sprintf("%s-%s", d.packageName, d.dataStream) + if d.dataStream != "" && d.dataStream != "." { + name = fmt.Sprintf("%s-%s", name, d.dataStream) + } + return name } // installDockerfile creates the files needed to run the custom elastic agent and returns // the directory with these files. func (d *CustomAgentDeployer) installDockerfile() (string, error) { customAgentDir := filepath.Join(d.profile.ProfilePath, fmt.Sprintf("agent-%s", d.agentName())) - err := os.MkdirAll(customAgentDir, 0o755) + err := os.MkdirAll(customAgentDir, 0755) if err != nil { return "", fmt.Errorf("failed to create directory for custom agent files: %w", err) } customAgentDockerfile := filepath.Join(customAgentDir, dockerCustomAgentDockerCompose) - err = os.WriteFile(customAgentDockerfile, dockerAgentDockerComposeContent, 0o644) + err = os.WriteFile(customAgentDockerfile, dockerAgentDockerComposeContent, 0644) if err != nil { return "", fmt.Errorf("failed to create docker compose file for custom agent: %w", err) } From 81b502322ae76adea0b7f45c0326374b908b215c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 14 Mar 2024 10:18:43 +0100 Subject: [PATCH 026/114] Add debug message --- internal/testrunner/runners/system/runner.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 1509c0365a..3127ea4df3 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1053,6 +1053,8 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic if !passed { return nil, testrunner.ErrTestCaseFailed{Reason: fmt.Sprintf("could not find hits in %s data stream", scenario.dataStream)} + } else { + logger.Debugf(">>>>> Final hits found: %d", hits.size()) // TODO: to remove } logger.Debugf("check whether or not synthetics is enabled (component template %s)...", componentTemplatePackage) From e35c1b5b84b37e094fc156865e5b22d04b5bd93b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 14 Mar 2024 12:26:16 +0100 Subject: [PATCH 027/114] Use docker custom names for elastic-agent in tests/configuration --- .../hit_count_assertion/_dev/deploy/docker/docker-compose.yml | 2 +- .../parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml | 4 ++-- .../ti_anomali/_dev/deploy/docker/files/test-intel.ndjson | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml b/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml index f2b41d4d26..63cb1aa00e 100644 --- a/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml @@ -4,4 +4,4 @@ services: image: akroh/stream:v0.0.1 volumes: - ./logs:/logs:ro - command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9999 -p=tcp /logs/generated.log + command: log --start-signal=SIGHUP --delay=5s --addr docker-custom-agent-hit_count_assertion-test:9999 -p=tcp /logs/generated.log diff --git a/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml b/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml index ba5c31a792..59d6f29f7d 100644 --- a/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml @@ -18,7 +18,7 @@ services: - ./sample_logs:/sample_logs:ro environment: - STREAM_PROTOCOL=webhook - - STREAM_ADDR=http://elastic-agent:9080/ + - STREAM_ADDR=http://docker-custom-agent-ti_anomali-threatstream:9080/ command: log --webhook-content-type application/x-ndjson --start-signal=SIGHUP --delay=5s /sample_logs/test-threatstream-ndjson.log threatstream-webhook-https: image: docker.elastic.co/observability/stream:v0.6.1 @@ -27,7 +27,7 @@ services: environment: - STREAM_PROTOCOL=webhook - STREAM_INSECURE=true - - STREAM_ADDR=https://elastic-agent:7443/ + - STREAM_ADDR=https://docker-custom-agent-ti_anomali-threatstream:7443/ command: log --webhook-content-type application/x-ndjson --start-signal=SIGHUP --delay=5s /sample_logs/test-threatstream-ndjson.log threatstream-integrator-test: image: docker.io/adrisr/filebeat-anomali-integrator-test:latest diff --git a/test/packages/parallel/ti_anomali/_dev/deploy/docker/files/test-intel.ndjson b/test/packages/parallel/ti_anomali/_dev/deploy/docker/files/test-intel.ndjson index 482303a3d7..9145513107 100644 --- a/test/packages/parallel/ti_anomali/_dev/deploy/docker/files/test-intel.ndjson +++ b/test/packages/parallel/ti_anomali/_dev/deploy/docker/files/test-intel.ndjson @@ -1,3 +1,3 @@ -{"message_type":"start","integrator_metadata":{"user_metadata":"{\"url\":\"http://elastic-agent:9080/\",\"secret\":\"TheSecret\",\"request_timeout\":10}","ioc_field_names":["domain","itype","classification","lat","update_id","source_feed_id","id","confidence","severity","trusted_circle_ids","lon","date_first","source","state","import_session_id","value_type","srcip","org","date_last","country","detail2","resource_uri","added_at"]}} +{"message_type":"start","integrator_metadata":{"user_metadata":"{\"url\":\"http://docker-custom-agent-ti_anomali-threatstream:9080/\",\"secret\":\"TheSecret\",\"request_timeout\":10}","ioc_field_names":["domain","itype","classification","lat","update_id","source_feed_id","id","confidence","severity","trusted_circle_ids","lon","date_first","source","state","import_session_id","value_type","srcip","org","date_last","country","detail2","resource_uri","added_at"]}} {"message_type":"data","intelligence":[["d4xgfj.example.net","mal_domain","public",-49.1,3786618776,3143,3135167627,20,"high","122",94.4,"2020-10-08T12:21:50","Default Organization","active",1400,"domain","89.160.20.156","OVH Hosting","2020-10-08T12:24:42","FR","imported by user 184","/api/v1/intelligence/P46279656657/","2020-10-08T12:22:11"]]} {"message_type":"finish","message":"goodbye!"} From e4d893ed00b257336cfe3d215abe45d5faa9858e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 14 Mar 2024 12:49:00 +0100 Subject: [PATCH 028/114] Example using environment variables --- internal/servicedeployer/compose.go | 7 +++++-- .../_dev/deploy/docker/docker-compose.yml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/servicedeployer/compose.go b/internal/servicedeployer/compose.go index 360a46ef2d..3500202255 100644 --- a/internal/servicedeployer/compose.go +++ b/internal/servicedeployer/compose.go @@ -68,7 +68,10 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer ymlPaths: d.ymlPaths, project: "elastic-package-service", variant: d.variant, - env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local)}, + env: []string{ + fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local), + fmt.Sprintf("%s=%s", agentHosnameEnv, "docker-custom-agent-hit_count_assertion-test"), + }, } outCtxt := inCtxt @@ -141,7 +144,7 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer logger.Debugf("adding service container %s internal ports to context", p.ContainerName(serviceName)) serviceComposeConfig, err := p.Config(compose.CommandOptions{ - Env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, outCtxt.Logs.Folder.Local)}, + Env: service.env, }) if err != nil { return nil, fmt.Errorf("could not get Docker Compose configuration for service: %w", err) diff --git a/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml b/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml index 63cb1aa00e..bda2944d81 100644 --- a/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml @@ -4,4 +4,4 @@ services: image: akroh/stream:v0.0.1 volumes: - ./logs:/logs:ro - command: log --start-signal=SIGHUP --delay=5s --addr docker-custom-agent-hit_count_assertion-test:9999 -p=tcp /logs/generated.log + command: log --start-signal=SIGHUP --delay=5s --addr ${AGENT_HOSTNAME}:9999 -p=tcp /logs/generated.log From 92340d964cd552c2c5e73f389f5ad61a9fa3f353 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 14 Mar 2024 13:19:52 +0100 Subject: [PATCH 029/114] Use custom shutdown time for service compose teardown --- internal/servicedeployer/compose.go | 8 +++++++- internal/servicedeployer/terraform.go | 7 ++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/servicedeployer/compose.go b/internal/servicedeployer/compose.go index 3500202255..495d89a907 100644 --- a/internal/servicedeployer/compose.go +++ b/internal/servicedeployer/compose.go @@ -42,6 +42,8 @@ type DockerComposeServiceDeployerOptions struct { type dockerComposeDeployedService struct { ctxt ServiceContext + shutdownTimeout string // default shutdown timeout 10 seconds + ymlPaths []string project string variant ServiceVariant @@ -225,6 +227,10 @@ func (s *dockerComposeDeployedService) TearDown() error { return fmt.Errorf("could not create Docker Compose project for service: %w", err) } + extraArgs := []string{} + if s.shutdownTimeout != "" { + extraArgs = append(extraArgs, "-t", s.shutdownTimeout) + } opts := compose.CommandOptions{ Env: append( s.env, @@ -232,7 +238,7 @@ func (s *dockerComposeDeployedService) TearDown() error { } if err := p.Stop(compose.CommandOptions{ Env: opts.Env, - ExtraArgs: []string{"-t", "300"}, // default shutdown timeout 10 seconds + ExtraArgs: extraArgs, }); err != nil { return fmt.Errorf("could not stop service using Docker Compose: %w", err) } diff --git a/internal/servicedeployer/terraform.go b/internal/servicedeployer/terraform.go index 381439133f..3dcbc07081 100644 --- a/internal/servicedeployer/terraform.go +++ b/internal/servicedeployer/terraform.go @@ -106,9 +106,10 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt) service := dockerComposeDeployedService{ - ymlPaths: ymlPaths, - project: "elastic-package-service", - env: tfEnvironment, + ymlPaths: ymlPaths, + project: "elastic-package-service", + env: tfEnvironment, + shutdownTimeout: "300", } outCtxt := inCtxt From 95c4b6f63072cfb38956e0390590987968b05083 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 14 Mar 2024 14:12:42 +0100 Subject: [PATCH 030/114] Update hostnames for elastic-agent in tests --- .../other/multiinput/_dev/deploy/docker/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/packages/other/multiinput/_dev/deploy/docker/docker-compose.yml b/test/packages/other/multiinput/_dev/deploy/docker/docker-compose.yml index 85ff94ee54..a51b762b6e 100644 --- a/test/packages/other/multiinput/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/other/multiinput/_dev/deploy/docker/docker-compose.yml @@ -4,9 +4,9 @@ services: image: akroh/stream:v0.0.1 volumes: - ./logs:/logs:ro - command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9999 -p=tcp /logs/generated.log + command: log --start-signal=SIGHUP --delay=5s --addr ${AGENT_HOSTNAME}:9999 -p=tcp /logs/generated.log test-udp: image: akroh/stream:v0.0.1 volumes: - ./logs:/logs:ro - command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9999 -p=udp /logs/generated.log + command: log --start-signal=SIGHUP --delay=5s --addr ${AGENT_HOSTNAME}:9999 -p=udp /logs/generated.log From 1bac588a1a8fb63659c8ada0f0f6d05203912c33 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 14 Mar 2024 17:49:18 +0100 Subject: [PATCH 031/114] Set dinamcally the agent hostname var in services --- internal/servicedeployer/compose.go | 2 +- internal/servicedeployer/context.go | 4 ++++ internal/testrunner/runners/system/runner.go | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/servicedeployer/compose.go b/internal/servicedeployer/compose.go index 495d89a907..0f75787cc0 100644 --- a/internal/servicedeployer/compose.go +++ b/internal/servicedeployer/compose.go @@ -72,7 +72,7 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer variant: d.variant, env: []string{ fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local), - fmt.Sprintf("%s=%s", agentHosnameEnv, "docker-custom-agent-hit_count_assertion-test"), + fmt.Sprintf("%s=%s", agentHosnameEnv, inCtxt.AgentHostname), }, } outCtxt := inCtxt diff --git a/internal/servicedeployer/context.go b/internal/servicedeployer/context.go index c01e9dacef..0bd082faa4 100644 --- a/internal/servicedeployer/context.go +++ b/internal/servicedeployer/context.go @@ -22,6 +22,10 @@ type ServiceContext struct { // the Agent container. Hostname string + // AgentHostname is the host name of the agent, as addressable from + // the Service container . + AgentHostname string + // Ports is a list of ports that the service listens on, as addressable // from the Agent container. Ports []int diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 3127ea4df3..ad96fbb4d7 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -770,6 +770,8 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic } ctxt.Logs.Folder.Local = agentInfo.Logs.Folder.Local + ctxt.AgentHostname = agentDeployed.Context().Hostname + if config.Service != "" { ctxt.Name = config.Service } From 21dbd60127884db5abaccef4f898d7267c97770f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 14 Mar 2024 18:56:00 +0100 Subject: [PATCH 032/114] Fix environment variable for agent hostname --- internal/agentdeployer/agent.go | 17 +++++++++++------ internal/agentdeployer/compose.go | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 9c04a1cf2c..81d0acc0f4 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -21,7 +21,7 @@ import ( ) const ( - dockerCustomAgentName = "docker-custom-agent" + dockerCustomAgentNamePrefix = "docker-custom-agent" dockerCustomAgentDir = "docker_custom_agent" dockerCustomAgentDockerCompose = "docker-agent-base.yml" defaultAgentPolicyName = "Elastic-Agent (elastic-package)" @@ -94,7 +94,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), fmt.Sprintf("%s=%s", fleetPolicyEnv, defaultAgentPolicyName), - fmt.Sprintf("%s=docker-custom-agent-%s", agentHosnameEnv, d.agentName()), + fmt.Sprintf("%s=%s", agentHosnameEnv, d.agentHostname()), ) configDir, err := d.installDockerfile() @@ -118,7 +118,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { ymlPaths: ymlPaths, project: composeProjectName, variant: AgentVariant{ - Name: dockerCustomAgentName, + Name: dockerCustomAgentNamePrefix, Env: env, }, } @@ -151,7 +151,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { } // Service name defined in the docker-compose file - inCtxt.Name = dockerCustomAgentName + inCtxt.Name = dockerCustomAgentNamePrefix serviceName := inCtxt.Name opts := compose.CommandOptions{ @@ -182,8 +182,9 @@ func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { return nil, fmt.Errorf("service is unhealthy: %w", err) } - // Build service container name - outCtxt.Hostname = p.ContainerName(serviceName) + // Build agent container name + // outCtxt.Hostname = p.ContainerName(serviceName) + outCtxt.Hostname = d.agentHostname() logger.Debugf("adding service container %s internal ports to context", p.ContainerName(serviceName)) serviceComposeConfig, err := p.Config(compose.CommandOptions{Env: env}) @@ -207,6 +208,10 @@ func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { return &service, nil } +func (d *CustomAgentDeployer) agentHostname() string { + return fmt.Sprintf("docker-custom-agent-%s", d.agentName()) +} + func (d *CustomAgentDeployer) agentName() string { name := d.packageName if d.variant.Name != "" { diff --git a/internal/agentdeployer/compose.go b/internal/agentdeployer/compose.go index 0fec1de505..d2068f5699 100644 --- a/internal/agentdeployer/compose.go +++ b/internal/agentdeployer/compose.go @@ -99,7 +99,7 @@ func (d *DockerComposeAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, err return nil, fmt.Errorf("service is unhealthy: %w", err) } - // Build service container name + // Build agent container name outCtxt.Hostname = p.ContainerName(agentName) // Connect service network with stack network (for the purpose of metrics collection) From d25da3daedac711890f1a7fed9928a056033d617 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 14 Mar 2024 19:56:34 +0100 Subject: [PATCH 033/114] Set agent hostname if agentDeployed is defined --- internal/testrunner/runners/system/runner.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index ad96fbb4d7..b426f30e46 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -770,8 +770,12 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic } ctxt.Logs.Folder.Local = agentInfo.Logs.Folder.Local - ctxt.AgentHostname = agentDeployed.Context().Hostname - + if agentDeployed != nil { + // In case of CustomAgents from servicedeployer where agent and service + // are deployed in the same docker-compose scenario (servicedeployer), + // so there is no agentDeployed in that scenario + ctxt.AgentHostname = agentDeployed.Context().Hostname + } if config.Service != "" { ctxt.Name = config.Service } From e834d9c8ad0d3c85985dc0ef4ccd38ad4aeb89a9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 20 Mar 2024 17:21:41 +0100 Subject: [PATCH 034/114] Check tags to filter agents --- internal/agentdeployer/agent.go | 2 ++ internal/agentdeployer/info.go | 14 ++++++++----- internal/kibana/agents.go | 1 + internal/testrunner/runners/system/runner.go | 21 ++++++++++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 81d0acc0f4..7c03fc671a 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/configuration/locations" @@ -95,6 +96,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), fmt.Sprintf("%s=%s", fleetPolicyEnv, defaultAgentPolicyName), fmt.Sprintf("%s=%s", agentHosnameEnv, d.agentHostname()), + fmt.Sprintf("%s=%s", elasticAgentTagsEnv, strings.Join(inCtxt.Tags, ",")), ) configDir, err := d.installDockerfile() diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go index ad0fda6076..941a104757 100644 --- a/internal/agentdeployer/info.go +++ b/internal/agentdeployer/info.go @@ -5,11 +5,12 @@ package agentdeployer const ( - localCACertEnv = "LOCAL_CA_CERT" - serviceLogsDirEnv = "SERVICE_LOGS_DIR" - testRunIDEnv = "TEST_RUN_ID" - fleetPolicyEnv = "FLEET_TOKEN_POLICY_NAME" - agentHosnameEnv = "AGENT_HOSTNAME" + localCACertEnv = "LOCAL_CA_CERT" + serviceLogsDirEnv = "SERVICE_LOGS_DIR" + testRunIDEnv = "TEST_RUN_ID" + fleetPolicyEnv = "FLEET_TOKEN_POLICY_NAME" + agentHosnameEnv = "AGENT_HOSTNAME" + elasticAgentTagsEnv = "ELASTIC_AGENT_TAGS" ) // AgentInfo encapsulates context that is both available to a AgentDeployer and @@ -71,4 +72,7 @@ type AgentInfo struct { // Service token Token string + + // Tags set in the agent + Tags []string } diff --git a/internal/kibana/agents.go b/internal/kibana/agents.go index 521c0fe76f..4d67b4d366 100644 --- a/internal/kibana/agents.go +++ b/internal/kibana/agents.go @@ -37,6 +37,7 @@ type Agent struct { } `json:"local_metadata"` Status string `json:"status"` EnrolledAt time.Time `json:"enrolled_at"` + Tags []string `json:"tags,omitempty"` } // String method returns string representation of an agent. diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index b426f30e46..69be20d186 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -308,6 +308,11 @@ func (r *runner) createAgentInfo(scenario scenarioTest) (agentdeployer.AgentInfo info.Logs.Folder.Agent = ServiceLogsAgentDir info.Test.RunID = createTestRunID() + info.Tags = append(info.Tags, "test", "system", scenario.pkgManifest.Name, info.Test.RunID) + if scenario.dataStreamManifest != nil { + info.Tags = append(info.Tags, scenario.dataStreamManifest.Name) + } + return info, nil } @@ -1739,6 +1744,22 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t continue } + // Tags are available starting in 8.3 + if len(agent.Tags) > 0 { + logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(agentInfo.Tags, ",")) + foundAllTags := true + for _, tag := range agentInfo.Tags { + if !slices.Contains(agent.Tags, tag) { + logger.Debugf("filtered agent (invalid tag found) %s - %q vs %q", tag, strings.Join(agent.Tags, ","), strings.Join(agentInfo.Tags, ",")) // TODO: remove + foundAllTags = false + break + } + } + if !foundAllTags { + continue + } + } + // FIXME: check for package and data stream name too ? // Current verson could be returning an unexpected agent if tests are parallelized hasAgentPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) From 50c331a932f65cdde0c37d4f5103b4428071df4e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 20 Mar 2024 17:27:39 +0100 Subject: [PATCH 035/114] Set tags in agent container docker-compose --- internal/agentdeployer/_static/docker-agent-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/agentdeployer/_static/docker-agent-base.yml b/internal/agentdeployer/_static/docker-agent-base.yml index 0554f79bb3..bd2960be5f 100644 --- a/internal/agentdeployer/_static/docker-agent-base.yml +++ b/internal/agentdeployer/_static/docker-agent-base.yml @@ -12,6 +12,7 @@ services: - FLEET_URL=https://fleet-server:8220 - KIBANA_HOST=https://kibana:5601 - FLEET_TOKEN_POLICY_NAME=${FLEET_TOKEN_POLICY_NAME} + - ELASTIC_AGENT_TAGS=${ELASTIC_AGENT_TAGS} volumes: - ${SERVICE_LOGS_DIR}:/tmp/service_logs/ - ${LOCAL_CA_CERT}:/etc/ssl/certs/elastic-package.pem From b9acfc20601bb173350929abe6d30e1830d03890 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 20 Mar 2024 17:30:38 +0100 Subject: [PATCH 036/114] Remove octal change --- internal/testrunner/runners/system/runner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 69be20d186..eae8b1dbc5 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1097,7 +1097,7 @@ func (r *runner) removeServiceStateFile() error { func (r *runner) createServiceStateDir() error { dirPath := filepath.Dir(r.serviceStateFilePath) - err := os.MkdirAll(dirPath, 0o755) + err := os.MkdirAll(dirPath, 0755) if err != nil { return fmt.Errorf("mkdir failed (path: %s): %w", dirPath, err) } @@ -1139,7 +1139,7 @@ func (r *runner) writeScenarioState(currentPolicy, origPolicy *kibana.Policy, co return fmt.Errorf("failed to marshall service setup data: %w", err) } - err = os.WriteFile(r.serviceStateFilePath, dataBytes, 0o644) + err = os.WriteFile(r.serviceStateFilePath, dataBytes, 0644) if err != nil { return fmt.Errorf("failed to write service setup JSON: %w", err) } @@ -1784,7 +1784,7 @@ func writeSampleEvent(path string, doc common.MapStr, specVersion semver.Version return fmt.Errorf("marshalling sample event failed: %w", err) } - err = os.WriteFile(filepath.Join(path, "sample_event.json"), body, 0o644) + err = os.WriteFile(filepath.Join(path, "sample_event.json"), body, 0644) if err != nil { return fmt.Errorf("writing sample event failed: %w", err) } From ead97623b9d5d931382d5b72d181977ccde8fe10 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 20 Mar 2024 18:40:43 +0100 Subject: [PATCH 037/114] Add flag to enable or disable independent agents --- cmd/testrunner.go | 36 ++- internal/benchrunner/runners/system/runner.go | 1 + internal/cobraext/flags.go | 3 + internal/service/boot.go | 1 + .../_static/elastic-agent-managed.yaml.tmpl | 281 ++++++++++++++++++ internal/servicedeployer/factory.go | 7 +- internal/servicedeployer/kubernetes.go | 92 ++++++ internal/servicedeployer/terraform.go | 11 +- internal/testrunner/runners/system/runner.go | 85 +++--- internal/testrunner/testrunner.go | 2 + .../_dev/deploy/docker/docker-compose.yml | 4 +- .../deploy/docker/files/test-intel.ndjson | 2 +- 12 files changed, 468 insertions(+), 57 deletions(-) create mode 100644 internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 9f8d5298ec..f60783b9cd 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -78,6 +78,8 @@ func setupTestCommand() *cobraext.Command { cmd.PersistentFlags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription) cmd.PersistentFlags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription) cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) + // By default, it keeps the same behaviour as previously + cmd.PersistentFlags().BoolP(cobraext.TestIndependentElasticAgentFlagName, "", false, cobraext.TestIndependentElasticAgentFlagDescription) for testType, runner := range testrunner.TestRunners() { action := testTypeCommandActionFactory(runner) @@ -185,6 +187,11 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command return fmt.Errorf("cannot determine if package has data streams: %w", err) } + runIndependentElasticAgent, err := cmd.Flags().GetBool(cobraext.TestIndependentElasticAgentFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.TestIndependentElasticAgentFlagName) + } + configFileFlag := "" runSetup := false runTearDown := false @@ -338,20 +345,21 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command var results []testrunner.TestResult for _, folder := range testFolders { r, err := testrunner.Run(testType, testrunner.TestOptions{ - Profile: profile, - TestFolder: folder, - PackageRootPath: packageRootPath, - GenerateTestResult: generateTestResult, - API: esAPI, - KibanaClient: kibanaClient, - DeferCleanup: deferCleanup, - ServiceVariant: variantFlag, - WithCoverage: testCoverage, - CoverageType: testCoverageFormat, - ConfigFilePath: configFileFlag, - RunSetup: runSetup, - RunTearDown: runTearDown, - RunTestsOnly: runTestsOnly, + Profile: profile, + TestFolder: folder, + PackageRootPath: packageRootPath, + GenerateTestResult: generateTestResult, + API: esAPI, + KibanaClient: kibanaClient, + DeferCleanup: deferCleanup, + ServiceVariant: variantFlag, + WithCoverage: testCoverage, + CoverageType: testCoverageFormat, + ConfigFilePath: configFileFlag, + RunSetup: runSetup, + RunTearDown: runTearDown, + RunTestsOnly: runTestsOnly, + RunIndependentElasticAgent: runIndependentElasticAgent, }) results = append(results, r...) diff --git a/internal/benchrunner/runners/system/runner.go b/internal/benchrunner/runners/system/runner.go index 7d58576156..6e411d362b 100644 --- a/internal/benchrunner/runners/system/runner.go +++ b/internal/benchrunner/runners/system/runner.go @@ -244,6 +244,7 @@ func (r *runner) run() (report reporters.Reportable, err error) { Profile: r.options.Profile, Type: servicedeployer.TypeBench, StackVersion: stackVersion.Version(), + DeployAgent: true, } serviceDeployer, err := servicedeployer.Factory(opts) diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 75df10fccd..8895c586cd 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -233,4 +233,7 @@ const ( SkipPullRequestFlagName = "skip-pull-request" SkipPullRequestFlagDescription = "skip opening a new pull request" + + TestIndependentElasticAgentFlagName = "test-independent-agent" + TestIndependentElasticAgentFlagDescription = "enable testing each test with its own Elastic Agent (technical preview)" ) diff --git a/internal/service/boot.go b/internal/service/boot.go index 7303b3dce0..b5926a02bf 100644 --- a/internal/service/boot.go +++ b/internal/service/boot.go @@ -41,6 +41,7 @@ func BootUp(options Options) error { DevDeployDir: options.DevDeployDir, Variant: options.Variant, StackVersion: options.StackVersion, + DeployAgent: true, }) if err != nil { return fmt.Errorf("can't create the service deployer instance: %w", err) diff --git a/internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl b/internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl new file mode 100644 index 0000000000..f621e8986e --- /dev/null +++ b/internal/servicedeployer/_static/elastic-agent-managed.yaml.tmpl @@ -0,0 +1,281 @@ +apiVersion: v1 +kind: Secret +metadata: + name: elastic-package-certs + namespace: kube-system +data: + ca-cert.pem: {{ .caCertPem }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: elastic-agent + namespace: kube-system + labels: + app: elastic-agent +spec: + selector: + matchLabels: + app: elastic-agent + template: + metadata: + labels: + app: elastic-agent + spec: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: elastic-agent + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: elastic-agent + image: {{ .elasticAgentImage }} + env: + - name: FLEET_ENROLL + value: "1" + # The ip:port pair of fleet server + - name: FLEET_URL + value: {{ .fleetURL }} + # If left empty KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed + - name: FLEET_ENROLLMENT_TOKEN + value: "" + - name: FLEET_TOKEN_POLICY_NAME + value: "{{ .elasticAgentTokenPolicyName }}" + - name: KIBANA_HOST + value: {{ .kibanaURL }} + - name: KIBANA_FLEET_USERNAME + value: "elastic" + - name: KIBANA_FLEET_PASSWORD + value: "changeme" + - name: SSL_CERT_DIR + value: "/etc/ssl/certs:/etc/ssl/elastic-package" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + securityContext: + runAsUser: 0 + resources: + limits: + memory: 500Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: elastic-package-ca + mountPath: /etc/ssl/elastic-package + readOnly: true + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: etc-kubernetes + mountPath: /hostfs/etc/kubernetes + - name: var-lib + mountPath: /hostfs/var/lib + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true + - name: passwd + mountPath: /hostfs/etc/passwd + readOnly: true + - name: group + mountPath: /hostfs/etc/group + readOnly: true + - name: etcsysmd + mountPath: /hostfs/etc/systemd + readOnly: true + volumes: + - name: elastic-package-ca + secret: + secretName: elastic-package-certs + - name: proc + hostPath: + path: /proc + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: var-lib + hostPath: + path: /var/lib + - name: passwd + hostPath: + path: /etc/passwd + - name: group + hostPath: + path: /etc/group + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlog + hostPath: + path: /var/log + - name: etcsysmd + hostPath: + path: /etc/systemd +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: elastic-agent +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: ClusterRole + name: elastic-agent + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: kube-system + name: elastic-agent +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: Role + name: elastic-agent + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: elastic-agent-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: Role + name: elastic-agent-kubeadm-config + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: elastic-agent + labels: + k8s-app: elastic-agent +rules: + - apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - services + - configmaps + - serviceaccounts + - persistentvolumes + - persistentvolumeclaims + verbs: ["get", "list", "watch"] + # Enable this rule only if planing to use kubernetes_secrets provider + #- apiGroups: [""] + # resources: + # - secrets + # verbs: ["get"] + - apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + - daemonsets + verbs: ["get", "list", "watch"] + - apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get + - apiGroups: [ "batch" ] + resources: + - jobs + - cronjobs + verbs: [ "get", "list", "watch" ] + # required for apiserver + - nonResourceURLs: + - "/metrics" + verbs: + - get + - apiGroups: ["rbac.authorization.k8s.io"] + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: + - ingressclasses + - ingresses + verbs: ["get", "list", "watch"] + - apiGroups: ["policy"] + resources: + - podsecuritypolicies + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: elastic-agent + # should be the namespace where elastic-agent is running + namespace: kube-system + labels: + k8s-app: elastic-agent +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: elastic-agent-kubeadm-config + namespace: kube-system + labels: + k8s-app: elastic-agent +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elastic-agent + namespace: kube-system + labels: + k8s-app: elastic-agent +--- \ No newline at end of file diff --git a/internal/servicedeployer/factory.go b/internal/servicedeployer/factory.go index 3b336798cd..0d39565e2e 100644 --- a/internal/servicedeployer/factory.go +++ b/internal/servicedeployer/factory.go @@ -27,6 +27,7 @@ type FactoryOptions struct { DevDeployDir string Type string StackVersion string + DeployAgent bool PackageName string DataStream string @@ -63,6 +64,7 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { RunSetup: options.RunSetup, RunTestsOnly: options.RunTestsOnly, RunTearDown: options.RunTearDown, + DeployAgent: options.DeployAgent, } return NewKubernetesServiceDeployer(opts) } @@ -112,7 +114,10 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { return nil, errors.New("terraform service deployer not supported to run by steps") } if _, err := os.Stat(serviceDeployerPath); err == nil { - return NewTerraformServiceDeployer(serviceDeployerPath) + opts := TerraformServiceDeployerOptions{ + DefinitionsDir: serviceDeployerPath, + } + return NewTerraformServiceDeployer(opts) } } return nil, fmt.Errorf("unsupported service deployer (name: %s)", serviceDeployerName) diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index 55a749db35..1eddc71648 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -5,14 +5,21 @@ package servicedeployer import ( + "bytes" _ "embed" + "encoding/base64" "fmt" + "os" "path/filepath" + "strings" + "text/template" + "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kind" "github.com/elastic/elastic-package/internal/kubectl" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/stack" ) // KubernetesServiceDeployer is responsible for deploying resources in the Kubernetes cluster. @@ -21,6 +28,8 @@ type KubernetesServiceDeployer struct { definitionsDir string stackVersion string + deployAgent bool + runSetup bool runTestsOnly bool runTearDown bool @@ -31,6 +40,8 @@ type KubernetesServiceDeployerOptions struct { DefinitionsDir string StackVersion string + DeployAgent bool + RunSetup bool RunTestsOnly bool RunTearDown bool @@ -90,6 +101,7 @@ func NewKubernetesServiceDeployer(opts KubernetesServiceDeployerOptions) (*Kuber runSetup: opts.RunSetup, runTestsOnly: opts.RunTestsOnly, runTearDown: opts.RunTearDown, + deployAgent: opts.DeployAgent, }, nil } @@ -110,6 +122,15 @@ func (ksd KubernetesServiceDeployer) SetUp(ctxt ServiceContext) (DeployedService } } + if ksd.runTearDown || ksd.runTestsOnly || !ksd.deployAgent { + logger.Debug("Skip install Elastic Agent in cluster") + } else { + err = installElasticAgentInCluster(ksd.profile, ksd.stackVersion) + if err != nil { + return nil, fmt.Errorf("can't install Elastic-Agent in the Kubernetes cluster: %w", err) + } + } + if !ksd.runTearDown { err = ksd.installCustomDefinitions() if err != nil { @@ -160,3 +181,74 @@ func findKubernetesDefinitions(definitionsDir string) ([]string, error) { definitionPaths = append(definitionPaths, files...) return definitionPaths, nil } + +func installElasticAgentInCluster(profile *profile.Profile, stackVersion string) error { + logger.Debug("install Elastic Agent in the Kubernetes cluster") + + elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion) + if err != nil { + return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) + } + + err = kubectl.ApplyStdin(elasticAgentManagedYaml) + if err != nil { + return fmt.Errorf("can't install Elastic-Agent in Kubernetes cluster: %w", err) + } + return nil +} + +//go:embed _static/elastic-agent-managed.yaml.tmpl +var elasticAgentManagedYamlTmpl string + +func getElasticAgentYAML(profile *profile.Profile, stackVersion string) ([]byte, error) { + logger.Debugf("Prepare YAML definition for Elastic Agent running in stack v%s", stackVersion) + + appConfig, err := install.Configuration() + if err != nil { + return nil, fmt.Errorf("can't read application configuration: %w", err) + } + + caCert, err := readCACertBase64(profile) + if err != nil { + return nil, fmt.Errorf("can't read certificate authority file: %w", err) + } + + tmpl := template.Must(template.New("elastic-agent.yml").Parse(elasticAgentManagedYamlTmpl)) + + var elasticAgentYaml bytes.Buffer + err = tmpl.Execute(&elasticAgentYaml, map[string]string{ + "fleetURL": "https://fleet-server:8220", + "kibanaURL": "https://kibana:5601", + "caCertPem": caCert, + "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, + "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion), + }) + if err != nil { + return nil, fmt.Errorf("can't generate elastic agent manifest: %w", err) + } + + return elasticAgentYaml.Bytes(), nil +} + +func readCACertBase64(profile *profile.Profile) (string, error) { + caCertPath, err := stack.FindCACertificate(profile) + if err != nil { + return "", fmt.Errorf("can't locate CA certificate: %w", err) + } + + d, err := os.ReadFile(caCertPath) + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(d), nil +} + +// getTokenPolicyName function returns the policy name for the 8.x Elastic stack. The agent's policy +// is predefined in the Kibana configuration file. The logic is not present in older stacks. +func getTokenPolicyName(stackVersion string) string { + if strings.HasPrefix(stackVersion, "8.") { + return "Elastic-Agent (elastic-package)" + } + return "" +} diff --git a/internal/servicedeployer/terraform.go b/internal/servicedeployer/terraform.go index 4c25b4ed6b..4391593f9f 100644 --- a/internal/servicedeployer/terraform.go +++ b/internal/servicedeployer/terraform.go @@ -42,6 +42,12 @@ var terraformDeployerDockerfileContent string // TerraformServiceDeployer is responsible for deploying infrastructure described with Terraform definitions. type TerraformServiceDeployer struct { definitionsDir string + deployAgent bool +} + +type TerraformServiceDeployerOptions struct { + DefinitionsDir string + DeployAgent bool } // addTerraformOutputs method reads the terraform outputs generated in the json format and @@ -82,9 +88,10 @@ func addTerraformOutputs(outCtxt ServiceContext) error { } // NewTerraformServiceDeployer creates an instance of TerraformServiceDeployer. -func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeployer, error) { +func NewTerraformServiceDeployer(opts TerraformServiceDeployerOptions) (*TerraformServiceDeployer, error) { return &TerraformServiceDeployer{ - definitionsDir: definitionsDir, + definitionsDir: opts.DefinitionsDir, + deployAgent: opts.DeployAgent, }, nil } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index eae8b1dbc5..5ba1e39288 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -293,6 +293,7 @@ func (r *runner) createServiceOptions(variantName string) servicedeployer.Factor RunTearDown: r.options.RunTearDown, RunTestsOnly: r.options.RunTestsOnly, RunSetup: r.options.RunSetup, + DeployAgent: r.options.RunIndependentElasticAgent, } } @@ -746,22 +747,24 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic return nil, fmt.Errorf("could not create agent runner: %w", err) } var agentDeployed agentdeployer.DeployedAgent - if agentDeployer != nil { - agentDeployed, err = agentDeployer.SetUp(agentInfo) - if err != nil { - return nil, fmt.Errorf("could not setup agent: %w", err) + if r.options.RunIndependentElasticAgent { + if agentDeployer != nil { + agentDeployed, err = agentDeployer.SetUp(agentInfo) + if err != nil { + return nil, fmt.Errorf("could not setup agent: %w", err) + } } - } - r.shutdownAgentHandler = func() error { - if agentDeployer == nil { + r.shutdownAgentHandler = func() error { + if agentDeployer == nil { + return nil + } + logger.Debug("tearing down agent...") + if err := agentDeployed.TearDown(); err != nil { + return fmt.Errorf("error tearing down agent: %w", err) + } + return nil } - logger.Debug("tearing down agent...") - if err := agentDeployed.TearDown(); err != nil { - return fmt.Errorf("error tearing down agent: %w", err) - } - - return nil } // Setup service. @@ -774,12 +777,16 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic return nil, fmt.Errorf("could not create service runner: %w", err) } - ctxt.Logs.Folder.Local = agentInfo.Logs.Folder.Local - if agentDeployed != nil { - // In case of CustomAgents from servicedeployer where agent and service - // are deployed in the same docker-compose scenario (servicedeployer), - // so there is no agentDeployed in that scenario - ctxt.AgentHostname = agentDeployed.Context().Hostname + if r.options.RunIndependentElasticAgent { + ctxt.Logs.Folder.Local = agentInfo.Logs.Folder.Local + if agentDeployed != nil { + // In case of CustomAgents from servicedeployer where agent and service + // are deployed in the same docker-compose scenario (servicedeployer), + // so there is no agentDeployed in that scenario + ctxt.AgentHostname = agentDeployed.Context().Hostname + } + } else { + ctxt.AgentHostname = "elastic-agent" } if config.Service != "" { ctxt.Name = config.Service @@ -934,7 +941,7 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic // FIXME: running per stages does not work when multiple agents are created var origPolicy kibana.Policy - agents, err := checkEnrolledAgents(r.options.KibanaClient, agentInfo, enrollingTime) + agents, err := checkEnrolledAgents(r.options.KibanaClient, agentInfo, enrollingTime, ctxt, r.options.RunIndependentElasticAgent) if err != nil { return nil, fmt.Errorf("can't check enrolled agents: %w", err) } @@ -1064,8 +1071,6 @@ func (r *runner) prepareScenario(config *testConfig, ctxt servicedeployer.Servic if !passed { return nil, testrunner.ErrTestCaseFailed{Reason: fmt.Sprintf("could not find hits in %s data stream", scenario.dataStream)} - } else { - logger.Debugf(">>>>> Final hits found: %d", hits.size()) // TODO: to remove } logger.Debugf("check whether or not synthetics is enabled (component template %s)...", componentTemplatePackage) @@ -1277,7 +1282,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return r.validateTestScenario(result, scenario, config, serviceOptions) } -func checkEnrolledAgents(client *kibana.Client, agentInfo agentdeployer.AgentInfo, threshold time.Time) ([]kibana.Agent, error) { +func checkEnrolledAgents(client *kibana.Client, agentInfo agentdeployer.AgentInfo, threshold time.Time, svcInfo servicedeployer.ServiceContext, runIndependentElasticAgent bool) ([]kibana.Agent, error) { var agents []kibana.Agent enrolled, err := waitUntilTrue(func() (bool, error) { @@ -1290,7 +1295,7 @@ func checkEnrolledAgents(client *kibana.Client, agentInfo agentdeployer.AgentInf return false, fmt.Errorf("could not list agents: %w", err) } - agents = filterAgents(allAgents, agentInfo, threshold) + agents = filterAgents(allAgents, agentInfo, threshold, svcInfo, runIndependentElasticAgent) logger.Debugf("found %d enrolled agent(s)", len(agents)) if len(agents) == 0 { return false, nil // selected agents are unavailable yet @@ -1714,7 +1719,7 @@ func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) } } -func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, threshold time.Time) []kibana.Agent { +func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, threshold time.Time, svcInfo servicedeployer.ServiceContext, runIndependentElasticAgent bool) []kibana.Agent { if agentInfo.Agent.Host.NamePrefix != "" { logger.Debugf("filter agents using criteria: NamePrefix=%s", agentInfo.Agent.Host.NamePrefix) } @@ -1739,13 +1744,13 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t continue } - if agent.EnrolledAt.Before(threshold) { + if runIndependentElasticAgent && agent.EnrolledAt.Before(threshold) { logger.Debugf("filtered agent (enrolling time) %q", agent.ID) // TODO: remove continue } // Tags are available starting in 8.3 - if len(agent.Tags) > 0 { + if runIndependentElasticAgent && len(agent.Tags) > 0 { logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(agentInfo.Tags, ",")) foundAllTags := true for _, tag := range agentInfo.Tags { @@ -1760,17 +1765,23 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t } } - // FIXME: check for package and data stream name too ? - // Current verson could be returning an unexpected agent if tests are parallelized - hasAgentPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) - // hasServicePrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, ctxt.Agent.Host.NamePrefix) - - // TODO: required for custom agents triggered from service deployers + if runIndependentElasticAgent { + // FIXME: check for package and data stream name too ? + // Current verson could be returning an unexpected agent if tests are parallelized + hasAgentPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) + // if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix && !hasServicePrefix { + if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix { + logger.Debugf("filtered agent (prefix) %q", agent.ID) // TODO: remove + continue + } + } else { + // TODO: required for custom agents triggered from service deployers ? + hasServicePrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, svcInfo.Agent.Host.NamePrefix) + if svcInfo.Agent.Host.NamePrefix != "" && !hasServicePrefix { + logger.Debugf("filtered agent (prefix) %q", agent.ID) // TODO: remove + continue + } - // if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix && !hasServicePrefix { - if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix { - logger.Debugf("filtered agent (prefix) %q", agent.ID) // TODO: remove - continue } filtered = append(filtered, agent) } diff --git a/internal/testrunner/testrunner.go b/internal/testrunner/testrunner.go index 6c0e6923a0..880abd8094 100644 --- a/internal/testrunner/testrunner.go +++ b/internal/testrunner/testrunner.go @@ -36,6 +36,8 @@ type TestOptions struct { API *elasticsearch.API KibanaClient *kibana.Client + RunIndependentElasticAgent bool + DeferCleanup time.Duration ServiceVariant string WithCoverage bool diff --git a/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml b/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml index 59d6f29f7d..47d1bd6cb5 100644 --- a/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml @@ -18,7 +18,7 @@ services: - ./sample_logs:/sample_logs:ro environment: - STREAM_PROTOCOL=webhook - - STREAM_ADDR=http://docker-custom-agent-ti_anomali-threatstream:9080/ + - STREAM_ADDR=http://${AGENT_HOSTNAME}:9080/ command: log --webhook-content-type application/x-ndjson --start-signal=SIGHUP --delay=5s /sample_logs/test-threatstream-ndjson.log threatstream-webhook-https: image: docker.elastic.co/observability/stream:v0.6.1 @@ -27,7 +27,7 @@ services: environment: - STREAM_PROTOCOL=webhook - STREAM_INSECURE=true - - STREAM_ADDR=https://docker-custom-agent-ti_anomali-threatstream:7443/ + - STREAM_ADDR=https://${AGENT_HOSTNAME}:7443/ command: log --webhook-content-type application/x-ndjson --start-signal=SIGHUP --delay=5s /sample_logs/test-threatstream-ndjson.log threatstream-integrator-test: image: docker.io/adrisr/filebeat-anomali-integrator-test:latest diff --git a/test/packages/parallel/ti_anomali/_dev/deploy/docker/files/test-intel.ndjson b/test/packages/parallel/ti_anomali/_dev/deploy/docker/files/test-intel.ndjson index 9145513107..482303a3d7 100644 --- a/test/packages/parallel/ti_anomali/_dev/deploy/docker/files/test-intel.ndjson +++ b/test/packages/parallel/ti_anomali/_dev/deploy/docker/files/test-intel.ndjson @@ -1,3 +1,3 @@ -{"message_type":"start","integrator_metadata":{"user_metadata":"{\"url\":\"http://docker-custom-agent-ti_anomali-threatstream:9080/\",\"secret\":\"TheSecret\",\"request_timeout\":10}","ioc_field_names":["domain","itype","classification","lat","update_id","source_feed_id","id","confidence","severity","trusted_circle_ids","lon","date_first","source","state","import_session_id","value_type","srcip","org","date_last","country","detail2","resource_uri","added_at"]}} +{"message_type":"start","integrator_metadata":{"user_metadata":"{\"url\":\"http://elastic-agent:9080/\",\"secret\":\"TheSecret\",\"request_timeout\":10}","ioc_field_names":["domain","itype","classification","lat","update_id","source_feed_id","id","confidence","severity","trusted_circle_ids","lon","date_first","source","state","import_session_id","value_type","srcip","org","date_last","country","detail2","resource_uri","added_at"]}} {"message_type":"data","intelligence":[["d4xgfj.example.net","mal_domain","public",-49.1,3786618776,3143,3135167627,20,"high","122",94.4,"2020-10-08T12:21:50","Default Organization","active",1400,"domain","89.160.20.156","OVH Hosting","2020-10-08T12:24:42","FR","imported by user 184","/api/v1/intelligence/P46279656657/","2020-10-08T12:22:11"]]} {"message_type":"finish","message":"goodbye!"} From 2b1140c86ae23c5ecd194e0cd733794bd23bfb52 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 20 Mar 2024 18:41:22 +0100 Subject: [PATCH 038/114] Rename variable --- internal/agentdeployer/agent.go | 2 +- internal/agentdeployer/info.go | 2 +- internal/servicedeployer/compose.go | 2 +- internal/servicedeployer/context.go | 2 +- internal/servicedeployer/custom_agent.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 7c03fc671a..6f99b96e22 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -95,7 +95,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt AgentInfo) (DeployedAgent, error) { fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), fmt.Sprintf("%s=%s", fleetPolicyEnv, defaultAgentPolicyName), - fmt.Sprintf("%s=%s", agentHosnameEnv, d.agentHostname()), + fmt.Sprintf("%s=%s", agentHostnameEnv, d.agentHostname()), fmt.Sprintf("%s=%s", elasticAgentTagsEnv, strings.Join(inCtxt.Tags, ",")), ) diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go index 941a104757..d6b588ffa6 100644 --- a/internal/agentdeployer/info.go +++ b/internal/agentdeployer/info.go @@ -9,7 +9,7 @@ const ( serviceLogsDirEnv = "SERVICE_LOGS_DIR" testRunIDEnv = "TEST_RUN_ID" fleetPolicyEnv = "FLEET_TOKEN_POLICY_NAME" - agentHosnameEnv = "AGENT_HOSTNAME" + agentHostnameEnv = "AGENT_HOSTNAME" elasticAgentTagsEnv = "ELASTIC_AGENT_TAGS" ) diff --git a/internal/servicedeployer/compose.go b/internal/servicedeployer/compose.go index 91101b6927..18652ab426 100644 --- a/internal/servicedeployer/compose.go +++ b/internal/servicedeployer/compose.go @@ -73,7 +73,7 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer variant: d.variant, env: []string{ fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local), - fmt.Sprintf("%s=%s", agentHosnameEnv, inCtxt.AgentHostname), + fmt.Sprintf("%s=%s", agentHostnameEnv, inCtxt.AgentHostname), }, } outCtxt := inCtxt diff --git a/internal/servicedeployer/context.go b/internal/servicedeployer/context.go index 0bd082faa4..5b832b7b41 100644 --- a/internal/servicedeployer/context.go +++ b/internal/servicedeployer/context.go @@ -8,7 +8,7 @@ const ( localCACertEnv = "LOCAL_CA_CERT" serviceLogsDirEnv = "SERVICE_LOGS_DIR" testRunIDEnv = "TEST_RUN_ID" - agentHosnameEnv = "AGENT_HOSTNAME" + agentHostnameEnv = "AGENT_HOSTNAME" ) // ServiceContext encapsulates context that is both available to a ServiceDeployer and diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index 06e48c75da..e52b498478 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -89,7 +89,7 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err appConfig.StackImageRefs(d.stackVersion).AsEnv(), fmt.Sprintf("%s=%s", serviceLogsDirEnv, inCtxt.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), - fmt.Sprintf("%s=%s-%s", agentHosnameEnv, dockerCustomAgentName, d.agentName()), + fmt.Sprintf("%s=%s-%s", agentHostnameEnv, dockerCustomAgentName, d.agentName()), ) configDir, err := d.installDockerfile() From 16b3cd90fb8840fe99b91b8ff71ea9df7eefd60a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 11:12:47 +0100 Subject: [PATCH 039/114] Remove outCtxt var and rename context by info methods --- internal/agentdeployer/agent.go | 42 ++++++++++---------- internal/agentdeployer/compose.go | 32 +++++++-------- internal/agentdeployer/deployed_agent.go | 8 ++-- internal/agentdeployer/kubernetes.go | 7 ++-- internal/testrunner/runners/system/runner.go | 2 +- 5 files changed, 46 insertions(+), 45 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 585d2a69e1..3a8a64ddc9 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -11,6 +11,8 @@ import ( "path/filepath" "strings" + "golang.org/x/net/context" + "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/docker" @@ -19,7 +21,6 @@ import ( "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/stack" - "golang.org/x/net/context" ) const ( @@ -117,7 +118,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D composeProjectName := fmt.Sprintf("elastic-package-agent-%s", d.agentName()) - service := dockerComposeDeployedAgent{ + agent := dockerComposeDeployedAgent{ ymlPaths: ymlPaths, project: composeProjectName, variant: AgentVariant{ @@ -126,10 +127,9 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D }, } - outCtxt := agentInfo - outCtxt.ConfigDir = configDir + agentInfo.ConfigDir = configDir - p, err := compose.NewProject(service.project, service.ymlPaths...) + p, err := compose.NewProject(agent.project, agent.ymlPaths...) if err != nil { return nil, fmt.Errorf("could not create Docker Compose project for agent: %w", err) } @@ -145,9 +145,9 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D // service logs folder must no be deleted to avoid breaking log files written // by the service. If this is required, those files should be rotated or truncated // so the service can still write to them. - logger.Debug("Skipping removing service logs folder folder %s", outCtxt.Logs.Folder.Local) + logger.Debug("Skipping removing service logs folder folder %s", agentInfo.Logs.Folder.Local) } else { - err = files.RemoveContent(outCtxt.Logs.Folder.Local) + err = files.RemoveContent(agentInfo.Logs.Folder.Local) if err != nil { return nil, fmt.Errorf("removing service logs failed: %w", err) } @@ -155,7 +155,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D // Service name defined in the docker-compose file agentInfo.Name = dockerCustomAgentNamePrefix - serviceName := agentInfo.Name + agentName := agentInfo.Name opts := compose.CommandOptions{ Env: env, @@ -170,7 +170,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D return nil, fmt.Errorf("could not boot up agent using Docker Compose: %w", err) } // Connect service network with stack network (for the purpose of metrics collection) - err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network(d.profile)) + err = docker.ConnectToNetwork(p.ContainerName(agentName), stack.Network(d.profile)) if err != nil { return nil, fmt.Errorf("can't attach service container to the stack network: %w", err) } @@ -181,34 +181,34 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D if err != nil { processAgentContainerLogs(ctx, p, compose.CommandOptions{ Env: opts.Env, - }, outCtxt.Name) + }, agentName) return nil, fmt.Errorf("service is unhealthy: %w", err) } // Build agent container name - // outCtxt.Hostname = p.ContainerName(serviceName) - outCtxt.Hostname = d.agentHostname() + // agentInfo.Hostname = p.ContainerName(serviceName) + agentInfo.Hostname = d.agentHostname() - logger.Debugf("adding service container %s internal ports to context", p.ContainerName(serviceName)) + logger.Debugf("adding service container %s internal ports to context", p.ContainerName(agentName)) serviceComposeConfig, err := p.Config(ctx, compose.CommandOptions{Env: env}) if err != nil { return nil, fmt.Errorf("could not get Docker Compose configuration for service: %w", err) } - s := serviceComposeConfig.Services[serviceName] - outCtxt.Ports = make([]int, len(s.Ports)) + s := serviceComposeConfig.Services[agentName] + agentInfo.Ports = make([]int, len(s.Ports)) for idx, port := range s.Ports { - outCtxt.Ports[idx] = port.InternalPort + agentInfo.Ports[idx] = port.InternalPort } // Shortcut to first port for convenience - if len(outCtxt.Ports) > 0 { - outCtxt.Port = outCtxt.Ports[0] + if len(agentInfo.Ports) > 0 { + agentInfo.Port = agentInfo.Ports[0] } - outCtxt.Agent.Host.NamePrefix = agentInfo.Name - service.agentInfo = outCtxt - return &service, nil + agentInfo.Agent.Host.NamePrefix = agentInfo.Name + agent.agentInfo = agentInfo + return &agent, nil } func (d *CustomAgentDeployer) agentHostname() string { diff --git a/internal/agentdeployer/compose.go b/internal/agentdeployer/compose.go index 63884d060c..935fcaed8b 100644 --- a/internal/agentdeployer/compose.go +++ b/internal/agentdeployer/compose.go @@ -9,6 +9,8 @@ import ( "path/filepath" "time" + "golang.org/x/net/context" + "github.com/elastic/elastic-package/internal/builder" "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/docker" @@ -16,7 +18,6 @@ import ( "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/stack" - "golang.org/x/net/context" ) // DockerComposeServiceDeployer knows how to deploy a service defined via @@ -55,7 +56,6 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI variant: d.variant, env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, agentInfo.Logs.Folder.Local)}, } - outCtxt := agentInfo p, err := compose.NewProject(agent.project, agent.ymlPaths...) if err != nil { @@ -69,7 +69,7 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI } // Clean service logs - err = files.RemoveContent(outCtxt.Logs.Folder.Local) + err = files.RemoveContent(agentInfo.Logs.Folder.Local) if err != nil { return nil, fmt.Errorf("removing service logs failed: %w", err) } @@ -96,12 +96,12 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI if err != nil { processAgentContainerLogs(ctx, p, compose.CommandOptions{ Env: opts.Env, - }, outCtxt.Name) + }, agentName) return nil, fmt.Errorf("service is unhealthy: %w", err) } // Build agent container name - outCtxt.Hostname = p.ContainerName(agentName) + agentInfo.Hostname = p.ContainerName(agentName) // Connect service network with stack network (for the purpose of metrics collection) err = docker.ConnectToNetwork(p.ContainerName(agentName), stack.Network(d.profile)) @@ -111,25 +111,25 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI logger.Debugf("adding agent container %s internal ports to context", p.ContainerName(agentName)) agentComposeConfig, err := p.Config(ctx, compose.CommandOptions{ - Env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, outCtxt.Logs.Folder.Local)}, + Env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, agentInfo.Logs.Folder.Local)}, }) if err != nil { return nil, fmt.Errorf("could not get Docker Compose configuration for agent: %w", err) } s := agentComposeConfig.Services[agentName] - outCtxt.Ports = make([]int, len(s.Ports)) + agentInfo.Ports = make([]int, len(s.Ports)) for idx, port := range s.Ports { - outCtxt.Ports[idx] = port.InternalPort + agentInfo.Ports[idx] = port.InternalPort } // Shortcut to first port for convenience - if len(outCtxt.Ports) > 0 { - outCtxt.Port = outCtxt.Ports[0] + if len(agentInfo.Ports) > 0 { + agentInfo.Port = agentInfo.Ports[0] } - outCtxt.Agent.Host.NamePrefix = "docker-custom-agent" - agent.agentInfo = outCtxt + agentInfo.Agent.Host.NamePrefix = "docker-custom-agent" + agent.agentInfo = agentInfo return &agent, nil } @@ -216,13 +216,13 @@ func (s *dockerComposeDeployedAgent) TearDown(ctx context.Context) error { return nil } -// Context returns the current context for the agent. -func (s *dockerComposeDeployedAgent) Context() AgentInfo { +// Info returns the current context for the agent. +func (s *dockerComposeDeployedAgent) Info() AgentInfo { return s.agentInfo } -// SetContext sets the current context for the agent. -func (s *dockerComposeDeployedAgent) SetContext(ctxt AgentInfo) error { +// SetInfo sets the current context for the agent. +func (s *dockerComposeDeployedAgent) SetInfo(ctxt AgentInfo) error { s.agentInfo = ctxt return nil } diff --git a/internal/agentdeployer/deployed_agent.go b/internal/agentdeployer/deployed_agent.go index 9c77e114b8..f5c508f73f 100644 --- a/internal/agentdeployer/deployed_agent.go +++ b/internal/agentdeployer/deployed_agent.go @@ -20,11 +20,11 @@ type DeployedAgent interface { // Signal sends a signal to the service. Signal(ctx context.Context, signal string) error - // Context returns the current context from the service. - Context() AgentInfo + // Info returns the current context from the service. + Info() AgentInfo - // SetContext sets the current context for the service. - SetContext(str AgentInfo) error + // SetInfo sets the current context for the service. + SetInfo(str AgentInfo) error // ExitCode returns true if the service is exited and its exit code. ExitCode(ctx context.Context, service string) (bool, int, error) diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index 63f89f0285..f189a527ec 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -13,13 +13,14 @@ import ( "strings" "text/template" + "golang.org/x/net/context" + "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kind" "github.com/elastic/elastic-package/internal/kubectl" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/stack" - "golang.org/x/net/context" ) // KubernetesAgentDeployer is responsible for deploying resources in the Kubernetes cluster. @@ -71,11 +72,11 @@ func (s kubernetesDeployedAgent) ExitCode(ctx context.Context, _ string) (bool, return false, -1, ErrNotSupported } -func (s kubernetesDeployedAgent) Context() AgentInfo { +func (s kubernetesDeployedAgent) Info() AgentInfo { return s.agentInfo } -func (s *kubernetesDeployedAgent) SetContext(sc AgentInfo) error { +func (s *kubernetesDeployedAgent) SetInfo(sc AgentInfo) error { s.agentInfo = sc return nil } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 48b6d6fb8e..b2b08fe3d1 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -791,7 +791,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, servic // In case of CustomAgents from servicedeployer where agent and service // are deployed in the same docker-compose scenario (servicedeployer), // so there is no agentDeployed in that scenario - serviceContext.AgentHostname = agentDeployed.Context().Hostname + serviceContext.AgentHostname = agentDeployed.Info().Hostname } } else { serviceContext.AgentHostname = "elastic-agent" From 3638f76bd027209a3bdef48270237e8a3bc9a34f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 11:16:48 +0100 Subject: [PATCH 040/114] Update import for context --- internal/agentdeployer/agent.go | 3 +-- internal/agentdeployer/compose.go | 3 +-- internal/agentdeployer/deployed_agent.go | 3 +-- internal/agentdeployer/deployer.go | 2 +- internal/agentdeployer/kubernetes.go | 3 +-- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 3a8a64ddc9..7f3a6f489d 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -5,14 +5,13 @@ package agentdeployer import ( + "context" _ "embed" "fmt" "os" "path/filepath" "strings" - "golang.org/x/net/context" - "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/docker" diff --git a/internal/agentdeployer/compose.go b/internal/agentdeployer/compose.go index 935fcaed8b..976bb52e99 100644 --- a/internal/agentdeployer/compose.go +++ b/internal/agentdeployer/compose.go @@ -4,13 +4,12 @@ package agentdeployer import ( + "context" "fmt" "os" "path/filepath" "time" - "golang.org/x/net/context" - "github.com/elastic/elastic-package/internal/builder" "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/docker" diff --git a/internal/agentdeployer/deployed_agent.go b/internal/agentdeployer/deployed_agent.go index f5c508f73f..69084af194 100644 --- a/internal/agentdeployer/deployed_agent.go +++ b/internal/agentdeployer/deployed_agent.go @@ -5,9 +5,8 @@ package agentdeployer import ( + "context" "errors" - - "golang.org/x/net/context" ) var ErrNotSupported error = errors.New("not supported") diff --git a/internal/agentdeployer/deployer.go b/internal/agentdeployer/deployer.go index a0fb1b250a..daa6b37a7f 100644 --- a/internal/agentdeployer/deployer.go +++ b/internal/agentdeployer/deployer.go @@ -4,7 +4,7 @@ package agentdeployer -import "golang.org/x/net/context" +import "context" // AgentDeployer defines the interface for deploying an agent. It defines methods for // controlling the lifecycle of an agent. diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index f189a527ec..1b93e65a75 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -6,6 +6,7 @@ package agentdeployer import ( "bytes" + "context" _ "embed" "encoding/base64" "fmt" @@ -13,8 +14,6 @@ import ( "strings" "text/template" - "golang.org/x/net/context" - "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kind" "github.com/elastic/elastic-package/internal/kubectl" From afbfafe4e3ea92524c4bf28c5eb625624a5b5aee Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 11:56:40 +0100 Subject: [PATCH 041/114] Support independent agents running system tests by stages --- internal/testrunner/runners/system/runner.go | 32 +++++++++++--------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index b2b08fe3d1..f8ba663b9e 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -747,9 +747,11 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, servic return nil, err } - // FIXME: this should be stored in service state or not used in - // --no-provision and --tear-down stages enrollingTime := time.Now() + if r.options.RunTearDown || r.options.RunTestsOnly { + enrollingTime = serviceStateData.EnrollingAgentTime + agentInfo.Tags = serviceStateData.Agent.Tags + } agentDeployer, err := agentdeployer.Factory(agentOptions) if err != nil { return nil, fmt.Errorf("could not create agent runner: %w", err) @@ -1087,7 +1089,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, servic scenario.docs = hits.getDocs(scenario.syntheticEnabled) if r.options.RunSetup { - err = r.writeScenarioState(policy, &origPolicy, config, origAgent) + err = r.writeScenarioState(policy, &origPolicy, config, origAgent, enrollingTime) if err != nil { return nil, err } @@ -1128,20 +1130,22 @@ func (r *runner) readServiceStateData() (ServiceState, error) { } type ServiceState struct { - OrigPolicy kibana.Policy `json:"orig_policy"` - CurrentPolicy kibana.Policy `json:"current_policy"` - Agent kibana.Agent `json:"agent"` - ConfigFilePath string `json:"config_file_path"` - VariantName string `json:"variant_name"` + OrigPolicy kibana.Policy `json:"orig_policy"` + CurrentPolicy kibana.Policy `json:"current_policy"` + Agent kibana.Agent `json:"agent"` + ConfigFilePath string `json:"config_file_path"` + VariantName string `json:"variant_name"` + EnrollingAgentTime time.Time `json:"enrolling_agent_time"` } -func (r *runner) writeScenarioState(currentPolicy, origPolicy *kibana.Policy, config *testConfig, agent kibana.Agent) error { +func (r *runner) writeScenarioState(currentPolicy, origPolicy *kibana.Policy, config *testConfig, agent kibana.Agent, enrollingTime time.Time) error { data := ServiceState{ - OrigPolicy: *origPolicy, - CurrentPolicy: *currentPolicy, - Agent: agent, - ConfigFilePath: config.Path, - VariantName: config.ServiceVariantName, + OrigPolicy: *origPolicy, + CurrentPolicy: *currentPolicy, + Agent: agent, + ConfigFilePath: config.Path, + VariantName: config.ServiceVariantName, + EnrollingAgentTime: enrollingTime, } dataBytes, err := json.Marshal(data) if err != nil { From 947ba062e5011ee3888ff9c9c11f131dd00cc6f7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 12:24:36 +0100 Subject: [PATCH 042/114] Add steps in CI to test with independent agents --- .buildkite/hooks/pre-command | 38 +++++++++++++++++-- .../pipeline.trigger.integration.tests.sh | 13 +++++-- scripts/test-check-packages.sh | 17 ++++++++- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index e6128c1c75..571055e7dd 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -46,13 +46,33 @@ export CREATED_DATE # Secrets must be redacted # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables -if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives) ]]; then +is_step_required_to_upload_safe_logs() { + if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then + return false + fi + if [[ "$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives ]]; then + return true + fi + return false +} + +if is_step_required_to_upload_safe_logs; then PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH} | jq -c) export PRIVATE_CI_GCS_CREDENTIALS_SECRET export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci" fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then +is_step_testing_gcp () { + if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then + return false + fi + if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp-agent-true" ]]; then + return true + fi + return false +} + +if use_step_testing_gcp; then ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH}) export ELASTIC_PACKAGE_GCP_PROJECT_SECRET ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -c) @@ -63,7 +83,19 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" = export GCP_PROJECT_ID=${ELASTIC_PACKAGE_GCP_PROJECT_SECRET} fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" == "integration-parallel-aws" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs") ]]; then +is_step_testing_aws () { + if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then + return false + fi + if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-aws-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws-agent-true" ]]; then + return true + fi + if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs-agent-true" ]]; then + return true + fi + return false +} +if use_step_testing_aws; then ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) export ELASTIC_PACKAGE_AWS_SECRET_KEY ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 7c292a96b4..2e0a59cccb 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -41,11 +41,14 @@ CHECK_PACKAGES_TESTS=( test-check-packages-benchmarks test-check-packages-with-logstash ) +for independent_agent in false true ; do for test in "${CHECK_PACKAGES_TESTS[@]}"; do - echo " - label: \":go: Running integration test: ${test}\"" + echo " - label: \":go: Running integration test: ${test} - independent_agent ${independent_agent}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" + echo " environment:" + echo " ELASTIC_PACKAGE_INDEPENDENT_AGENT: ${independent_agent}" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/elastic-stack-dump/check-*/logs/*.log" @@ -55,6 +58,7 @@ for test in "${CHECK_PACKAGES_TESTS[@]}"; do echo " - build/kubectl-dump.txt" fi done +done pushd test/packages/false_positives > /dev/null for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do @@ -74,23 +78,26 @@ done popd > /dev/null pushd test/packages/parallel > /dev/null +for independent_agent in false true; do for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do package_name=$(basename "${package}") if [[ "$package_name" == "aws" || "$package_name" == "aws_logs" || "$package_name" == "gcp" ]] ; then echoerr "Skip temporarily ${package_name}" continue fi - echo " - label: \":go: Running integration test: ${package_name}\"" - echo " key: \"integration-parallel-${package_name}\"" + echo " - label: \":go: Running integration test: ${package_name}\" - independent_agent ${independent_agent}" + echo " key: \"integration-parallel-${package_name}-agent-${independent_agent}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" echo " env:" echo " UPLOAD_SAFE_LOGS: 1" + echo " ELASTIC_PACKAGE_INDEPENDENT_AGENT: ${independent_agent}" echo " agents:" echo " provider: \"gcp\"" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package done +done popd > /dev/null diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index f0a9c88a9c..713432633f 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -6,7 +6,8 @@ cleanup() { r=$? # Dump stack logs - elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}" + elastic-package stack dump -v \ + --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}" if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then # Dump kubectl details @@ -40,6 +41,8 @@ cleanup() { trap cleanup EXIT +ELASTIC_PACKAGE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_INDEPENDENT_AGENT:-"false"} +export ELASTIC_PACKAGE_INDEPENDENT_AGENT ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml" export ELASTIC_PACKAGE_LINKS_FILE_PATH @@ -106,8 +109,18 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do elif [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ] && [ "${PACKAGE_UNDER_TEST:-*}" == "system_benchmark" ]; then elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s else + ELASTIC_PACKAGE_TEST_OPTS="" + if [[ "${ELASTIC_PACKAGE_INDEPENDENT_AGENT}" == "true" ]]; then + ELASTIC_PACKAGE_TEST_OPTS="--test-independent-agent" + fi # defer-cleanup is set to a short period to verify that the option is available - elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic + elastic-package test -v \ + --report-format xUnit \ + --report-output file \ + --defer-cleanup 1s \ + --test-coverage \ + --coverage-format=generic \ + ${ELASTIC_PACKAGE_TEST_OPTS} fi ) cd - From bc582d465609897f184365df2f324d86c0c43415 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 12:42:23 +0100 Subject: [PATCH 043/114] Do not use independent agents now in service or benchrunner --- internal/benchrunner/runners/system/runner.go | 14 +++++----- internal/service/boot.go | 14 +++++----- internal/servicedeployer/factory.go | 26 +++++++++---------- internal/servicedeployer/kubernetes.go | 20 +++++++------- internal/testrunner/runners/system/runner.go | 26 +++++++++---------- 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/internal/benchrunner/runners/system/runner.go b/internal/benchrunner/runners/system/runner.go index 79998bfc4c..3e27137b65 100644 --- a/internal/benchrunner/runners/system/runner.go +++ b/internal/benchrunner/runners/system/runner.go @@ -240,13 +240,13 @@ func (r *runner) run(ctx context.Context) (report reporters.Reportable, err erro logger.Debug("setting up service...") devDeployDir := filepath.Clean(filepath.Join(r.options.BenchPath, "deploy")) opts := servicedeployer.FactoryOptions{ - PackageRootPath: r.options.PackageRootPath, - DevDeployDir: devDeployDir, - Variant: r.options.Variant, - Profile: r.options.Profile, - Type: servicedeployer.TypeBench, - StackVersion: stackVersion.Version(), - DeployAgent: true, + PackageRootPath: r.options.PackageRootPath, + DevDeployDir: devDeployDir, + Variant: r.options.Variant, + Profile: r.options.Profile, + Type: servicedeployer.TypeBench, + StackVersion: stackVersion.Version(), + DeployIndependentAgent: false, } serviceDeployer, err := servicedeployer.Factory(opts) diff --git a/internal/service/boot.go b/internal/service/boot.go index a72a5704b1..bea9542653 100644 --- a/internal/service/boot.go +++ b/internal/service/boot.go @@ -36,13 +36,13 @@ type Options struct { func BootUp(ctx context.Context, options Options) error { logger.Debugf("Create new instance of the service deployer") serviceDeployer, err := servicedeployer.Factory(servicedeployer.FactoryOptions{ - Profile: options.Profile, - PackageRootPath: options.DataStreamRootPath, - DataStreamRootPath: options.DataStreamRootPath, - DevDeployDir: options.DevDeployDir, - Variant: options.Variant, - StackVersion: options.StackVersion, - DeployAgent: true, + Profile: options.Profile, + PackageRootPath: options.DataStreamRootPath, + DataStreamRootPath: options.DataStreamRootPath, + DevDeployDir: options.DevDeployDir, + Variant: options.Variant, + StackVersion: options.StackVersion, + DeployIndependentAgent: false, }) if err != nil { return fmt.Errorf("can't create the service deployer instance: %w", err) diff --git a/internal/servicedeployer/factory.go b/internal/servicedeployer/factory.go index 0d39565e2e..fb6f27dc6b 100644 --- a/internal/servicedeployer/factory.go +++ b/internal/servicedeployer/factory.go @@ -22,12 +22,12 @@ const ( type FactoryOptions struct { Profile *profile.Profile - PackageRootPath string - DataStreamRootPath string - DevDeployDir string - Type string - StackVersion string - DeployAgent bool + PackageRootPath string + DataStreamRootPath string + DevDeployDir string + Type string + StackVersion string + DeployIndependentAgent bool PackageName string DataStream string @@ -58,13 +58,13 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { case "k8s": if _, err := os.Stat(serviceDeployerPath); err == nil { opts := KubernetesServiceDeployerOptions{ - Profile: options.Profile, - DefinitionsDir: serviceDeployerPath, - StackVersion: options.StackVersion, - RunSetup: options.RunSetup, - RunTestsOnly: options.RunTestsOnly, - RunTearDown: options.RunTearDown, - DeployAgent: options.DeployAgent, + Profile: options.Profile, + DefinitionsDir: serviceDeployerPath, + StackVersion: options.StackVersion, + RunSetup: options.RunSetup, + RunTestsOnly: options.RunTestsOnly, + RunTearDown: options.RunTearDown, + DeployIndependentAgent: options.DeployIndependentAgent, } return NewKubernetesServiceDeployer(opts) } diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index c84133af5a..f5e82363ed 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -29,7 +29,7 @@ type KubernetesServiceDeployer struct { definitionsDir string stackVersion string - deployAgent bool + deployIndependentAgent bool runSetup bool runTestsOnly bool @@ -41,7 +41,7 @@ type KubernetesServiceDeployerOptions struct { DefinitionsDir string StackVersion string - DeployAgent bool + DeployIndependentAgent bool RunSetup bool RunTestsOnly bool @@ -96,13 +96,13 @@ var _ DeployedService = new(kubernetesDeployedService) // NewKubernetesServiceDeployer function creates a new instance of KubernetesServiceDeployer. func NewKubernetesServiceDeployer(opts KubernetesServiceDeployerOptions) (*KubernetesServiceDeployer, error) { return &KubernetesServiceDeployer{ - profile: opts.Profile, - definitionsDir: opts.DefinitionsDir, - stackVersion: opts.StackVersion, - runSetup: opts.RunSetup, - runTestsOnly: opts.RunTestsOnly, - runTearDown: opts.RunTearDown, - deployAgent: opts.DeployAgent, + profile: opts.Profile, + definitionsDir: opts.DefinitionsDir, + stackVersion: opts.StackVersion, + runSetup: opts.RunSetup, + runTestsOnly: opts.RunTestsOnly, + runTearDown: opts.RunTearDown, + deployIndependentAgent: opts.DeployIndependentAgent, }, nil } @@ -123,7 +123,7 @@ func (ksd KubernetesServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceI } } - if ksd.runTearDown || ksd.runTestsOnly || !ksd.deployAgent { + if ksd.runTearDown || ksd.runTestsOnly || ksd.deployIndependentAgent { logger.Debug("Skip install Elastic Agent in cluster") } else { err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index f8ba663b9e..bcd9ae75f8 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -281,19 +281,19 @@ func (r *runner) createAgentOptions(scenario scenarioTest) agentdeployer.Factory func (r *runner) createServiceOptions(variantName string) servicedeployer.FactoryOptions { return servicedeployer.FactoryOptions{ - Profile: r.options.Profile, - PackageRootPath: r.options.PackageRootPath, - DataStreamRootPath: r.dataStreamPath, - DevDeployDir: DevDeployDir, - Variant: variantName, - Type: servicedeployer.TypeTest, - StackVersion: r.stackVersion.Version(), - PackageName: "", // to be filled in prepareScenario - DataStream: "", // to be filled in prepareScenario - RunTearDown: r.options.RunTearDown, - RunTestsOnly: r.options.RunTestsOnly, - RunSetup: r.options.RunSetup, - DeployAgent: r.options.RunIndependentElasticAgent, + Profile: r.options.Profile, + PackageRootPath: r.options.PackageRootPath, + DataStreamRootPath: r.dataStreamPath, + DevDeployDir: DevDeployDir, + Variant: variantName, + Type: servicedeployer.TypeTest, + StackVersion: r.stackVersion.Version(), + PackageName: "", // to be filled in prepareScenario + DataStream: "", // to be filled in prepareScenario + RunTearDown: r.options.RunTearDown, + RunTestsOnly: r.options.RunTestsOnly, + RunSetup: r.options.RunSetup, + DeployIndependentAgent: r.options.RunIndependentElasticAgent, } } From a577063029100babba08d7b4ecbab52b3f13a9f9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 12:49:54 +0100 Subject: [PATCH 044/114] Skip independent agent in parallel packages temporarily --- .buildkite/pipeline.trigger.integration.tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 2e0a59cccb..8e1881f783 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -78,7 +78,7 @@ done popd > /dev/null pushd test/packages/parallel > /dev/null -for independent_agent in false true; do +for independent_agent in false; do for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do package_name=$(basename "${package}") if [[ "$package_name" == "aws" || "$package_name" == "aws_logs" || "$package_name" == "gcp" ]] ; then From d22f0a4bc7c7e37fdd8acd69813efb3ddeb269f1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 12:57:16 +0100 Subject: [PATCH 045/114] Add tags in kubernetes agentdeployer --- .../_static/elastic-agent-managed.yaml.tmpl | 2 ++ internal/agentdeployer/kubernetes.go | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl b/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl index c2ce38713a..d868087f6a 100644 --- a/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl +++ b/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl @@ -63,6 +63,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: ELASTIC_AGENT_TAGS + value: "{{ .elasticAgentTags }}" securityContext: runAsUser: 0 resources: diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index 1b93e65a75..47c9948ae6 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -52,7 +52,7 @@ type kubernetesDeployedAgent struct { } func (s kubernetesDeployedAgent) TearDown(ctx context.Context) error { - elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion) + elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion, s.agentInfo.Tags) if err != nil { return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) } @@ -114,7 +114,7 @@ func (ksd KubernetesAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInf if ksd.runTearDown || ksd.runTestsOnly { logger.Debug("Skip install Elastic Agent in cluster") } else { - err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion) + err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion, agentInfo.Tags) if err != nil { return nil, fmt.Errorf("can't install Elastic-Agent in the Kubernetes cluster: %w", err) } @@ -135,10 +135,10 @@ func (ksd KubernetesAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInf var _ AgentDeployer = new(KubernetesAgentDeployer) -func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, stackVersion string) error { +func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, stackVersion string, tags []string) error { logger.Debug("install Elastic Agent in the Kubernetes cluster") - elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion) + elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion, tags) if err != nil { return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) } @@ -156,7 +156,7 @@ func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, //go:embed _static/elastic-agent-managed.yaml.tmpl var elasticAgentManagedYamlTmpl string -func getElasticAgentYAML(profile *profile.Profile, stackVersion string) ([]byte, error) { +func getElasticAgentYAML(profile *profile.Profile, stackVersion string, tags []string) ([]byte, error) { logger.Debugf("Prepare YAML definition for Elastic Agent running in stack v%s", stackVersion) appConfig, err := install.Configuration() @@ -178,6 +178,7 @@ func getElasticAgentYAML(profile *profile.Profile, stackVersion string) ([]byte, "caCertPem": caCert, "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion), + "elasticAgentTags": strings.Join(tags, ","), }) if err != nil { return nil, fmt.Errorf("can't generate elastic agent manifest: %w", err) From 51316d5ba81ecf4ef731fc1f775a556c3d25b9b2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 13:14:57 +0100 Subject: [PATCH 046/114] Assign tags for custom agent (servicedeployer) --- .../_static/docker-custom-agent-base.yml | 1 + internal/servicedeployer/custom_agent.go | 2 ++ internal/servicedeployer/info.go | 12 ++++++---- internal/testrunner/runners/system/runner.go | 23 ++++++++++++------- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/internal/servicedeployer/_static/docker-custom-agent-base.yml b/internal/servicedeployer/_static/docker-custom-agent-base.yml index 039bb49e3e..f0f4dbc54e 100644 --- a/internal/servicedeployer/_static/docker-custom-agent-base.yml +++ b/internal/servicedeployer/_static/docker-custom-agent-base.yml @@ -11,6 +11,7 @@ services: - FLEET_ENROLL=1 - FLEET_URL=https://fleet-server:8220 - KIBANA_HOST=https://kibana:5601 + - ELASTIC_AGENT_TAGS=${ELASTIC_AGENT_TAGS} volumes: - ${SERVICE_LOGS_DIR}:/tmp/service_logs/ - ${LOCAL_CA_CERT}:/etc/ssl/certs/elastic-package.pem diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index 30c4b6b6d9..1e6df460aa 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/configuration/locations" @@ -91,6 +92,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D fmt.Sprintf("%s=%s", serviceLogsDirEnv, svcInfo.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), fmt.Sprintf("%s=%s-%s", agentHostnameEnv, dockerCustomAgentName, d.agentName()), + fmt.Sprintf("%s=%s", elasticAgentTagsEnv, strings.Join(svcInfo.Tags, ",")), ) configDir, err := d.installDockerfile() diff --git a/internal/servicedeployer/info.go b/internal/servicedeployer/info.go index 3f8a24135a..505cebed82 100644 --- a/internal/servicedeployer/info.go +++ b/internal/servicedeployer/info.go @@ -5,10 +5,11 @@ package servicedeployer const ( - localCACertEnv = "LOCAL_CA_CERT" - serviceLogsDirEnv = "SERVICE_LOGS_DIR" - testRunIDEnv = "TEST_RUN_ID" - agentHostnameEnv = "AGENT_HOSTNAME" + localCACertEnv = "LOCAL_CA_CERT" + serviceLogsDirEnv = "SERVICE_LOGS_DIR" + testRunIDEnv = "TEST_RUN_ID" + agentHostnameEnv = "AGENT_HOSTNAME" + elasticAgentTagsEnv = "ELASTIC_AGENT_TAGS" ) // ServiceInfo encapsulates context that is both available to a ServiceDeployer and @@ -68,6 +69,9 @@ type ServiceInfo struct { // Directory to store any outputs generated OutputDir string + + // Tags assigned to the agent + Tags []string } // Aliases method returned aliases to properties of the service context. diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index bcd9ae75f8..614d1ef6cc 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -297,10 +297,18 @@ func (r *runner) createServiceOptions(variantName string) servicedeployer.Factor } } -func (r *runner) createAgentInfo(scenario scenarioTest) (agentdeployer.AgentInfo, error) { +func (r *runner) createAgentInfo() (agentdeployer.AgentInfo, error) { var info agentdeployer.AgentInfo - dirPath, err := agentdeployer.CreateServiceLogsDir(r.locationManager, fmt.Sprintf("agent-%s-%s", scenario.pkgManifest.Name, scenario.dataStreamManifest.Name)) + info.Tags = append(info.Tags, "test", "system", info.Test.RunID, r.options.TestFolder.Package) + folderName := fmt.Sprintf("agent-%s", r.options.TestFolder.Package) + + if r.options.TestFolder.DataStream != "" { + folderName = fmt.Sprintf("%s-%s", folderName, r.options.TestFolder.DataStream) + info.Tags = append(info.Tags, r.options.TestFolder.DataStream) + } + + dirPath, err := agentdeployer.CreateServiceLogsDir(r.locationManager, folderName) if err != nil { return agentdeployer.AgentInfo{}, fmt.Errorf("failed to create service logs dir: %w", err) } @@ -309,11 +317,6 @@ func (r *runner) createAgentInfo(scenario scenarioTest) (agentdeployer.AgentInfo info.Logs.Folder.Agent = ServiceLogsAgentDir info.Test.RunID = createTestRunID() - info.Tags = append(info.Tags, "test", "system", scenario.pkgManifest.Name, info.Test.RunID) - if scenario.dataStreamManifest != nil { - info.Tags = append(info.Tags, scenario.dataStreamManifest.Name) - } - return info, nil } @@ -330,6 +333,10 @@ func (r *runner) createServiceInfo() (servicedeployer.ServiceInfo, error) { } svcInfo.OutputDir = outputDir + svcInfo.Tags = append(svcInfo.Tags, "test", "system", svcInfo.Test.RunID, r.options.TestFolder.Package) + if r.options.TestFolder.DataStream != "" { + svcInfo.Tags = append(svcInfo.Tags, r.options.TestFolder.DataStream) + } return svcInfo, nil } @@ -742,7 +749,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, servic // Setup agent logger.Debug("setting up agent...") agentOptions := r.createAgentOptions(scenario) - agentInfo, err := r.createAgentInfo(scenario) + agentInfo, err := r.createAgentInfo() if err != nil { return nil, err } From 35f6ff45294967acc67b28e444aa98de79b065cd Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 13:18:53 +0100 Subject: [PATCH 047/114] Fix double quotes in label --- .buildkite/pipeline.trigger.integration.tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 8e1881f783..b1ee706730 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -85,7 +85,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echoerr "Skip temporarily ${package_name}" continue fi - echo " - label: \":go: Running integration test: ${package_name}\" - independent_agent ${independent_agent}" + echo " - label: \":go: Running integration test: ${package_name} - independent_agent ${independent_agent}\"" echo " key: \"integration-parallel-${package_name}-agent-${independent_agent}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" echo " env:" From 5bf00e7693427a27011816190aef11126d8c6a40 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 13:37:46 +0100 Subject: [PATCH 048/114] Fix name for environment array --- .buildkite/pipeline.trigger.integration.tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index b1ee706730..17fdbb8219 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -47,7 +47,7 @@ for test in "${CHECK_PACKAGES_TESTS[@]}"; do echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" - echo " environment:" + echo " env:" echo " ELASTIC_PACKAGE_INDEPENDENT_AGENT: ${independent_agent}" echo " artifact_paths:" echo " - build/test-results/*.xml" From ec852760024db72cf6f7a03e96b559102b713bdc Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 13:38:58 +0100 Subject: [PATCH 049/114] Fix empty RunID value --- internal/testrunner/runners/system/runner.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 614d1ef6cc..99c9b15972 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -300,6 +300,10 @@ func (r *runner) createServiceOptions(variantName string) servicedeployer.Factor func (r *runner) createAgentInfo() (agentdeployer.AgentInfo, error) { var info agentdeployer.AgentInfo + info.Name = r.options.TestFolder.Package + info.Logs.Folder.Agent = ServiceLogsAgentDir + info.Test.RunID = createTestRunID() + info.Tags = append(info.Tags, "test", "system", info.Test.RunID, r.options.TestFolder.Package) folderName := fmt.Sprintf("agent-%s", r.options.TestFolder.Package) @@ -312,10 +316,7 @@ func (r *runner) createAgentInfo() (agentdeployer.AgentInfo, error) { if err != nil { return agentdeployer.AgentInfo{}, fmt.Errorf("failed to create service logs dir: %w", err) } - info.Name = r.options.TestFolder.Package info.Logs.Folder.Local = dirPath - info.Logs.Folder.Agent = ServiceLogsAgentDir - info.Test.RunID = createTestRunID() return info, nil } From 398044624fe606dcd4a07787ddf8cc4a44c0b5f8 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 13:50:01 +0100 Subject: [PATCH 050/114] Set variant in agent Options --- internal/testrunner/runners/system/runner.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 99c9b15972..a47a6d9ff3 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -263,7 +263,7 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes return result.WithSuccess() } -func (r *runner) createAgentOptions(scenario scenarioTest) agentdeployer.FactoryOptions { +func (r *runner) createAgentOptions(variantName string) agentdeployer.FactoryOptions { return agentdeployer.FactoryOptions{ Profile: r.options.Profile, PackageRootPath: r.options.PackageRootPath, @@ -271,8 +271,9 @@ func (r *runner) createAgentOptions(scenario scenarioTest) agentdeployer.Factory DevDeployDir: DevDeployDir, Type: agentdeployer.TypeTest, StackVersion: r.stackVersion.Version(), - PackageName: scenario.pkgManifest.Name, - DataStream: scenario.dataStreamManifest.Name, + PackageName: r.options.TestFolder.Package, + DataStream: r.options.TestFolder.DataStream, + Variant: variantName, RunTearDown: r.options.RunTearDown, RunTestsOnly: r.options.RunTestsOnly, RunSetup: r.options.RunSetup, @@ -749,7 +750,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, servic } // Setup agent logger.Debug("setting up agent...") - agentOptions := r.createAgentOptions(scenario) + agentOptions := r.createAgentOptions(serviceOptions.Variant) agentInfo, err := r.createAgentInfo() if err != nil { return nil, err From 52dd175e44607111e0b54484a980d5473b903b7d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 14:54:49 +0100 Subject: [PATCH 051/114] Use exit codes instead true/false --- .buildkite/hooks/pre-command | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 571055e7dd..358bad3c5e 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -48,12 +48,12 @@ export CREATED_DATE is_step_required_to_upload_safe_logs() { if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then - return false + return 1 fi if [[ "$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives ]]; then - return true + return 0 fi - return false + return 1 } if is_step_required_to_upload_safe_logs; then @@ -64,12 +64,12 @@ fi is_step_testing_gcp () { if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then - return false + return 1 fi if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp-agent-true" ]]; then - return true + return 0 fi - return false + return 1 } if use_step_testing_gcp; then @@ -85,16 +85,17 @@ fi is_step_testing_aws () { if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then - return false + return 1 fi if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-aws-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws-agent-true" ]]; then - return true + return 0 fi if [[ "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs-agent-false" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs-agent-true" ]]; then - return true + return 0 fi - return false + return 1 } + if use_step_testing_aws; then ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) export ELASTIC_PACKAGE_AWS_SECRET_KEY From 0f1f280b39326c333aacb050780001e962d39a1e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 15:15:45 +0100 Subject: [PATCH 052/114] Rename vars --- internal/testrunner/runners/system/runner.go | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index a47a6d9ff3..656ae813fd 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -202,12 +202,12 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes } serviceOptions := r.createServiceOptions(variant) - serviceContext, err := r.createServiceInfo() + svcInfo, err := r.createServiceInfo() if err != nil { return result.WithError(err) } - testConfig, err := newConfig(configFile, serviceContext, variant) + testConfig, err := newConfig(configFile, svcInfo, variant) if err != nil { return nil, fmt.Errorf("unable to load system test case file '%s': %w", configFile, err) } @@ -224,7 +224,7 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes } result = r.newResult(fmt.Sprintf("%s - %s", resultName, testConfig.Name())) - scenario, err := r.prepareScenario(ctx, testConfig, serviceContext, serviceOptions) + scenario, err := r.prepareScenario(ctx, testConfig, svcInfo, serviceOptions) if r.options.RunSetup && err != nil { tdErr := r.tearDownTest(ctx) if tdErr != nil { @@ -535,19 +535,19 @@ func (r *runner) run(ctx context.Context) (results []testrunner.TestResult, err func (r *runner) runTestPerVariant(ctx context.Context, result *testrunner.ResultComposer, cfgFile, variantName string) ([]testrunner.TestResult, error) { serviceOptions := r.createServiceOptions(variantName) - serviceContext, err := r.createServiceInfo() + svcInfo, err := r.createServiceInfo() if err != nil { return result.WithError(err) } configFile := filepath.Join(r.options.TestFolder.Path, cfgFile) - testConfig, err := newConfig(configFile, serviceContext, variantName) + testConfig, err := newConfig(configFile, svcInfo, variantName) if err != nil { return nil, fmt.Errorf("unable to load system test case file '%s': %w", configFile, err) } logger.Debugf("Using config: %q", testConfig.Name()) - partial, err := r.runTest(ctx, testConfig, serviceContext, serviceOptions) + partial, err := r.runTest(ctx, testConfig, svcInfo, serviceOptions) tdErr := r.tearDownTest(ctx) if err != nil { @@ -707,7 +707,7 @@ type scenarioTest struct { docs []common.MapStr } -func (r *runner) prepareScenario(ctx context.Context, config *testConfig, serviceContext servicedeployer.ServiceInfo, serviceOptions servicedeployer.FactoryOptions) (*scenarioTest, error) { +func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInfo servicedeployer.ServiceInfo, serviceOptions servicedeployer.FactoryOptions) (*scenarioTest, error) { var err error var serviceStateData ServiceState if r.options.RunSetup { @@ -797,24 +797,24 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, servic } if r.options.RunIndependentElasticAgent { - serviceContext.Logs.Folder.Local = agentInfo.Logs.Folder.Local + svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local if agentDeployed != nil { // In case of CustomAgents from servicedeployer where agent and service // are deployed in the same docker-compose scenario (servicedeployer), // so there is no agentDeployed in that scenario - serviceContext.AgentHostname = agentDeployed.Info().Hostname + svcInfo.AgentHostname = agentDeployed.Info().Hostname } } else { - serviceContext.AgentHostname = "elastic-agent" + svcInfo.AgentHostname = "elastic-agent" } if config.Service != "" { - serviceContext.Name = config.Service + svcInfo.Name = config.Service } - service, err := serviceDeployer.SetUp(ctx, serviceContext) + service, err := serviceDeployer.SetUp(ctx, svcInfo) if err != nil { return nil, fmt.Errorf("could not setup service: %w", err) } - serviceContext = service.Info() + svcInfo = service.Info() r.shutdownServiceHandler = func(ctx context.Context) error { logger.Debug("tearing down service...") if err := service.TearDown(ctx); err != nil { @@ -825,7 +825,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, servic } // Reload test config with ctx variable substitution. - config, err = newConfig(config.Path, serviceContext, serviceOptions.Variant) + config, err = newConfig(config.Path, svcInfo, serviceOptions.Variant) if err != nil { return nil, fmt.Errorf("unable to reload system test case configuration: %w", err) } @@ -960,7 +960,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, servic // FIXME: running per stages does not work when multiple agents are created var origPolicy kibana.Policy - agents, err := checkEnrolledAgents(ctx, r.options.KibanaClient, agentInfo, enrollingTime, serviceContext, r.options.RunIndependentElasticAgent) + agents, err := checkEnrolledAgents(ctx, r.options.KibanaClient, agentInfo, enrollingTime, svcInfo, r.options.RunIndependentElasticAgent) if err != nil { return nil, fmt.Errorf("can't check enrolled agents: %w", err) } @@ -1276,7 +1276,7 @@ func (r *runner) validateTestScenario(ctx context.Context, result *testrunner.Re return result.WithSuccess() } -func (r *runner) runTest(ctx context.Context, config *testConfig, serviceContext servicedeployer.ServiceInfo, serviceOptions servicedeployer.FactoryOptions) ([]testrunner.TestResult, error) { +func (r *runner) runTest(ctx context.Context, config *testConfig, svcInfo servicedeployer.ServiceInfo, serviceOptions servicedeployer.FactoryOptions) ([]testrunner.TestResult, error) { result := r.newResult(config.Name()) if config.Skip != nil { @@ -1288,7 +1288,7 @@ func (r *runner) runTest(ctx context.Context, config *testConfig, serviceContext logger.Debugf("running test with configuration '%s'", config.Name()) - scenario, err := r.prepareScenario(ctx, config, serviceContext, serviceOptions) + scenario, err := r.prepareScenario(ctx, config, svcInfo, serviceOptions) if err != nil { return result.WithError(err) } From bea6a2a3f903f35864bfed5124ed89187292204b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 15:32:42 +0100 Subject: [PATCH 053/114] Use the right tags for custom agent (servicedeployer) --- .../agentdeployer/_static/custom-agent.yml | 8 -------- .../_static/docker-agent-base.yml | 2 +- internal/agentdeployer/agent.go | 20 +++++++++---------- internal/agentdeployer/compose.go | 2 +- internal/testrunner/runners/system/runner.go | 12 ++++++++--- 5 files changed, 21 insertions(+), 23 deletions(-) delete mode 100644 internal/agentdeployer/_static/custom-agent.yml diff --git a/internal/agentdeployer/_static/custom-agent.yml b/internal/agentdeployer/_static/custom-agent.yml deleted file mode 100644 index 531c24ced9..0000000000 --- a/internal/agentdeployer/_static/custom-agent.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: "2.3" -services: - docker-custom-agent: - pid: host - cap_add: - - AUDIT_CONTROL - - AUDIT_READ - user: root diff --git a/internal/agentdeployer/_static/docker-agent-base.yml b/internal/agentdeployer/_static/docker-agent-base.yml index bd2960be5f..212d7e2058 100644 --- a/internal/agentdeployer/_static/docker-agent-base.yml +++ b/internal/agentdeployer/_static/docker-agent-base.yml @@ -1,6 +1,6 @@ version: "2.3" services: - docker-custom-agent: + docker-test-agent: hostname: ${AGENT_HOSTNAME} image: "${ELASTIC_AGENT_IMAGE_REF}" healthcheck: diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 7f3a6f489d..c4cf824786 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -23,10 +23,10 @@ import ( ) const ( - dockerCustomAgentNamePrefix = "docker-custom-agent" - dockerCustomAgentDir = "docker_custom_agent" - dockerCustomAgentDockerCompose = "docker-agent-base.yml" - defaultAgentPolicyName = "Elastic-Agent (elastic-package)" + dockerTestAgentNamePrefix = "docker-test-agent" + dockerTestgentDir = "docker_test_agent" + dockerTestAgentDockerCompose = "docker-agent-base.yml" + defaultAgentPolicyName = "Elastic-Agent (elastic-package)" ) //go:embed _static/docker-agent-base.yml @@ -106,12 +106,12 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D } ymlPaths := []string{ - filepath.Join(configDir, dockerCustomAgentDockerCompose), + filepath.Join(configDir, dockerTestAgentDockerCompose), } if d.dockerComposeFile != "" { ymlPaths = []string{ d.dockerComposeFile, - filepath.Join(configDir, dockerCustomAgentDockerCompose), + filepath.Join(configDir, dockerTestAgentDockerCompose), } } @@ -121,7 +121,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D ymlPaths: ymlPaths, project: composeProjectName, variant: AgentVariant{ - Name: dockerCustomAgentNamePrefix, + Name: dockerTestAgentNamePrefix, Env: env, }, } @@ -153,7 +153,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D } // Service name defined in the docker-compose file - agentInfo.Name = dockerCustomAgentNamePrefix + agentInfo.Name = dockerTestAgentNamePrefix agentName := agentInfo.Name opts := compose.CommandOptions{ @@ -211,7 +211,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D } func (d *CustomAgentDeployer) agentHostname() string { - return fmt.Sprintf("docker-custom-agent-%s", d.agentName()) + return fmt.Sprintf("%s-%s", dockerTestAgentNamePrefix, d.agentName()) } func (d *CustomAgentDeployer) agentName() string { @@ -234,7 +234,7 @@ func (d *CustomAgentDeployer) installDockerfile() (string, error) { return "", fmt.Errorf("failed to create directory for custom agent files: %w", err) } - customAgentDockerfile := filepath.Join(customAgentDir, dockerCustomAgentDockerCompose) + customAgentDockerfile := filepath.Join(customAgentDir, dockerTestAgentDockerCompose) err = os.WriteFile(customAgentDockerfile, dockerAgentDockerComposeContent, 0644) if err != nil { return "", fmt.Errorf("failed to create docker compose file for custom agent: %w", err) diff --git a/internal/agentdeployer/compose.go b/internal/agentdeployer/compose.go index 976bb52e99..da918857d5 100644 --- a/internal/agentdeployer/compose.go +++ b/internal/agentdeployer/compose.go @@ -127,7 +127,7 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI agentInfo.Port = agentInfo.Ports[0] } - agentInfo.Agent.Host.NamePrefix = "docker-custom-agent" + agentInfo.Agent.Host.NamePrefix = dockerTestAgentNamePrefix agent.agentInfo = agentInfo return &agent, nil } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 656ae813fd..6c6e3062f2 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1717,6 +1717,12 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t logger.Debugf("filter agents using service criteria: NamePrefix=%s", svcInfo.Agent.Host.NamePrefix) } + expectedTags := agentInfo.Tags + if svcInfo.Agent.Host.NamePrefix == "docker-custom-agent" { + // custom agents are still started from servicedeployer, they are defined in the same docker compose scenario + expectedTags = svcInfo.Tags + } + // filtered list of agents must contain all agents started by the stack // they could be assigned the default policy (elastic-agent-managed-ep) or the test policy (ep-test-system-*) var filtered []kibana.Agent @@ -1743,10 +1749,10 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t } // Tags are available starting in 8.3 - if runIndependentElasticAgent && len(agent.Tags) > 0 { - logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(agentInfo.Tags, ",")) + if runIndependentElasticAgent && len(expectedTags) > 0 && len(agent.Tags) > 0 { + logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) foundAllTags := true - for _, tag := range agentInfo.Tags { + for _, tag := range expectedTags { if !slices.Contains(agent.Tags, tag) { logger.Debugf("filtered agent (invalid tag found) %s - %q vs %q", tag, strings.Join(agent.Tags, ","), strings.Join(agentInfo.Tags, ",")) // TODO: remove foundAllTags = false From 2b050b6cd47eb3ab26c1c219a221387843f7946e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 16:32:09 +0100 Subject: [PATCH 054/114] Test system flags with independent agent --- .buildkite/pipeline.trigger.integration.tests.sh | 4 ++++ scripts/test-system-test-flags.sh | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 17fdbb8219..fa5a48ed02 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -123,10 +123,14 @@ echo " provider: \"gcp\"" echo " artifact_paths:" echo " - build/elastic-stack-dump/install-zip-shellinit/logs/*.log" +for independent_agent in false true; do echo " - label: \":go: Running integration test: test-system-test-flags\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-system-test-flags" echo " agents:" echo " provider: \"gcp\"" +echo " env:" +echo " ELASTIC_PACKAGE_INDEPENDENT_AGENT: ${independent_agent}" +done echo " - label: \":go: Running integration test: test-profiles-command\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-profiles-command" diff --git a/scripts/test-system-test-flags.sh b/scripts/test-system-test-flags.sh index c2aca5ae06..6a32f1ab77 100755 --- a/scripts/test-system-test-flags.sh +++ b/scripts/test-system-test-flags.sh @@ -201,6 +201,11 @@ run_tests_for_package() { AGENT_CONTAINER_NAME="${DEFAULT_AGENT_CONTAINER_NAME}" fi + ELASTIC_PACKAGE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_INDEPENDENT_AGENT:-"false"} + ELASTIC_PACKAGE_TEST_OPTS="" + if [[ "${ELASTIC_PACKAGE_INDEPENDENT_AGENT}" == "true" ]] ; then + ELASTIC_PACKAGE_TEST_OPTS="--test-independent-agent" + fi pushd "${package_folder}" > /dev/null echo "--- [${package_name} - ${variant}] Setup service without tear-down" @@ -208,6 +213,7 @@ run_tests_for_package() { --report-format xUnit --report-output file \ --config-file "${config_file}" \ ${variant_flag} \ + ${ELASTIC_PACKAGE_TEST_OPTS} \ --setup # Tests after --setup @@ -223,6 +229,7 @@ run_tests_for_package() { echo "--- Iteration #${i} --no-provision" elastic-package test system -v \ --report-format xUnit --report-output file \ + ${ELASTIC_PACKAGE_TEST_OPTS} \ --no-provision # Tests after --no-provision @@ -237,6 +244,7 @@ run_tests_for_package() { echo "--- [${package_name} - ${variant}] Run tear-down process" elastic-package test system -v \ --report-format xUnit --report-output file \ + ${ELASTIC_PACKAGE_TEST_OPTS} \ --tear-down if ! tests_for_tear_down \ From 6dc4fce036157dca69aa2e251d6da7b1c25e43b5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 16:33:30 +0100 Subject: [PATCH 055/114] Shorten label titles --- .../pipeline.trigger.integration.tests.sh | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index fa5a48ed02..c27dc0f5c1 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -24,7 +24,7 @@ STACK_COMMAND_TESTS=( ) for test in "${STACK_COMMAND_TESTS[@]}"; do - echo " - label: \":go: Running integration test: ${test}\"" + echo " - label: \":go: Integration test: ${test}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" @@ -43,7 +43,7 @@ CHECK_PACKAGES_TESTS=( ) for independent_agent in false true ; do for test in "${CHECK_PACKAGES_TESTS[@]}"; do - echo " - label: \":go: Running integration test: ${test} - independent_agent ${independent_agent}\"" + echo " - label: \":go: Integration test: ${test} - independent_agent ${independent_agent}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" @@ -63,7 +63,7 @@ done pushd test/packages/false_positives > /dev/null for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do package_name=$(basename "${package}") - echo " - label: \":go: Running integration test (false positive): ${package_name}\"" + echo " - label: \":go: Integration test (false positive): ${package_name}\"" echo " key: \"integration-false_positives-${package_name}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}" echo " env:" @@ -85,7 +85,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echoerr "Skip temporarily ${package_name}" continue fi - echo " - label: \":go: Running integration test: ${package_name} - independent_agent ${independent_agent}\"" + echo " - label: \":go: Integration test: ${package_name} - independent_agent ${independent_agent}\"" echo " key: \"integration-parallel-${package_name}-agent-${independent_agent}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" echo " env:" @@ -101,7 +101,7 @@ done popd > /dev/null -echo " - label: \":go: Running integration test: test-build-zip\"" +echo " - label: \":go: Integration test: test-build-zip\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-build-zip" echo " agents:" echo " provider: \"gcp\"" @@ -109,14 +109,14 @@ echo " artifact_paths:" echo " - build/elastic-stack-dump/build-zip/logs/*.log" echo " - build/packages/*.sig" -echo " - label: \":go: Running integration test: test-install-zip\"" +echo " - label: \":go: Integration test: test-install-zip\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-install-zip" echo " agents:" echo " provider: \"gcp\"" echo " artifact_paths:" echo " - build/elastic-stack-dump/install-zip/logs/*.log" -echo " - label: \":go: Running integration test: test-install-zip-shellinit\"" +echo " - label: \":go: Integration test: test-install-zip-shellinit\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-install-zip-shellinit" echo " agents:" echo " provider: \"gcp\"" @@ -124,7 +124,7 @@ echo " artifact_paths:" echo " - build/elastic-stack-dump/install-zip-shellinit/logs/*.log" for independent_agent in false true; do -echo " - label: \":go: Running integration test: test-system-test-flags\"" +echo " - label: \":go: Integration test: test-system-test-flags\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-system-test-flags" echo " agents:" echo " provider: \"gcp\"" @@ -132,7 +132,7 @@ echo " env:" echo " ELASTIC_PACKAGE_INDEPENDENT_AGENT: ${independent_agent}" done -echo " - label: \":go: Running integration test: test-profiles-command\"" +echo " - label: \":go: Integration test: test-profiles-command\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-profiles-command" echo " env:" echo " DOCKER_COMPOSE_VERSION: \"false\"" @@ -142,7 +142,7 @@ echo " image: \"${LINUX_AGENT_IMAGE}\"" echo " cpu: \"8\"" echo " memory: \"4G\"" -echo " - label: \":go: Running integration test: test-check-update-version\"" +echo " - label: \":go: Integration test: test-check-update-version\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-update-version" echo " env:" echo " DEFAULT_VERSION_TAG: v0.80.0" From 3349ac76354e6e470e63d23bfe7aaffff223117e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 21 Mar 2024 17:09:59 +0100 Subject: [PATCH 056/114] Add reference to agent in label title --- .buildkite/pipeline.trigger.integration.tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index c27dc0f5c1..5ba9dd5146 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -124,7 +124,7 @@ echo " artifact_paths:" echo " - build/elastic-stack-dump/install-zip-shellinit/logs/*.log" for independent_agent in false true; do -echo " - label: \":go: Integration test: test-system-test-flags\"" +echo " - label: \":go: Integration test: test-system-test-flags - independent_agent ${independent_agent}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-system-test-flags" echo " agents:" echo " provider: \"gcp\"" From 9584449f8e44c262e24a9112cf90eab0237d3ed3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 11:09:24 +0100 Subject: [PATCH 057/114] Check logs from agents - just docker-compose --- internal/agentdeployer/compose.go | 17 +++++ internal/agentdeployer/deployed_agent.go | 4 ++ internal/agentdeployer/kubernetes.go | 6 ++ internal/testrunner/runners/system/runner.go | 68 +++++++++++++++++++- 4 files changed, 94 insertions(+), 1 deletion(-) diff --git a/internal/agentdeployer/compose.go b/internal/agentdeployer/compose.go index da918857d5..3f8a347d06 100644 --- a/internal/agentdeployer/compose.go +++ b/internal/agentdeployer/compose.go @@ -174,6 +174,23 @@ func (s *dockerComposeDeployedAgent) ExitCode(ctx context.Context, agent string) return p.ServiceExitCode(ctx, agent, opts) } +// Logs returns the logs from the agent starting at the given time +func (s *dockerComposeDeployedAgent) Logs(ctx context.Context, t time.Time) ([]byte, error) { + p, err := compose.NewProject(s.project, s.ymlPaths...) + if err != nil { + return nil, fmt.Errorf("could not create Docker Compose project for agent: %w", err) + } + + opts := compose.CommandOptions{ + Env: append( + s.env, + s.variant.Env..., + ), + } + + return p.Logs(ctx, opts) +} + // TearDown tears down the agent. func (s *dockerComposeDeployedAgent) TearDown(ctx context.Context) error { logger.Debugf("tearing down agent using Docker Compose runner") diff --git a/internal/agentdeployer/deployed_agent.go b/internal/agentdeployer/deployed_agent.go index 69084af194..4a036d8584 100644 --- a/internal/agentdeployer/deployed_agent.go +++ b/internal/agentdeployer/deployed_agent.go @@ -7,6 +7,7 @@ package agentdeployer import ( "context" "errors" + "time" ) var ErrNotSupported error = errors.New("not supported") @@ -27,4 +28,7 @@ type DeployedAgent interface { // ExitCode returns true if the service is exited and its exit code. ExitCode(ctx context.Context, service string) (bool, int, error) + + // Logs returns the logs from the agent starting at the given time + Logs(ctx context.Context, t time.Time) ([]byte, error) } diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index 47c9948ae6..367806799a 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -13,6 +13,7 @@ import ( "os" "strings" "text/template" + "time" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kind" @@ -80,6 +81,11 @@ func (s *kubernetesDeployedAgent) SetInfo(sc AgentInfo) error { return nil } +// Logs returns the logs from the agent starting at the given time +func (s *kubernetesDeployedAgent) Logs(ctx context.Context, t time.Time) ([]byte, error) { + return nil, nil +} + var _ DeployedAgent = new(kubernetesDeployedAgent) // NewKubernetesAgentDeployer function creates a new instance of KubernetesAgentDeployer. diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 6c6e3062f2..6549e6dbcb 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -705,6 +705,8 @@ type scenarioTest struct { kibanaDataStream kibana.PackageDataStream syntheticEnabled bool docs []common.MapStr + agent agentdeployer.DeployedAgent + enrollingTime time.Time } func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInfo servicedeployer.ServiceInfo, serviceOptions servicedeployer.FactoryOptions) (*scenarioTest, error) { @@ -784,6 +786,8 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf return nil } + scenario.agent = agentDeployed + scenario.enrollingTime = enrollingTime } // Setup service. @@ -1273,6 +1277,16 @@ func (r *runner) validateTestScenario(ctx context.Context, result *testrunner.Re return result.WithError(err) } + if scenario.agent != nil { + logResults, err := r.checkNewAgentLogs(ctx, scenario.agent, scenario.enrollingTime, errorPatterns) + if err != nil { + return result.WithError(err) + } + if len(logResults) > 0 { + return logResults, nil + } + } + return result.WithSuccess() } @@ -1754,7 +1768,7 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t foundAllTags := true for _, tag := range expectedTags { if !slices.Contains(agent.Tags, tag) { - logger.Debugf("filtered agent (invalid tag found) %s - %q vs %q", tag, strings.Join(agent.Tags, ","), strings.Join(agentInfo.Tags, ",")) // TODO: remove + logger.Debugf("filtered agent (invalid tag found) %s - %q vs %q", tag, strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) // TODO: remove foundAllTags = false break } @@ -1871,6 +1885,58 @@ func (r *runner) generateTestResult(docs []common.MapStr, specVersion semver.Ver return nil } +func (r *runner) checkNewAgentLogs(ctx context.Context, agent agentdeployer.DeployedAgent, startTesting time.Time, errorPatterns []logsByContainer) (results []testrunner.TestResult, err error) { + if agent == nil { + return nil, nil + } + + f, err := os.CreateTemp("", "elastic-agent.logs") + if err != nil { + return nil, fmt.Errorf("failed to create temp file for logs: %w", err) + } + defer os.Remove(f.Name()) + + for _, patternsContainer := range errorPatterns { + if patternsContainer.containerName != "elastic-agent" { + continue + } + + startTime := time.Now() + + outputBytes, err := agent.Logs(ctx, startTesting) + if err != nil { + return nil, fmt.Errorf("check log messages failed: %s", err) + } + _, err = f.Write(outputBytes) + if err != nil { + return nil, fmt.Errorf("write log messages failed: %s", err) + } + + err = r.anyErrorMessages(f.Name(), startTesting, patternsContainer.patterns) + if e, ok := err.(testrunner.ErrTestCaseFailed); ok { + tr := testrunner.TestResult{ + TestType: TestType, + Name: fmt.Sprintf("(%s logs)", patternsContainer.containerName), + Package: r.options.TestFolder.Package, + DataStream: r.options.TestFolder.DataStream, + } + tr.FailureMsg = e.Error() + tr.FailureDetails = e.Details + tr.TimeElapsed = time.Since(startTime) + results = append(results, tr) + // Just check elastic-agent + break + } + + if err != nil { + return nil, fmt.Errorf("check log messages failed: %s", err) + } + // Just check elastic-agent + break + } + return results, nil +} + func (r *runner) checkAgentLogs(dumpOptions stack.DumpOptions, startTesting time.Time, errorPatterns []logsByContainer) (results []testrunner.TestResult, err error) { for _, patternsContainer := range errorPatterns { startTime := time.Now() From d8be15978368b4f35cd561e325140d6648388fa5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 12:25:05 +0100 Subject: [PATCH 058/114] Check the right tags in no-provision or tear-down stages --- internal/testrunner/runners/system/runner.go | 34 +++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 6549e6dbcb..a21fce491a 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -762,6 +762,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf if r.options.RunTearDown || r.options.RunTestsOnly { enrollingTime = serviceStateData.EnrollingAgentTime agentInfo.Tags = serviceStateData.Agent.Tags + svcInfo.Tags = serviceStateData.Agent.Tags } agentDeployer, err := agentdeployer.Factory(agentOptions) if err != nil { @@ -1102,7 +1103,14 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf scenario.docs = hits.getDocs(scenario.syntheticEnabled) if r.options.RunSetup { - err = r.writeScenarioState(policy, &origPolicy, config, origAgent, enrollingTime) + opts := scenarioStateOpts{ + origPolicy: &origPolicy, + currentPolicy: policy, + config: config, + agent: origAgent, + enrollingTime: enrollingTime, + } + err = r.writeScenarioState(opts) if err != nil { return nil, err } @@ -1149,16 +1157,26 @@ type ServiceState struct { ConfigFilePath string `json:"config_file_path"` VariantName string `json:"variant_name"` EnrollingAgentTime time.Time `json:"enrolling_agent_time"` + ServiceInfoTags []string `json:"service_info_tags,omitempty"` + AgentInfoTags []string `json:"agent_info_tags,omitempty"` } -func (r *runner) writeScenarioState(currentPolicy, origPolicy *kibana.Policy, config *testConfig, agent kibana.Agent, enrollingTime time.Time) error { +type scenarioStateOpts struct { + currentPolicy *kibana.Policy + origPolicy *kibana.Policy + config *testConfig + agent kibana.Agent + enrollingTime time.Time +} + +func (r *runner) writeScenarioState(opts scenarioStateOpts) error { data := ServiceState{ - OrigPolicy: *origPolicy, - CurrentPolicy: *currentPolicy, - Agent: agent, - ConfigFilePath: config.Path, - VariantName: config.ServiceVariantName, - EnrollingAgentTime: enrollingTime, + OrigPolicy: *opts.origPolicy, + CurrentPolicy: *opts.currentPolicy, + Agent: opts.agent, + ConfigFilePath: opts.config.Path, + VariantName: opts.config.ServiceVariantName, + EnrollingAgentTime: opts.enrollingTime, } dataBytes, err := json.Marshal(data) if err != nil { From 8cfd1b47b6a6501e4503764937c916ddd10e04d4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 12:26:20 +0100 Subject: [PATCH 059/114] Fail step if test command fails (system flags) --- scripts/test-system-test-flags.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/test-system-test-flags.sh b/scripts/test-system-test-flags.sh index 6a32f1ab77..9a579250b3 100755 --- a/scripts/test-system-test-flags.sh +++ b/scripts/test-system-test-flags.sh @@ -209,12 +209,14 @@ run_tests_for_package() { pushd "${package_folder}" > /dev/null echo "--- [${package_name} - ${variant}] Setup service without tear-down" - elastic-package test system -v \ + if ! elastic-package test system -v \ --report-format xUnit --report-output file \ --config-file "${config_file}" \ ${variant_flag} \ ${ELASTIC_PACKAGE_TEST_OPTS} \ - --setup + --setup ; then + return 1 + fi # Tests after --setup if ! tests_for_setup \ @@ -227,10 +229,12 @@ run_tests_for_package() { echo "--- [${package_name} - ${variant}] Run tests without provisioning" for i in $(seq 2); do echo "--- Iteration #${i} --no-provision" - elastic-package test system -v \ + if ! elastic-package test system -v \ --report-format xUnit --report-output file \ ${ELASTIC_PACKAGE_TEST_OPTS} \ - --no-provision + --no-provision ; then + return 1 + fi # Tests after --no-provision if ! tests_for_no_provision \ @@ -242,10 +246,12 @@ run_tests_for_package() { done echo "--- [${package_name} - ${variant}] Run tear-down process" - elastic-package test system -v \ + if ! elastic-package test system -v \ --report-format xUnit --report-output file \ ${ELASTIC_PACKAGE_TEST_OPTS} \ - --tear-down + --tear-down ; then + return 1 + fi if ! tests_for_tear_down \ "${service_deployer_type}" \ From dce59dc6aee2fdae21c9ac19a9b8c41c76b72d3d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 13:05:11 +0100 Subject: [PATCH 060/114] Support Elastic stack 7,* --- internal/agentdeployer/agent.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index c4cf824786..08e7787f01 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -91,11 +91,20 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D return nil, fmt.Errorf("can't locate CA certificate: %w", err) } + // Local Elastic stacks have a default Agent Policy created, + // but Cloud or Serverless Projects could not have one + agentPolicyName := defaultAgentPolicyName + if strings.HasPrefix(d.stackVersion, "7.") { + // Local Elastic stacks 7.* have an Agent Policy that is set as default + // No need to set an Agent Policy Name + agentPolicyName = "" + } + env := append( appConfig.StackImageRefs(d.stackVersion).AsEnv(), fmt.Sprintf("%s=%s", serviceLogsDirEnv, agentInfo.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), - fmt.Sprintf("%s=%s", fleetPolicyEnv, defaultAgentPolicyName), + fmt.Sprintf("%s=%s", fleetPolicyEnv, agentPolicyName), fmt.Sprintf("%s=%s", agentHostnameEnv, d.agentHostname()), fmt.Sprintf("%s=%s", elasticAgentTagsEnv, strings.Join(agentInfo.Tags, ",")), ) From 4267d7ba451e0bcbee79a144cf683aa758e6a287 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 13:44:46 +0100 Subject: [PATCH 061/114] Add note --- internal/agentdeployer/agent.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 08e7787f01..f1e777aaf1 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -194,7 +194,9 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D } // Build agent container name - // agentInfo.Hostname = p.ContainerName(serviceName) + // For those packages that require to do requests to agent ports in their tests (e.g. ti_anomali), + // using the ContainerName of the agent (p.ContainerName(agentName)) as in servicedeployer does not work, + // probably because it is in another compose project in case of ti_anomali?. agentInfo.Hostname = d.agentHostname() logger.Debugf("adding service container %s internal ports to context", p.ContainerName(agentName)) From 536cfb559cc5a151290e27d62574ba2b1d539f5f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 13:46:16 +0100 Subject: [PATCH 062/114] Add helper --- internal/servicedeployer/custom_agent.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index 1e6df460aa..92f54916cc 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -91,7 +91,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D appConfig.StackImageRefs(d.stackVersion).AsEnv(), fmt.Sprintf("%s=%s", serviceLogsDirEnv, svcInfo.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), - fmt.Sprintf("%s=%s-%s", agentHostnameEnv, dockerCustomAgentName, d.agentName()), + fmt.Sprintf("%s=%s", agentHostnameEnv, d.agentHostname()), fmt.Sprintf("%s=%s", elasticAgentTagsEnv, strings.Join(svcInfo.Tags, ",")), ) @@ -195,10 +195,18 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D } func (d *CustomAgentDeployer) agentName() string { + name := d.packageName if d.variant.Name != "" { - return fmt.Sprintf("%s-%s-%s", d.packageName, d.variant.Name, d.dataStream) + name = fmt.Sprintf("%s-%s", name, d.variant.Name) } - return fmt.Sprintf("%s-%s", d.packageName, d.dataStream) + if d.dataStream != "" && d.dataStream != "." { + name = fmt.Sprintf("%s-%s", name, d.dataStream) + } + return name +} + +func (d *CustomAgentDeployer) agentHostname() string { + return fmt.Sprintf("%s-%s", dockerCustomAgentName, d.agentName()) } // installDockerfile creates the files needed to run the custom elastic agent and returns From 702bf93d3ce5ab0dcf2ad0e399a90012c0216ec7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 13:48:55 +0100 Subject: [PATCH 063/114] Fill package and datastream names in service factoryOptions --- internal/testrunner/runners/system/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index a21fce491a..07727cb78c 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -289,8 +289,8 @@ func (r *runner) createServiceOptions(variantName string) servicedeployer.Factor Variant: variantName, Type: servicedeployer.TypeTest, StackVersion: r.stackVersion.Version(), - PackageName: "", // to be filled in prepareScenario - DataStream: "", // to be filled in prepareScenario + PackageName: r.options.TestFolder.Package, + DataStream: r.options.TestFolder.DataStream, RunTearDown: r.options.RunTearDown, RunTestsOnly: r.options.RunTestsOnly, RunSetup: r.options.RunSetup, From 606491a10c00f845840ab18ee1ced2504111d334 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 16:39:39 +0100 Subject: [PATCH 064/114] Remove usages of tags for backwards compatibility --- internal/agentdeployer/agent.go | 5 +- internal/servicedeployer/compose.go | 2 + internal/servicedeployer/custom_agent.go | 11 ++- internal/testrunner/runners/system/runner.go | 92 +++++++++++++------- 4 files changed, 74 insertions(+), 36 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index f1e777aaf1..8f4a96e953 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -41,6 +41,8 @@ type CustomAgentDeployer struct { variant AgentVariant + agentRunID string + packageName string dataStream string @@ -80,6 +82,7 @@ func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDep // SetUp sets up the service and returns any relevant information. func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (DeployedAgent, error) { logger.Debug("setting up service using Docker Compose agent deployer") + d.agentRunID = agentInfo.Test.RunID appConfig, err := install.Configuration() if err != nil { @@ -222,7 +225,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D } func (d *CustomAgentDeployer) agentHostname() string { - return fmt.Sprintf("%s-%s", dockerTestAgentNamePrefix, d.agentName()) + return fmt.Sprintf("%s-%s-%s", dockerTestAgentNamePrefix, d.agentName(), d.agentRunID) } func (d *CustomAgentDeployer) agentName() string { diff --git a/internal/servicedeployer/compose.go b/internal/servicedeployer/compose.go index 2c5e492d0b..793c9c0011 100644 --- a/internal/servicedeployer/compose.go +++ b/internal/servicedeployer/compose.go @@ -74,6 +74,8 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic variant: d.variant, env: []string{ fmt.Sprintf("%s=%s", serviceLogsDirEnv, svcInfo.Logs.Folder.Local), + // Hostname envioronment varible is required since some packages require to run + // queries to the elastic-agent container (e.g. ti_anomali) fmt.Sprintf("%s=%s", agentHostnameEnv, svcInfo.AgentHostname), }, } diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index 92f54916cc..0b7867de1f 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -41,6 +41,8 @@ type CustomAgentDeployer struct { packageName string dataStream string + agentRunID string + runTearDown bool runTestsOnly bool } @@ -77,6 +79,8 @@ func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDep func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (DeployedService, error) { logger.Debug("setting up service using Docker Compose service deployer") + d.agentRunID = svcInfo.Test.RunID + appConfig, err := install.Configuration() if err != nil { return nil, fmt.Errorf("can't read application configuration: %w", err) @@ -170,7 +174,10 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D } // Build service container name - svcInfo.Hostname = p.ContainerName(serviceName) + // Set the same hostname as in the docker-compose (environment variable) + //svcInfo.Hostname = p.ContainerName(serviceName) + svcInfo.Hostname = d.agentHostname() + svcInfo.AgentHostname = d.agentHostname() logger.Debugf("adding service container %s internal ports to context", p.ContainerName(serviceName)) serviceComposeConfig, err := p.Config(ctx, compose.CommandOptions{Env: env}) @@ -206,7 +213,7 @@ func (d *CustomAgentDeployer) agentName() string { } func (d *CustomAgentDeployer) agentHostname() string { - return fmt.Sprintf("%s-%s", dockerCustomAgentName, d.agentName()) + return fmt.Sprintf("%s-%s-%s", dockerCustomAgentName, d.agentName(), d.agentRunID) } // installDockerfile creates the files needed to run the custom elastic agent and returns diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 07727cb78c..dbe3761eb1 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -761,8 +761,15 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf enrollingTime := time.Now() if r.options.RunTearDown || r.options.RunTestsOnly { enrollingTime = serviceStateData.EnrollingAgentTime + agentInfo.Tags = serviceStateData.Agent.Tags + agentInfo.Test.RunID = serviceStateData.AgentRunID + agentInfo.Hostname = serviceStateData.AgentHostname + svcInfo.Tags = serviceStateData.Agent.Tags + svcInfo.Test.RunID = serviceStateData.ServiceRunID + svcInfo.AgentHostname = serviceStateData.ServiceAgentHostname + svcInfo.Hostname = serviceStateData.ServiceAgentHostname } agentDeployer, err := agentdeployer.Factory(agentOptions) if err != nil { @@ -775,6 +782,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf if err != nil { return nil, fmt.Errorf("could not setup agent: %w", err) } + agentInfo = agentDeployed.Info() } r.shutdownAgentHandler = func(ctx context.Context) error { if agentDeployer == nil { @@ -801,16 +809,15 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf return nil, fmt.Errorf("could not create service runner: %w", err) } + svcInfo.AgentHostname = "elastic-agent" if r.options.RunIndependentElasticAgent { svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local if agentDeployed != nil { - // In case of CustomAgents from servicedeployer where agent and service - // are deployed in the same docker-compose scenario (servicedeployer), - // so there is no agentDeployed in that scenario + // Not all agents are created from "agentdeployer", currently agent and + // service containers are created from "servicedeployer" package for + // CustomAgent scenario svcInfo.AgentHostname = agentDeployed.Info().Hostname } - } else { - svcInfo.AgentHostname = "elastic-agent" } if config.Service != "" { svcInfo.Name = config.Service @@ -1109,6 +1116,8 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf config: config, agent: origAgent, enrollingTime: enrollingTime, + agentInfo: agentInfo, + svcInfo: svcInfo, } err = r.writeScenarioState(opts) if err != nil { @@ -1151,14 +1160,18 @@ func (r *runner) readServiceStateData() (ServiceState, error) { } type ServiceState struct { - OrigPolicy kibana.Policy `json:"orig_policy"` - CurrentPolicy kibana.Policy `json:"current_policy"` - Agent kibana.Agent `json:"agent"` - ConfigFilePath string `json:"config_file_path"` - VariantName string `json:"variant_name"` - EnrollingAgentTime time.Time `json:"enrolling_agent_time"` - ServiceInfoTags []string `json:"service_info_tags,omitempty"` - AgentInfoTags []string `json:"agent_info_tags,omitempty"` + OrigPolicy kibana.Policy `json:"orig_policy"` + CurrentPolicy kibana.Policy `json:"current_policy"` + Agent kibana.Agent `json:"agent"` + ConfigFilePath string `json:"config_file_path"` + VariantName string `json:"variant_name"` + EnrollingAgentTime time.Time `json:"enrolling_agent_time"` + ServiceInfoTags []string `json:"service_info_tags,omitempty"` + AgentInfoTags []string `json:"agent_info_tags,omitempty"` + ServiceRunID string `json:"service_info_run_id"` + AgentRunID string `json:"agent_info_run_id"` + AgentHostname string `json:"agent_hostname"` + ServiceAgentHostname string `json:"service_agent_hostnme"` } type scenarioStateOpts struct { @@ -1167,16 +1180,22 @@ type scenarioStateOpts struct { config *testConfig agent kibana.Agent enrollingTime time.Time + agentInfo agentdeployer.AgentInfo + svcInfo servicedeployer.ServiceInfo } func (r *runner) writeScenarioState(opts scenarioStateOpts) error { data := ServiceState{ - OrigPolicy: *opts.origPolicy, - CurrentPolicy: *opts.currentPolicy, - Agent: opts.agent, - ConfigFilePath: opts.config.Path, - VariantName: opts.config.ServiceVariantName, - EnrollingAgentTime: opts.enrollingTime, + OrigPolicy: *opts.origPolicy, + CurrentPolicy: *opts.currentPolicy, + Agent: opts.agent, + ConfigFilePath: opts.config.Path, + VariantName: opts.config.ServiceVariantName, + EnrollingAgentTime: opts.enrollingTime, + ServiceRunID: opts.svcInfo.Test.RunID, + AgentRunID: opts.agentInfo.Test.RunID, + AgentHostname: opts.agentInfo.Hostname, + ServiceAgentHostname: opts.svcInfo.AgentHostname, } dataBytes, err := json.Marshal(data) if err != nil { @@ -1749,10 +1768,10 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t logger.Debugf("filter agents using service criteria: NamePrefix=%s", svcInfo.Agent.Host.NamePrefix) } - expectedTags := agentInfo.Tags + expectedHostname := agentInfo.Hostname if svcInfo.Agent.Host.NamePrefix == "docker-custom-agent" { // custom agents are still started from servicedeployer, they are defined in the same docker compose scenario - expectedTags = svcInfo.Tags + expectedHostname = svcInfo.AgentHostname } // filtered list of agents must contain all agents started by the stack @@ -1780,21 +1799,28 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t continue } - // Tags are available starting in 8.3 - if runIndependentElasticAgent && len(expectedTags) > 0 && len(agent.Tags) > 0 { - logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) - foundAllTags := true - for _, tag := range expectedTags { - if !slices.Contains(agent.Tags, tag) { - logger.Debugf("filtered agent (invalid tag found) %s - %q vs %q", tag, strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) // TODO: remove - foundAllTags = false - break - } - } - if !foundAllTags { + if runIndependentElasticAgent && agent.LocalMetadata.Host.Name != "kind-control-plane" { + logger.Debugf("Checking hostname %s in agent hostname %s", expectedHostname, agent.LocalMetadata.Host.Name) + if expectedHostname != agent.LocalMetadata.Host.Name { + logger.Debugf("filtered agent (invalid hostname suffix) %s - %q: %s", expectedHostname, agent.LocalMetadata.Host.Name, agent.ID) // TODO: remove continue } } + // // Tags are available starting in 8.3 + // if runIndependentElasticAgent && len(expectedTags) > 0 && len(agent.Tags) > 0 { + // logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) + // foundAllTags := true + // for _, tag := range expectedTags { + // if !slices.Contains(agent.Tags, tag) { + // logger.Debugf("filtered agent (invalid tag found) %s - %q vs %q", tag, strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) // TODO: remove + // foundAllTags = false + // break + // } + // } + // if !foundAllTags { + // continue + // } + // } if runIndependentElasticAgent { // FIXME: check for package and data stream name too ? From 125554e29f2c390de53b1ec9ccf1ff1be7941fe3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 16:47:58 +0100 Subject: [PATCH 065/114] Recovered tags for kubernetes --- internal/testrunner/runners/system/runner.go | 34 +++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index dbe3761eb1..04436f42ce 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1769,9 +1769,11 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t } expectedHostname := agentInfo.Hostname + expectedTags := agentInfo.Tags if svcInfo.Agent.Host.NamePrefix == "docker-custom-agent" { // custom agents are still started from servicedeployer, they are defined in the same docker compose scenario expectedHostname = svcInfo.AgentHostname + expectedTags = svcInfo.Tags } // filtered list of agents must contain all agents started by the stack @@ -1806,21 +1808,23 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t continue } } - // // Tags are available starting in 8.3 - // if runIndependentElasticAgent && len(expectedTags) > 0 && len(agent.Tags) > 0 { - // logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) - // foundAllTags := true - // for _, tag := range expectedTags { - // if !slices.Contains(agent.Tags, tag) { - // logger.Debugf("filtered agent (invalid tag found) %s - %q vs %q", tag, strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) // TODO: remove - // foundAllTags = false - // break - // } - // } - // if !foundAllTags { - // continue - // } - // } + // Tags are available starting in 8.3 + // Kubernetes Agent cannot set a different hostname + // Using tags allow to detect the right agent from the list + if runIndependentElasticAgent && len(expectedTags) > 0 && len(agent.Tags) > 0 { + logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) + foundAllTags := true + for _, tag := range expectedTags { + if !slices.Contains(agent.Tags, tag) { + logger.Debugf("filtered agent (invalid tag found) %s - %q vs %q", tag, strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) // TODO: remove + foundAllTags = false + break + } + } + if !foundAllTags { + continue + } + } if runIndependentElasticAgent { // FIXME: check for package and data stream name too ? From 81763497b6d6edbff3fd126cf6f3d25fae3c2ea4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 16:53:08 +0100 Subject: [PATCH 066/114] Remove unused agent deployer --- internal/agentdeployer/agent.go | 141 ++++++++++++++- internal/agentdeployer/compose.go | 285 ------------------------------ internal/agentdeployer/factory.go | 2 +- internal/agentdeployer/logs.go | 55 ++++++ 4 files changed, 188 insertions(+), 295 deletions(-) delete mode 100644 internal/agentdeployer/compose.go create mode 100644 internal/agentdeployer/logs.go diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 8f4a96e953..44fe97fb0a 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/configuration/locations" @@ -34,7 +35,7 @@ var dockerAgentDockerComposeContent []byte // CustomAgentDeployer knows how to deploy a custom elastic-agent defined via // a Docker Compose file. -type CustomAgentDeployer struct { +type DockerComposeAgentDeployer struct { profile *profile.Profile dockerComposeFile string stackVersion string @@ -50,7 +51,7 @@ type CustomAgentDeployer struct { runTestsOnly bool } -type CustomAgentDeployerOptions struct { +type DockerComposeAgentDeployerOptions struct { Profile *profile.Profile DockerComposeFile string StackVersion string @@ -63,11 +64,22 @@ type CustomAgentDeployerOptions struct { RunTestsOnly bool } -var _ AgentDeployer = new(CustomAgentDeployer) +var _ AgentDeployer = new(DockerComposeAgentDeployer) + +type dockerComposeDeployedAgent struct { + agentInfo AgentInfo + + ymlPaths []string + project string + variant AgentVariant + env []string +} + +var _ DeployedAgent = new(dockerComposeDeployedAgent) // NewCustomAgentDeployer returns a new instance of a deployedCustomAgent. -func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDeployer, error) { - return &CustomAgentDeployer{ +func NewCustomAgentDeployer(options DockerComposeAgentDeployerOptions) (*DockerComposeAgentDeployer, error) { + return &DockerComposeAgentDeployer{ profile: options.Profile, dockerComposeFile: options.DockerComposeFile, stackVersion: options.StackVersion, @@ -80,7 +92,7 @@ func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDep } // SetUp sets up the service and returns any relevant information. -func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (DeployedAgent, error) { +func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (DeployedAgent, error) { logger.Debug("setting up service using Docker Compose agent deployer") d.agentRunID = agentInfo.Test.RunID @@ -224,11 +236,11 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (D return &agent, nil } -func (d *CustomAgentDeployer) agentHostname() string { +func (d *DockerComposeAgentDeployer) agentHostname() string { return fmt.Sprintf("%s-%s-%s", dockerTestAgentNamePrefix, d.agentName(), d.agentRunID) } -func (d *CustomAgentDeployer) agentName() string { +func (d *DockerComposeAgentDeployer) agentName() string { name := d.packageName if d.variant.Name != "" { name = fmt.Sprintf("%s-%s", name, d.variant.Name) @@ -241,7 +253,7 @@ func (d *CustomAgentDeployer) agentName() string { // installDockerfile creates the files needed to run the custom elastic agent and returns // the directory with these files. -func (d *CustomAgentDeployer) installDockerfile() (string, error) { +func (d *DockerComposeAgentDeployer) installDockerfile() (string, error) { customAgentDir := filepath.Join(d.profile.ProfilePath, fmt.Sprintf("agent-%s", d.agentName())) err := os.MkdirAll(customAgentDir, 0755) if err != nil { @@ -265,3 +277,114 @@ func CreateServiceLogsDir(elasticPackagePath *locations.LocationManager, name st } return dirPath, nil } + +// Signal sends a signal to the agent. +func (s *dockerComposeDeployedAgent) Signal(ctx context.Context, signal string) error { + p, err := compose.NewProject(s.project, s.ymlPaths...) + if err != nil { + return fmt.Errorf("could not create Docker Compose project for service: %w", err) + } + + opts := compose.CommandOptions{ + Env: append( + s.env, + s.variant.Env..., + ), + ExtraArgs: []string{"-s", signal}, + } + if s.agentInfo.Name != "" { + opts.Services = append(opts.Services, s.agentInfo.Name) + } + + err = p.Kill(ctx, opts) + if err != nil { + return fmt.Errorf("could not send %q signal: %w", signal, err) + } + return nil +} + +// ExitCode returns true if the agent is exited and its exit code. +func (s *dockerComposeDeployedAgent) ExitCode(ctx context.Context, agent string) (bool, int, error) { + p, err := compose.NewProject(s.project, s.ymlPaths...) + if err != nil { + return false, -1, fmt.Errorf("could not create Docker Compose project for agent: %w", err) + } + + opts := compose.CommandOptions{ + Env: append( + s.env, + s.variant.Env..., + ), + } + + return p.ServiceExitCode(ctx, agent, opts) +} + +// Logs returns the logs from the agent starting at the given time +func (s *dockerComposeDeployedAgent) Logs(ctx context.Context, t time.Time) ([]byte, error) { + p, err := compose.NewProject(s.project, s.ymlPaths...) + if err != nil { + return nil, fmt.Errorf("could not create Docker Compose project for agent: %w", err) + } + + opts := compose.CommandOptions{ + Env: append( + s.env, + s.variant.Env..., + ), + } + + return p.Logs(ctx, opts) +} + +// TearDown tears down the agent. +func (s *dockerComposeDeployedAgent) TearDown(ctx context.Context) error { + logger.Debugf("tearing down agent using Docker Compose runner") + defer func() { + err := files.RemoveContent(s.agentInfo.Logs.Folder.Local) + if err != nil { + logger.Errorf("could not remove the agent logs (path: %s)", s.agentInfo.Logs.Folder.Local) + } + // Remove the outputs generated by the service container + if err = os.RemoveAll(s.agentInfo.OutputDir); err != nil { + logger.Errorf("could not remove the temporary output files %w", err) + } + + // Remove the configuration dir (e.g. compose scenario files) + if err = os.RemoveAll(s.agentInfo.ConfigDir); err != nil { + logger.Errorf("could not remove the agent configuration directory %w", err) + } + }() + + p, err := compose.NewProject(s.project, s.ymlPaths...) + if err != nil { + return fmt.Errorf("could not create Docker Compose project for service: %w", err) + } + + opts := compose.CommandOptions{ + Env: append( + s.env, + s.variant.Env..., + ), + } + processAgentContainerLogs(ctx, p, opts, s.agentInfo.Name) + + if err := p.Down(ctx, compose.CommandOptions{ + Env: opts.Env, + ExtraArgs: []string{"--volumes"}, // Remove associated volumes. + }); err != nil { + return fmt.Errorf("could not shut down agent using Docker Compose: %w", err) + } + return nil +} + +// Info returns the current context for the agent. +func (s *dockerComposeDeployedAgent) Info() AgentInfo { + return s.agentInfo +} + +// SetInfo sets the current context for the agent. +func (s *dockerComposeDeployedAgent) SetInfo(ctxt AgentInfo) error { + s.agentInfo = ctxt + return nil +} diff --git a/internal/agentdeployer/compose.go b/internal/agentdeployer/compose.go deleted file mode 100644 index 3f8a347d06..0000000000 --- a/internal/agentdeployer/compose.go +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. -package agentdeployer - -import ( - "context" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/elastic/elastic-package/internal/builder" - "github.com/elastic/elastic-package/internal/compose" - "github.com/elastic/elastic-package/internal/docker" - "github.com/elastic/elastic-package/internal/files" - "github.com/elastic/elastic-package/internal/logger" - "github.com/elastic/elastic-package/internal/profile" - "github.com/elastic/elastic-package/internal/stack" -) - -// DockerComposeServiceDeployer knows how to deploy a service defined via -// a Docker Compose file. -type DockerComposeAgentDeployer struct { - profile *profile.Profile - ymlPaths []string - variant AgentVariant -} - -type dockerComposeDeployedAgent struct { - agentInfo AgentInfo - - ymlPaths []string - project string - variant AgentVariant - env []string -} - -// NewDockerComposeAgentDeployer returns a new instance of a DockerComposeAgentDeployer. -func NewDockerComposeAgentDeployer(profile *profile.Profile, ymlPaths []string) (*DockerComposeAgentDeployer, error) { - return &DockerComposeAgentDeployer{ - profile: profile, - ymlPaths: ymlPaths, - }, nil -} - -var _ AgentDeployer = new(DockerComposeAgentDeployer) - -// SetUp sets up the service and returns any relevant information. -func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (DeployedAgent, error) { - logger.Debug("setting up agent using Docker Compose agent deployer") - agent := dockerComposeDeployedAgent{ - ymlPaths: d.ymlPaths, - project: "elastic-package-agent", - variant: d.variant, - env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, agentInfo.Logs.Folder.Local)}, - } - - p, err := compose.NewProject(agent.project, agent.ymlPaths...) - if err != nil { - return nil, fmt.Errorf("could not create Docker Compose project for service: %w", err) - } - - // Verify the Elastic stack network - err = stack.EnsureStackNetworkUp(d.profile) - if err != nil { - return nil, fmt.Errorf("stack network is not ready: %w", err) - } - - // Clean service logs - err = files.RemoveContent(agentInfo.Logs.Folder.Local) - if err != nil { - return nil, fmt.Errorf("removing service logs failed: %w", err) - } - - // Boot up service - if d.variant.active() { - logger.Infof("Using service variant: %s", d.variant.String()) - } - - agentName := agentInfo.Name - opts := compose.CommandOptions{ - Env: append( - agent.env, - d.variant.Env..., - ), - ExtraArgs: []string{"--build", "-d"}, - } - err = p.Up(ctx, opts) - if err != nil { - return nil, fmt.Errorf("could not boot up service using Docker Compose: %w", err) - } - - err = p.WaitForHealthy(ctx, opts) - if err != nil { - processAgentContainerLogs(ctx, p, compose.CommandOptions{ - Env: opts.Env, - }, agentName) - return nil, fmt.Errorf("service is unhealthy: %w", err) - } - - // Build agent container name - agentInfo.Hostname = p.ContainerName(agentName) - - // Connect service network with stack network (for the purpose of metrics collection) - err = docker.ConnectToNetwork(p.ContainerName(agentName), stack.Network(d.profile)) - if err != nil { - return nil, fmt.Errorf("can't attach service container to the stack network: %w", err) - } - - logger.Debugf("adding agent container %s internal ports to context", p.ContainerName(agentName)) - agentComposeConfig, err := p.Config(ctx, compose.CommandOptions{ - Env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, agentInfo.Logs.Folder.Local)}, - }) - if err != nil { - return nil, fmt.Errorf("could not get Docker Compose configuration for agent: %w", err) - } - - s := agentComposeConfig.Services[agentName] - agentInfo.Ports = make([]int, len(s.Ports)) - for idx, port := range s.Ports { - agentInfo.Ports[idx] = port.InternalPort - } - - // Shortcut to first port for convenience - if len(agentInfo.Ports) > 0 { - agentInfo.Port = agentInfo.Ports[0] - } - - agentInfo.Agent.Host.NamePrefix = dockerTestAgentNamePrefix - agent.agentInfo = agentInfo - return &agent, nil -} - -// Signal sends a signal to the agent. -func (s *dockerComposeDeployedAgent) Signal(ctx context.Context, signal string) error { - p, err := compose.NewProject(s.project, s.ymlPaths...) - if err != nil { - return fmt.Errorf("could not create Docker Compose project for service: %w", err) - } - - opts := compose.CommandOptions{ - Env: append( - s.env, - s.variant.Env..., - ), - ExtraArgs: []string{"-s", signal}, - } - if s.agentInfo.Name != "" { - opts.Services = append(opts.Services, s.agentInfo.Name) - } - - err = p.Kill(ctx, opts) - if err != nil { - return fmt.Errorf("could not send %q signal: %w", signal, err) - } - return nil -} - -// ExitCode returns true if the agent is exited and its exit code. -func (s *dockerComposeDeployedAgent) ExitCode(ctx context.Context, agent string) (bool, int, error) { - p, err := compose.NewProject(s.project, s.ymlPaths...) - if err != nil { - return false, -1, fmt.Errorf("could not create Docker Compose project for agent: %w", err) - } - - opts := compose.CommandOptions{ - Env: append( - s.env, - s.variant.Env..., - ), - } - - return p.ServiceExitCode(ctx, agent, opts) -} - -// Logs returns the logs from the agent starting at the given time -func (s *dockerComposeDeployedAgent) Logs(ctx context.Context, t time.Time) ([]byte, error) { - p, err := compose.NewProject(s.project, s.ymlPaths...) - if err != nil { - return nil, fmt.Errorf("could not create Docker Compose project for agent: %w", err) - } - - opts := compose.CommandOptions{ - Env: append( - s.env, - s.variant.Env..., - ), - } - - return p.Logs(ctx, opts) -} - -// TearDown tears down the agent. -func (s *dockerComposeDeployedAgent) TearDown(ctx context.Context) error { - logger.Debugf("tearing down agent using Docker Compose runner") - defer func() { - err := files.RemoveContent(s.agentInfo.Logs.Folder.Local) - if err != nil { - logger.Errorf("could not remove the agent logs (path: %s)", s.agentInfo.Logs.Folder.Local) - } - // Remove the outputs generated by the service container - if err = os.RemoveAll(s.agentInfo.OutputDir); err != nil { - logger.Errorf("could not remove the temporary output files %w", err) - } - - // Remove the configuration dir (e.g. compose scenario files) - if err = os.RemoveAll(s.agentInfo.ConfigDir); err != nil { - logger.Errorf("could not remove the agent configuration directory %w", err) - } - }() - - p, err := compose.NewProject(s.project, s.ymlPaths...) - if err != nil { - return fmt.Errorf("could not create Docker Compose project for service: %w", err) - } - - opts := compose.CommandOptions{ - Env: append( - s.env, - s.variant.Env..., - ), - } - processAgentContainerLogs(ctx, p, opts, s.agentInfo.Name) - - if err := p.Down(ctx, compose.CommandOptions{ - Env: opts.Env, - ExtraArgs: []string{"--volumes"}, // Remove associated volumes. - }); err != nil { - return fmt.Errorf("could not shut down agent using Docker Compose: %w", err) - } - return nil -} - -// Info returns the current context for the agent. -func (s *dockerComposeDeployedAgent) Info() AgentInfo { - return s.agentInfo -} - -// SetInfo sets the current context for the agent. -func (s *dockerComposeDeployedAgent) SetInfo(ctxt AgentInfo) error { - s.agentInfo = ctxt - return nil -} - -var _ DeployedAgent = new(dockerComposeDeployedAgent) - -func processAgentContainerLogs(ctx context.Context, p *compose.Project, opts compose.CommandOptions, agentName string) { - content, err := p.Logs(ctx, opts) - if err != nil { - logger.Errorf("can't export service logs: %v", err) - return - } - - if len(content) == 0 { - logger.Info("service container hasn't written anything logs.") - return - } - - err = writeAgentContainerLogs(agentName, content) - if err != nil { - logger.Errorf("can't write service container logs: %v", err) - } -} - -func writeAgentContainerLogs(agentName string, content []byte) error { - buildDir, err := builder.BuildDirectory() - if err != nil { - return fmt.Errorf("locating build directory failed: %w", err) - } - - containerLogsDir := filepath.Join(buildDir, "container-logs") - err = os.MkdirAll(containerLogsDir, 0o755) - if err != nil { - return fmt.Errorf("can't create directory for agent container logs (path: %s): %w", containerLogsDir, err) - } - - containerLogsFilepath := filepath.Join(containerLogsDir, fmt.Sprintf("%s-%d.log", agentName, time.Now().UnixNano())) - logger.Infof("Write container logs to file: %s", containerLogsFilepath) - err = os.WriteFile(containerLogsFilepath, content, 0o644) - if err != nil { - return fmt.Errorf("can't write container logs to file (path: %s): %w", containerLogsFilepath, err) - } - return nil -} diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go index f52dcb662f..e607c75eb7 100644 --- a/internal/agentdeployer/factory.go +++ b/internal/agentdeployer/factory.go @@ -69,7 +69,7 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { if err != nil { return nil, fmt.Errorf("can't use service variant: %w", err) } - opts := CustomAgentDeployerOptions{ + opts := DockerComposeAgentDeployerOptions{ Profile: options.Profile, DockerComposeFile: "", Variant: sv, diff --git a/internal/agentdeployer/logs.go b/internal/agentdeployer/logs.go new file mode 100644 index 0000000000..3d23f6219f --- /dev/null +++ b/internal/agentdeployer/logs.go @@ -0,0 +1,55 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. +package agentdeployer + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/elastic/elastic-package/internal/builder" + "github.com/elastic/elastic-package/internal/compose" + "github.com/elastic/elastic-package/internal/logger" +) + +func processAgentContainerLogs(ctx context.Context, p *compose.Project, opts compose.CommandOptions, agentName string) { + content, err := p.Logs(ctx, opts) + if err != nil { + logger.Errorf("can't export service logs: %v", err) + return + } + + if len(content) == 0 { + logger.Info("service container hasn't written anything logs.") + return + } + + err = writeAgentContainerLogs(agentName, content) + if err != nil { + logger.Errorf("can't write service container logs: %v", err) + } +} + +func writeAgentContainerLogs(agentName string, content []byte) error { + buildDir, err := builder.BuildDirectory() + if err != nil { + return fmt.Errorf("locating build directory failed: %w", err) + } + + containerLogsDir := filepath.Join(buildDir, "container-logs") + err = os.MkdirAll(containerLogsDir, 0o755) + if err != nil { + return fmt.Errorf("can't create directory for agent container logs (path: %s): %w", containerLogsDir, err) + } + + containerLogsFilepath := filepath.Join(containerLogsDir, fmt.Sprintf("%s-%d.log", agentName, time.Now().UnixNano())) + logger.Infof("Write container logs to file: %s", containerLogsFilepath) + err = os.WriteFile(containerLogsFilepath, content, 0o644) + if err != nil { + return fmt.Errorf("can't write container logs to file (path: %s): %w", containerLogsFilepath, err) + } + return nil +} From 995cfc75fe54a41459b6ccd7c54ce802c9f59a07 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 17:26:50 +0100 Subject: [PATCH 067/114] Refactor setup agent and service into their own functions --- internal/testrunner/runners/system/runner.go | 136 +++++++++++-------- 1 file changed, 78 insertions(+), 58 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 04436f42ce..e534017102 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -752,7 +752,6 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf } // Setup agent logger.Debug("setting up agent...") - agentOptions := r.createAgentOptions(serviceOptions.Variant) agentInfo, err := r.createAgentInfo() if err != nil { return nil, err @@ -771,69 +770,22 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf svcInfo.AgentHostname = serviceStateData.ServiceAgentHostname svcInfo.Hostname = serviceStateData.ServiceAgentHostname } - agentDeployer, err := agentdeployer.Factory(agentOptions) + + agentDeployed, agentInfo, err := r.setupAgent(ctx, serviceOptions.Variant, agentInfo) if err != nil { - return nil, fmt.Errorf("could not create agent runner: %w", err) + return nil, err } - var agentDeployed agentdeployer.DeployedAgent - if r.options.RunIndependentElasticAgent { - if agentDeployer != nil { - agentDeployed, err = agentDeployer.SetUp(ctx, agentInfo) - if err != nil { - return nil, fmt.Errorf("could not setup agent: %w", err) - } - agentInfo = agentDeployed.Info() - } - r.shutdownAgentHandler = func(ctx context.Context) error { - if agentDeployer == nil { - return nil - } - logger.Debug("tearing down agent...") - if err := agentDeployed.TearDown(ctx); err != nil { - return fmt.Errorf("error tearing down agent: %w", err) - } - - return nil - } - scenario.agent = agentDeployed - scenario.enrollingTime = enrollingTime + if agentDeployed != nil { + agentInfo = agentDeployed.Info() } - // Setup service. - logger.Debug("setting up service...") - serviceOptions.PackageName = scenario.pkgManifest.Name - serviceOptions.DataStream = scenario.dataStreamManifest.Name - - serviceDeployer, err := servicedeployer.Factory(serviceOptions) - if err != nil { - return nil, fmt.Errorf("could not create service runner: %w", err) - } + scenario.enrollingTime = enrollingTime + scenario.agent = agentDeployed - svcInfo.AgentHostname = "elastic-agent" - if r.options.RunIndependentElasticAgent { - svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local - if agentDeployed != nil { - // Not all agents are created from "agentdeployer", currently agent and - // service containers are created from "servicedeployer" package for - // CustomAgent scenario - svcInfo.AgentHostname = agentDeployed.Info().Hostname - } - } - if config.Service != "" { - svcInfo.Name = config.Service - } - service, err := serviceDeployer.SetUp(ctx, svcInfo) + // Setup service. + service, svcInfo, err := r.setupService(ctx, config, serviceOptions, svcInfo, agentInfo, agentDeployed) if err != nil { - return nil, fmt.Errorf("could not setup service: %w", err) - } - svcInfo = service.Info() - r.shutdownServiceHandler = func(ctx context.Context) error { - logger.Debug("tearing down service...") - if err := service.TearDown(ctx); err != nil { - return fmt.Errorf("error tearing down service: %w", err) - } - - return nil + return nil, err } // Reload test config with ctx variable substitution. @@ -1128,6 +1080,74 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf return &scenario, nil } +func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOptions servicedeployer.FactoryOptions, svcInfo servicedeployer.ServiceInfo, agentInfo agentdeployer.AgentInfo, agentDeployed agentdeployer.DeployedAgent) (servicedeployer.DeployedService, servicedeployer.ServiceInfo, error) { + logger.Debug("setting up service...") + serviceDeployer, err := servicedeployer.Factory(serviceOptions) + if err != nil { + return nil, svcInfo, fmt.Errorf("could not create service runner: %w", err) + } + + svcInfo.AgentHostname = "elastic-agent" + if r.options.RunIndependentElasticAgent { + svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local + if agentDeployed != nil { + // Not all agents are created from "agentdeployer", currently agent and + // service containers are created from "servicedeployer" package for + // CustomAgent scenario + // And there are packages that require to run queries (requests) to the + // elastic agent container to inject logs (e.g. ti_anomali) + svcInfo.AgentHostname = agentDeployed.Info().Hostname + } + } + if config.Service != "" { + svcInfo.Name = config.Service + } + service, err := serviceDeployer.SetUp(ctx, svcInfo) + if err != nil { + return nil, svcInfo, fmt.Errorf("could not setup service: %w", err) + } + + r.shutdownServiceHandler = func(ctx context.Context) error { + logger.Debug("tearing down service...") + if err := service.TearDown(ctx); err != nil { + return fmt.Errorf("error tearing down service: %w", err) + } + + return nil + } + return service, service.Info(), nil +} + +func (r *runner) setupAgent(ctx context.Context, variant string, agentInfo agentdeployer.AgentInfo) (agentdeployer.DeployedAgent, agentdeployer.AgentInfo, error) { + if !r.options.RunIndependentElasticAgent { + return nil, agentInfo, nil + } + agentOptions := r.createAgentOptions(variant) + agentDeployer, err := agentdeployer.Factory(agentOptions) + if err != nil { + return nil, agentInfo, fmt.Errorf("could not create agent runner: %w", err) + } + if agentDeployer == nil { + return nil, agentInfo, nil + } + agentDeployed, err := agentDeployer.SetUp(ctx, agentInfo) + if err != nil { + return nil, agentInfo, fmt.Errorf("could not setup agent: %w", err) + } + r.shutdownAgentHandler = func(ctx context.Context) error { + if agentDeployer == nil { + return nil + } + logger.Debug("tearing down agent...") + if err := agentDeployed.TearDown(ctx); err != nil { + return fmt.Errorf("error tearing down agent: %w", err) + } + + return nil + } + return agentDeployed, agentDeployed.Info(), nil +} + func (r *runner) removeServiceStateFile() error { err := os.Remove(r.serviceStateFilePath) if err != nil { From f1989bf199ebd800b07206ccd73a79d0acd2fce4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 22 Mar 2024 17:38:19 +0100 Subject: [PATCH 068/114] Restored usage of variant - to be reviewed --- internal/agentdeployer/agent.go | 12 +++++++----- internal/agentdeployer/factory.go | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 44fe97fb0a..c1cc487284 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -93,7 +93,7 @@ func NewCustomAgentDeployer(options DockerComposeAgentDeployerOptions) (*DockerC // SetUp sets up the service and returns any relevant information. func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInfo) (DeployedAgent, error) { - logger.Debug("setting up service using Docker Compose agent deployer") + logger.Debug("setting up agent using Docker Compose agent deployer") d.agentRunID = agentInfo.Test.RunID appConfig, err := install.Configuration() @@ -144,10 +144,8 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI agent := dockerComposeDeployedAgent{ ymlPaths: ymlPaths, project: composeProjectName, - variant: AgentVariant{ - Name: dockerTestAgentNamePrefix, - Env: env, - }, + variant: d.variant, + env: env, } agentInfo.ConfigDir = configDir @@ -180,6 +178,10 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI agentInfo.Name = dockerTestAgentNamePrefix agentName := agentInfo.Name + if d.variant.active() { + logger.Infof("Using variant: %s", d.variant.String()) + } + opts := compose.CommandOptions{ Env: env, ExtraArgs: []string{"--build", "-d"}, diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go index e607c75eb7..570d31cfaa 100644 --- a/internal/agentdeployer/factory.go +++ b/internal/agentdeployer/factory.go @@ -65,14 +65,14 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { if options.Type != TypeTest { return nil, fmt.Errorf("agent deployer is not supported for type %s", options.Type) } - sv, err := useAgentVariant(devDeployPath, options.Variant) + variant, err := useAgentVariant(devDeployPath, options.Variant) if err != nil { return nil, fmt.Errorf("can't use service variant: %w", err) } opts := DockerComposeAgentDeployerOptions{ Profile: options.Profile, DockerComposeFile: "", - Variant: sv, + Variant: variant, StackVersion: options.StackVersion, PackageName: options.PackageName, DataStream: options.DataStream, From b4df467355fdcda2210606a3cda630c23306504f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 25 Mar 2024 11:30:28 +0100 Subject: [PATCH 069/114] Use environment vars instead of flag --- .buildkite/pipeline.trigger.integration.tests.sh | 4 ++-- cmd/testrunner.go | 12 +++++++----- internal/cobraext/flags.go | 3 --- scripts/test-check-packages.sh | 6 +----- scripts/test-system-test-flags.sh | 10 ++-------- 5 files changed, 12 insertions(+), 23 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 5ba9dd5146..6bb7546473 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -90,7 +90,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" echo " env:" echo " UPLOAD_SAFE_LOGS: 1" - echo " ELASTIC_PACKAGE_INDEPENDENT_AGENT: ${independent_agent}" + echo " ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT: ${independent_agent}" echo " agents:" echo " provider: \"gcp\"" echo " artifact_paths:" @@ -129,7 +129,7 @@ echo " command: ./.buildkite/scripts/integration_tests.sh -t test-system- echo " agents:" echo " provider: \"gcp\"" echo " env:" -echo " ELASTIC_PACKAGE_INDEPENDENT_AGENT: ${independent_agent}" +echo " ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT: ${independent_agent}" done echo " - label: \":go: Integration test: test-profiles-command\"" diff --git a/cmd/testrunner.go b/cmd/testrunner.go index ef82c39c7a..77225a5906 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -18,6 +18,7 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/environment" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" @@ -52,6 +53,8 @@ These tests allow you to test a package's ability to ingest data end-to-end. For details on how to configure amd run system tests, review the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/system_testing.md).` +var enableIndependentAgents = environment.WithElasticPackagePrefix("TEST_ENABLE_INDEPENDENT_AGENT") + func setupTestCommand() *cobraext.Command { var testTypeCmdActions []cobraext.CommandAction @@ -79,8 +82,6 @@ func setupTestCommand() *cobraext.Command { cmd.PersistentFlags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription) cmd.PersistentFlags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription) cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) - // By default, it keeps the same behaviour as previously - cmd.PersistentFlags().BoolP(cobraext.TestIndependentElasticAgentFlagName, "", false, cobraext.TestIndependentElasticAgentFlagDescription) for testType, runner := range testrunner.TestRunners() { action := testTypeCommandActionFactory(runner) @@ -188,9 +189,10 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command return fmt.Errorf("cannot determine if package has data streams: %w", err) } - runIndependentElasticAgent, err := cmd.Flags().GetBool(cobraext.TestIndependentElasticAgentFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.TestIndependentElasticAgentFlagName) + runIndependentElasticAgent := false + v, ok := os.LookupEnv(enableIndependentAgents) + if ok && strings.ToLower(v) != "false" { + runIndependentElasticAgent = true } configFileFlag := "" diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 8895c586cd..75df10fccd 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -233,7 +233,4 @@ const ( SkipPullRequestFlagName = "skip-pull-request" SkipPullRequestFlagDescription = "skip opening a new pull request" - - TestIndependentElasticAgentFlagName = "test-independent-agent" - TestIndependentElasticAgentFlagDescription = "enable testing each test with its own Elastic Agent (technical preview)" ) diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 713432633f..e17a0362e3 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -41,7 +41,7 @@ cleanup() { trap cleanup EXIT -ELASTIC_PACKAGE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_INDEPENDENT_AGENT:-"false"} +ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT:-"false"} export ELASTIC_PACKAGE_INDEPENDENT_AGENT ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml" export ELASTIC_PACKAGE_LINKS_FILE_PATH @@ -109,10 +109,6 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do elif [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ] && [ "${PACKAGE_UNDER_TEST:-*}" == "system_benchmark" ]; then elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s else - ELASTIC_PACKAGE_TEST_OPTS="" - if [[ "${ELASTIC_PACKAGE_INDEPENDENT_AGENT}" == "true" ]]; then - ELASTIC_PACKAGE_TEST_OPTS="--test-independent-agent" - fi # defer-cleanup is set to a short period to verify that the option is available elastic-package test -v \ --report-format xUnit \ diff --git a/scripts/test-system-test-flags.sh b/scripts/test-system-test-flags.sh index 9a579250b3..6b2bde2500 100755 --- a/scripts/test-system-test-flags.sh +++ b/scripts/test-system-test-flags.sh @@ -201,11 +201,8 @@ run_tests_for_package() { AGENT_CONTAINER_NAME="${DEFAULT_AGENT_CONTAINER_NAME}" fi - ELASTIC_PACKAGE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_INDEPENDENT_AGENT:-"false"} - ELASTIC_PACKAGE_TEST_OPTS="" - if [[ "${ELASTIC_PACKAGE_INDEPENDENT_AGENT}" == "true" ]] ; then - ELASTIC_PACKAGE_TEST_OPTS="--test-independent-agent" - fi + ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT:-"false"} + export ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT pushd "${package_folder}" > /dev/null echo "--- [${package_name} - ${variant}] Setup service without tear-down" @@ -213,7 +210,6 @@ run_tests_for_package() { --report-format xUnit --report-output file \ --config-file "${config_file}" \ ${variant_flag} \ - ${ELASTIC_PACKAGE_TEST_OPTS} \ --setup ; then return 1 fi @@ -231,7 +227,6 @@ run_tests_for_package() { echo "--- Iteration #${i} --no-provision" if ! elastic-package test system -v \ --report-format xUnit --report-output file \ - ${ELASTIC_PACKAGE_TEST_OPTS} \ --no-provision ; then return 1 fi @@ -248,7 +243,6 @@ run_tests_for_package() { echo "--- [${package_name} - ${variant}] Run tear-down process" if ! elastic-package test system -v \ --report-format xUnit --report-output file \ - ${ELASTIC_PACKAGE_TEST_OPTS} \ --tear-down ; then return 1 fi From d3a317d88a9359c0c1dcc59dafd58a0792578b3f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 25 Mar 2024 11:58:20 +0100 Subject: [PATCH 070/114] Remove leftover --- scripts/test-check-packages.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index e17a0362e3..cc4f2a54d3 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -115,8 +115,7 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do --report-output file \ --defer-cleanup 1s \ --test-coverage \ - --coverage-format=generic \ - ${ELASTIC_PACKAGE_TEST_OPTS} + --coverage-format=generic fi ) cd - From 6c21c1fdda046ca4ed7a575db9c0c0e39ab634ab Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 25 Mar 2024 13:33:45 +0100 Subject: [PATCH 071/114] Re-order agent hostname usages --- .../servicedeployer/_static/terraform_deployer.yml | 1 + internal/servicedeployer/custom_agent.go | 14 +++++++------- internal/servicedeployer/terraform_env.go | 1 + 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/internal/servicedeployer/_static/terraform_deployer.yml b/internal/servicedeployer/_static/terraform_deployer.yml index 794446d1ba..6c22f7644c 100644 --- a/internal/servicedeployer/_static/terraform_deployer.yml +++ b/internal/servicedeployer/_static/terraform_deployer.yml @@ -11,6 +11,7 @@ services: - TF_VAR_BUILD_ID=${BUILD_ID:-unknown} - TF_VAR_ENVIRONMENT=${ENVIRONMENT:-unknown} - TF_VAR_REPO=${REPO:-unknown} + - AGENT_HOSTNAME=${AGENT_HOSTNAME:-elastic-agent} volumes: - ${TF_DIR}:/stage - ${TF_OUTPUT_DIR}:/output diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index 0b7867de1f..23f730b850 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -91,11 +91,17 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D return nil, fmt.Errorf("can't locate CA certificate: %w", err) } + // Build service container name + // FIXME: Currently, this service deployer starts a new agent on its own and + // it cannot use directly the `svcInfo.AgentHostname` value + svcInfo.Hostname = d.agentHostname() + svcInfo.AgentHostname = d.agentHostname() + env := append( appConfig.StackImageRefs(d.stackVersion).AsEnv(), fmt.Sprintf("%s=%s", serviceLogsDirEnv, svcInfo.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), - fmt.Sprintf("%s=%s", agentHostnameEnv, d.agentHostname()), + fmt.Sprintf("%s=%s", agentHostnameEnv, svcInfo.AgentHostname), fmt.Sprintf("%s=%s", elasticAgentTagsEnv, strings.Join(svcInfo.Tags, ",")), ) @@ -173,12 +179,6 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D return nil, fmt.Errorf("service is unhealthy: %w", err) } - // Build service container name - // Set the same hostname as in the docker-compose (environment variable) - //svcInfo.Hostname = p.ContainerName(serviceName) - svcInfo.Hostname = d.agentHostname() - svcInfo.AgentHostname = d.agentHostname() - logger.Debugf("adding service container %s internal ports to context", p.ContainerName(serviceName)) serviceComposeConfig, err := p.Config(ctx, compose.CommandOptions{Env: env}) if err != nil { diff --git a/internal/servicedeployer/terraform_env.go b/internal/servicedeployer/terraform_env.go index 428aefe352..b99c8705a4 100644 --- a/internal/servicedeployer/terraform_env.go +++ b/internal/servicedeployer/terraform_env.go @@ -26,6 +26,7 @@ func (tsd TerraformServiceDeployer) buildTerraformExecutorEnvironment(info Servi vars[tfTestRunID] = info.Test.RunID vars[tfDir] = tsd.definitionsDir vars[tfOutputDir] = info.OutputDir + vars[agentHostnameEnv] = info.AgentHostname var pairs []string for k, v := range vars { From 90cd7909965dd51c2042920513d8070f7d80e26c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 25 Mar 2024 13:34:11 +0100 Subject: [PATCH 072/114] Test with gcp stack agent --- .buildkite/pipeline.trigger.integration.tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 6bb7546473..50757842cd 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -81,7 +81,7 @@ pushd test/packages/parallel > /dev/null for independent_agent in false; do for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do package_name=$(basename "${package}") - if [[ "$package_name" == "aws" || "$package_name" == "aws_logs" || "$package_name" == "gcp" ]] ; then + if [[ "$package_name" == "aws" || "$package_name" == "aws_logs" ]] ; then echoerr "Skip temporarily ${package_name}" continue fi From 9e08d9a8afeaf5d69100677577ae9ee6626477a6 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 25 Mar 2024 15:01:26 +0100 Subject: [PATCH 073/114] Fix conditions for terraform packages in hook --- .buildkite/hooks/pre-command | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 358bad3c5e..b051542557 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -72,7 +72,7 @@ is_step_testing_gcp () { return 1 } -if use_step_testing_gcp; then +if is_step_testing_gcp; then ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH}) export ELASTIC_PACKAGE_GCP_PROJECT_SECRET ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -c) @@ -96,7 +96,7 @@ is_step_testing_aws () { return 1 } -if use_step_testing_aws; then +if is_step_testing_aws; then ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) export ELASTIC_PACKAGE_AWS_SECRET_KEY ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) From 785da957db172ba33996b76d2d18d3831f7bef1a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 25 Mar 2024 15:27:53 +0100 Subject: [PATCH 074/114] Fix env. name to enable independent agent --- .buildkite/pipeline.trigger.integration.tests.sh | 4 ++-- scripts/test-check-packages.sh | 4 ++-- scripts/test-system-test-flags.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 50757842cd..4e42be9446 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -90,7 +90,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" echo " env:" echo " UPLOAD_SAFE_LOGS: 1" - echo " ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT: ${independent_agent}" + echo " ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}" echo " agents:" echo " provider: \"gcp\"" echo " artifact_paths:" @@ -129,7 +129,7 @@ echo " command: ./.buildkite/scripts/integration_tests.sh -t test-system- echo " agents:" echo " provider: \"gcp\"" echo " env:" -echo " ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT: ${independent_agent}" +echo " ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}" done echo " - label: \":go: Integration test: test-profiles-command\"" diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index cc4f2a54d3..f5aaa84f75 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -41,8 +41,8 @@ cleanup() { trap cleanup EXIT -ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT:-"false"} -export ELASTIC_PACKAGE_INDEPENDENT_AGENT +ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-"false"} +export ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml" export ELASTIC_PACKAGE_LINKS_FILE_PATH diff --git a/scripts/test-system-test-flags.sh b/scripts/test-system-test-flags.sh index 6b2bde2500..b4679de3e6 100755 --- a/scripts/test-system-test-flags.sh +++ b/scripts/test-system-test-flags.sh @@ -201,8 +201,8 @@ run_tests_for_package() { AGENT_CONTAINER_NAME="${DEFAULT_AGENT_CONTAINER_NAME}" fi - ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT:-"false"} - export ELASTIC_PACKAGE_ENABLE_INDEPENDENT_AGENT + ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-"false"} + export ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT pushd "${package_folder}" > /dev/null echo "--- [${package_name} - ${variant}] Setup service without tear-down" From f782e584210a03267e43b41664a94559a4ab88c3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 25 Mar 2024 15:52:18 +0100 Subject: [PATCH 075/114] Test with independent agent test packages parallel --- .buildkite/pipeline.trigger.integration.tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 4e42be9446..e7a48e49c4 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -78,7 +78,7 @@ done popd > /dev/null pushd test/packages/parallel > /dev/null -for independent_agent in false; do +for independent_agent in false true; do for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do package_name=$(basename "${package}") if [[ "$package_name" == "aws" || "$package_name" == "aws_logs" ]] ; then From 26e3b6589a603b476859cc414745199dd6dad183 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 25 Mar 2024 16:45:21 +0100 Subject: [PATCH 076/114] Skip ti_anomali package with independent agent --- .buildkite/pipeline.trigger.integration.tests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index e7a48e49c4..a637674078 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -85,6 +85,11 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echoerr "Skip temporarily ${package_name}" continue fi + # Currently, ti_anomali package has the container name hardcoded in one of the configuration files + if [[ "$package_name" == "ti_anomali" && "$independent_agent" == "true" ]] ; then + echoerr "Skip temporarily ${package_name}" + continue + fi echo " - label: \":go: Integration test: ${package_name} - independent_agent ${independent_agent}\"" echo " key: \"integration-parallel-${package_name}-agent-${independent_agent}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" From ceeb7a794c05ae3c9971159bd55a6c84486de4ec Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 25 Mar 2024 18:21:38 +0100 Subject: [PATCH 077/114] Set different folder for independent agent --- .buildkite/scripts/integration_tests.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index 9973585301..56ec2d1727 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -41,6 +41,7 @@ KIND_TARGET="test-check-packages-with-kind" SYSTEM_TEST_FLAGS_TARGET="test-system-test-flags" TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX" GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json" +ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-"false"} REPO_NAME=$(repo_name "${BUILDKITE_REPO}") REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)" @@ -132,21 +133,25 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT set -e if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then + package_folder="${PACKAGE}" + if [[ "${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT}" != "false" ]]; then + package_folder="${package_folder}-independent_agent" + fi upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*.*" \ - "insecure-logs/${PACKAGE}/elastic-agent-logs/" + "insecure-logs/${package_folder}/elastic-agent-logs/" # required for <8.6.0 upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/default/*" \ - "insecure-logs/${PACKAGE}/elastic-agent-logs/default/" + "insecure-logs/${package_folder}/elastic-agent-logs/default/" upload_safe_logs \ "${JOB_GCS_BUCKET_INTERNAL}" \ "build/container-logs/*.log" \ - "insecure-logs/${PACKAGE}/container-logs/" + "insecure-logs/${package_folder}/container-logs/" fi if [ $testReturnCode != 0 ]; then From c20bd7c3fddadc78f06c1bff47cdd99fafb77331 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 26 Mar 2024 10:57:50 +0100 Subject: [PATCH 078/114] Connect services to the agent network When independent agents are set, service containers will connect to the network of the agent. That allows to set the same "elastic-agent" alias that it was also set in the Elastic Agent running in the stack. As service will not be connected to the Stack network there should not be two containers with the same alias elastic-agent. --- .../_static/docker-agent-base.yml | 2 +- internal/agentdeployer/agent.go | 5 +-- internal/agentdeployer/info.go | 2 ++ internal/servicedeployer/compose.go | 31 +++++++++++++------ internal/servicedeployer/custom_agent.go | 6 +++- internal/servicedeployer/factory.go | 11 ++++--- internal/servicedeployer/info.go | 2 ++ internal/testrunner/runners/system/runner.go | 22 ++++++------- .../_dev/deploy/docker/docker-compose.yml | 2 +- .../_dev/deploy/docker/docker-compose.yml | 4 +-- .../_dev/deploy/docker/docker-compose.yml | 4 +-- 11 files changed, 57 insertions(+), 34 deletions(-) diff --git a/internal/agentdeployer/_static/docker-agent-base.yml b/internal/agentdeployer/_static/docker-agent-base.yml index 212d7e2058..2fa4a5b2c4 100644 --- a/internal/agentdeployer/_static/docker-agent-base.yml +++ b/internal/agentdeployer/_static/docker-agent-base.yml @@ -1,6 +1,6 @@ version: "2.3" services: - docker-test-agent: + elastic-agent: hostname: ${AGENT_HOSTNAME} image: "${ELASTIC_AGENT_IMAGE_REF}" healthcheck: diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index c1cc487284..bc85baa2eb 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -24,7 +24,7 @@ import ( ) const ( - dockerTestAgentNamePrefix = "docker-test-agent" + dockerTestAgentNamePrefix = "elastic-agent" dockerTestgentDir = "docker_test_agent" dockerTestAgentDockerCompose = "docker-agent-base.yml" defaultAgentPolicyName = "Elastic-Agent (elastic-package)" @@ -149,6 +149,7 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI } agentInfo.ConfigDir = configDir + agentInfo.NetworkName = fmt.Sprintf("%s_default", composeProjectName) p, err := compose.NewProject(agent.project, agent.ymlPaths...) if err != nil { @@ -197,7 +198,7 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI // Connect service network with stack network (for the purpose of metrics collection) err = docker.ConnectToNetwork(p.ContainerName(agentName), stack.Network(d.profile)) if err != nil { - return nil, fmt.Errorf("can't attach service container to the stack network: %w", err) + return nil, fmt.Errorf("can't attach agent container to the stack network: %w", err) } } diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go index d6b588ffa6..f40837cdf3 100644 --- a/internal/agentdeployer/info.go +++ b/internal/agentdeployer/info.go @@ -24,6 +24,8 @@ type AgentInfo struct { // the Agent container. Hostname string + NetworkName string + // Ports is a list of ports that the service listens on, as addressable // from the Agent container. Ports []int diff --git a/internal/servicedeployer/compose.go b/internal/servicedeployer/compose.go index 793c9c0011..dd75605407 100644 --- a/internal/servicedeployer/compose.go +++ b/internal/servicedeployer/compose.go @@ -28,6 +28,8 @@ type DockerComposeServiceDeployer struct { ymlPaths []string variant ServiceVariant + deployIndependentAgent bool + runTearDown bool runTestsOnly bool } @@ -37,6 +39,8 @@ type DockerComposeServiceDeployerOptions struct { YmlPaths []string Variant ServiceVariant + DeployIndependentAgent bool + RunTearDown bool RunTestsOnly bool } @@ -57,11 +61,12 @@ var _ ServiceDeployer = new(DockerComposeServiceDeployer) // NewDockerComposeServiceDeployer returns a new instance of a DockerComposeServiceDeployer. func NewDockerComposeServiceDeployer(options DockerComposeServiceDeployerOptions) (*DockerComposeServiceDeployer, error) { return &DockerComposeServiceDeployer{ - profile: options.Profile, - ymlPaths: options.YmlPaths, - variant: options.Variant, - runTearDown: options.RunTearDown, - runTestsOnly: options.RunTestsOnly, + profile: options.Profile, + ymlPaths: options.YmlPaths, + variant: options.Variant, + runTearDown: options.RunTearDown, + runTestsOnly: options.RunTestsOnly, + deployIndependentAgent: options.DeployIndependentAgent, }, nil } @@ -137,10 +142,18 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic if d.runTearDown || d.runTestsOnly { logger.Debug("Skipping connect container to network (non setup steps)") } else { - // Connect service network with stack network (for the purpose of metrics collection) - err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network(d.profile)) - if err != nil { - return nil, fmt.Errorf("can't attach service container to the stack network: %w", err) + if !d.deployIndependentAgent { + // Connect service network with stack network (for the purpose of metrics collection) + err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network(d.profile)) + if err != nil { + return nil, fmt.Errorf("can't attach service container to the stack network: %w", err) + } + } else { + // Connect service network with agent network + err = docker.ConnectToNetwork(p.ContainerName(serviceName), svcInfo.AgentNetworkName) + if err != nil { + return nil, fmt.Errorf("can't attach service container to the agent network: %w", err) + } } } diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index 23f730b850..a5e8eb998f 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -95,7 +95,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D // FIXME: Currently, this service deployer starts a new agent on its own and // it cannot use directly the `svcInfo.AgentHostname` value svcInfo.Hostname = d.agentHostname() - svcInfo.AgentHostname = d.agentHostname() + svcInfo.AgentHostname = dockerCustomAgentName // Alias for custom agent env := append( appConfig.StackImageRefs(d.stackVersion).AsEnv(), @@ -163,6 +163,10 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D if err != nil { return nil, fmt.Errorf("could not boot up service using Docker Compose: %w", err) } + + // TODO: if this agent is moved to "agentdeployer", this container should be connected + // to the network of the agent as done in servicedeployer/compose.go + // Connect service network with stack network (for the purpose of metrics collection) err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network(d.profile)) if err != nil { diff --git a/internal/servicedeployer/factory.go b/internal/servicedeployer/factory.go index fb6f27dc6b..45f4dd2fb0 100644 --- a/internal/servicedeployer/factory.go +++ b/internal/servicedeployer/factory.go @@ -76,11 +76,12 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { return nil, fmt.Errorf("can't use service variant: %w", err) } opts := DockerComposeServiceDeployerOptions{ - Profile: options.Profile, - YmlPaths: []string{dockerComposeYMLPath}, - Variant: sv, - RunTearDown: options.RunTearDown, - RunTestsOnly: options.RunTestsOnly, + Profile: options.Profile, + YmlPaths: []string{dockerComposeYMLPath}, + Variant: sv, + RunTearDown: options.RunTearDown, + RunTestsOnly: options.RunTestsOnly, + DeployIndependentAgent: options.DeployIndependentAgent, } return NewDockerComposeServiceDeployer(opts) } diff --git a/internal/servicedeployer/info.go b/internal/servicedeployer/info.go index 505cebed82..c73dfdf11f 100644 --- a/internal/servicedeployer/info.go +++ b/internal/servicedeployer/info.go @@ -27,6 +27,8 @@ type ServiceInfo struct { // the Service container . AgentHostname string + AgentNetworkName string + // Ports is a list of ports that the service listens on, as addressable // from the Agent container. Ports []int diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index e534017102..3b02e2dacf 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -769,6 +769,8 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf svcInfo.Test.RunID = serviceStateData.ServiceRunID svcInfo.AgentHostname = serviceStateData.ServiceAgentHostname svcInfo.Hostname = serviceStateData.ServiceAgentHostname + // By default using agent running in the Elastic stack + svcInfo.AgentNetworkName = stack.Network(r.options.Profile) } agentDeployed, agentInfo, err := r.setupAgent(ctx, serviceOptions.Variant, agentInfo) @@ -777,6 +779,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf } if agentDeployed != nil { agentInfo = agentDeployed.Info() + svcInfo.AgentNetworkName = agentInfo.NetworkName } scenario.enrollingTime = enrollingTime @@ -1087,17 +1090,12 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp return nil, svcInfo, fmt.Errorf("could not create service runner: %w", err) } + // Elastic Agent from stack and Elastic Agents started independently + // will have a network alias "elastic-agent" that services can use + // Docker custom agents would have another alias "docker-custom-agent" svcInfo.AgentHostname = "elastic-agent" if r.options.RunIndependentElasticAgent { svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local - if agentDeployed != nil { - // Not all agents are created from "agentdeployer", currently agent and - // service containers are created from "servicedeployer" package for - // CustomAgent scenario - // And there are packages that require to run queries (requests) to the - // elastic agent container to inject logs (e.g. ti_anomali) - svcInfo.AgentHostname = agentDeployed.Info().Hostname - } } if config.Service != "" { svcInfo.Name = config.Service @@ -1191,7 +1189,7 @@ type ServiceState struct { ServiceRunID string `json:"service_info_run_id"` AgentRunID string `json:"agent_info_run_id"` AgentHostname string `json:"agent_hostname"` - ServiceAgentHostname string `json:"service_agent_hostnme"` + ServiceAgentHostname string `json:"service_agent_hostname"` } type scenarioStateOpts struct { @@ -1821,16 +1819,18 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t continue } - if runIndependentElasticAgent && agent.LocalMetadata.Host.Name != "kind-control-plane" { + if (runIndependentElasticAgent && agent.LocalMetadata.Host.Name != "kind-control-plane") || svcInfo.Agent.Host.NamePrefix == "docker-custom-agent" { + // All agents created except the ones from Kubernetes should have a different hostname logger.Debugf("Checking hostname %s in agent hostname %s", expectedHostname, agent.LocalMetadata.Host.Name) if expectedHostname != agent.LocalMetadata.Host.Name { logger.Debugf("filtered agent (invalid hostname suffix) %s - %q: %s", expectedHostname, agent.LocalMetadata.Host.Name, agent.ID) // TODO: remove continue } } + // Tags are available starting in 8.3 // Kubernetes Agent cannot set a different hostname - // Using tags allow to detect the right agent from the list + // Using tags allow us to detect the right agent from the list if runIndependentElasticAgent && len(expectedTags) > 0 && len(agent.Tags) > 0 { logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) foundAllTags := true diff --git a/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml b/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml index bda2944d81..f2b41d4d26 100644 --- a/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/other/hit_count_assertion/_dev/deploy/docker/docker-compose.yml @@ -4,4 +4,4 @@ services: image: akroh/stream:v0.0.1 volumes: - ./logs:/logs:ro - command: log --start-signal=SIGHUP --delay=5s --addr ${AGENT_HOSTNAME}:9999 -p=tcp /logs/generated.log + command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9999 -p=tcp /logs/generated.log diff --git a/test/packages/other/multiinput/_dev/deploy/docker/docker-compose.yml b/test/packages/other/multiinput/_dev/deploy/docker/docker-compose.yml index a51b762b6e..85ff94ee54 100644 --- a/test/packages/other/multiinput/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/other/multiinput/_dev/deploy/docker/docker-compose.yml @@ -4,9 +4,9 @@ services: image: akroh/stream:v0.0.1 volumes: - ./logs:/logs:ro - command: log --start-signal=SIGHUP --delay=5s --addr ${AGENT_HOSTNAME}:9999 -p=tcp /logs/generated.log + command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9999 -p=tcp /logs/generated.log test-udp: image: akroh/stream:v0.0.1 volumes: - ./logs:/logs:ro - command: log --start-signal=SIGHUP --delay=5s --addr ${AGENT_HOSTNAME}:9999 -p=udp /logs/generated.log + command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9999 -p=udp /logs/generated.log diff --git a/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml b/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml index 47d1bd6cb5..ba5c31a792 100644 --- a/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/parallel/ti_anomali/_dev/deploy/docker/docker-compose.yml @@ -18,7 +18,7 @@ services: - ./sample_logs:/sample_logs:ro environment: - STREAM_PROTOCOL=webhook - - STREAM_ADDR=http://${AGENT_HOSTNAME}:9080/ + - STREAM_ADDR=http://elastic-agent:9080/ command: log --webhook-content-type application/x-ndjson --start-signal=SIGHUP --delay=5s /sample_logs/test-threatstream-ndjson.log threatstream-webhook-https: image: docker.elastic.co/observability/stream:v0.6.1 @@ -27,7 +27,7 @@ services: environment: - STREAM_PROTOCOL=webhook - STREAM_INSECURE=true - - STREAM_ADDR=https://${AGENT_HOSTNAME}:7443/ + - STREAM_ADDR=https://elastic-agent:7443/ command: log --webhook-content-type application/x-ndjson --start-signal=SIGHUP --delay=5s /sample_logs/test-threatstream-ndjson.log threatstream-integrator-test: image: docker.io/adrisr/filebeat-anomali-integrator-test:latest From 2e57c29689fcf8984e30b16be3bde1b21df7eef8 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 26 Mar 2024 12:12:56 +0100 Subject: [PATCH 079/114] Test ti_anomali package --- .buildkite/pipeline.trigger.integration.tests.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index a637674078..e7a48e49c4 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -85,11 +85,6 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echoerr "Skip temporarily ${package_name}" continue fi - # Currently, ti_anomali package has the container name hardcoded in one of the configuration files - if [[ "$package_name" == "ti_anomali" && "$independent_agent" == "true" ]] ; then - echoerr "Skip temporarily ${package_name}" - continue - fi echo " - label: \":go: Integration test: ${package_name} - independent_agent ${independent_agent}\"" echo " key: \"integration-parallel-${package_name}-agent-${independent_agent}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" From beddc7ca99bab0119c9af34c679bc6e0b11806cb Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 26 Mar 2024 12:44:58 +0100 Subject: [PATCH 080/114] Re-add test package aws --- .buildkite/pipeline.trigger.integration.tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index e7a48e49c4..9db1d24129 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -81,7 +81,7 @@ pushd test/packages/parallel > /dev/null for independent_agent in false true; do for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do package_name=$(basename "${package}") - if [[ "$package_name" == "aws" || "$package_name" == "aws_logs" ]] ; then + if [[ "$package_name" == "aws_logs" ]] ; then echoerr "Skip temporarily ${package_name}" continue fi From f2dd19c5e78c3bc6c5cdc69599c2a8d9e32e4d98 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 26 Mar 2024 13:44:03 +0100 Subject: [PATCH 081/114] Re-add test package awslogs --- .buildkite/pipeline.trigger.integration.tests.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 9db1d24129..abdba7203a 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -3,10 +3,6 @@ # exit immediately on failure, or if an undefined variable is used set -eu -echoerr() { - echo "$@" 1>&2 -} - # begin the pipeline.yml file echo "steps:" echo " - group: \":terminal: Integration test suite\"" @@ -81,10 +77,6 @@ pushd test/packages/parallel > /dev/null for independent_agent in false true; do for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do package_name=$(basename "${package}") - if [[ "$package_name" == "aws_logs" ]] ; then - echoerr "Skip temporarily ${package_name}" - continue - fi echo " - label: \":go: Integration test: ${package_name} - independent_agent ${independent_agent}\"" echo " key: \"integration-parallel-${package_name}-agent-${independent_agent}\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" From 94bcd1b71ed1f153b8d9baf5a6ffe9281536dab0 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 3 Apr 2024 17:12:57 +0200 Subject: [PATCH 082/114] Removed return error from SetInfo --- internal/agentdeployer/agent.go | 5 ++--- internal/agentdeployer/deployed_agent.go | 12 ++++++------ internal/agentdeployer/kubernetes.go | 5 ++--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index bc85baa2eb..00b40de5fc 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -387,7 +387,6 @@ func (s *dockerComposeDeployedAgent) Info() AgentInfo { } // SetInfo sets the current context for the agent. -func (s *dockerComposeDeployedAgent) SetInfo(ctxt AgentInfo) error { - s.agentInfo = ctxt - return nil +func (s *dockerComposeDeployedAgent) SetInfo(info AgentInfo) { + s.agentInfo = info } diff --git a/internal/agentdeployer/deployed_agent.go b/internal/agentdeployer/deployed_agent.go index 4a036d8584..ec372b4cdf 100644 --- a/internal/agentdeployer/deployed_agent.go +++ b/internal/agentdeployer/deployed_agent.go @@ -12,19 +12,19 @@ import ( var ErrNotSupported error = errors.New("not supported") -// DeployedAgent defines the interface for interacting with a service that has been deployed. +// DeployedAgent defines the interface for interacting with an agent that has been deployed. type DeployedAgent interface { - // TearDown implements the logic for tearing down a service. + // TearDown implements the logic for tearing down an agent. TearDown(ctx context.Context) error - // Signal sends a signal to the service. + // Signal sends a signal to the agent. Signal(ctx context.Context, signal string) error - // Info returns the current context from the service. + // Info returns the current information from the agent. Info() AgentInfo - // SetInfo sets the current context for the service. - SetInfo(str AgentInfo) error + // SetInfo sets the current information about the agent. + SetInfo(AgentInfo) // ExitCode returns true if the service is exited and its exit code. ExitCode(ctx context.Context, service string) (bool, int, error) diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index 367806799a..c6fd352386 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -76,9 +76,8 @@ func (s kubernetesDeployedAgent) Info() AgentInfo { return s.agentInfo } -func (s *kubernetesDeployedAgent) SetInfo(sc AgentInfo) error { - s.agentInfo = sc - return nil +func (s *kubernetesDeployedAgent) SetInfo(info AgentInfo) { + s.agentInfo = info } // Logs returns the logs from the agent starting at the given time From 36ac22efcafa30453a70f696e38aa4cfb0024cce Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 3 Apr 2024 17:24:23 +0200 Subject: [PATCH 083/114] Remove Signal from DeployedAgent interface --- internal/agentdeployer/agent.go | 25 ------------------------ internal/agentdeployer/deployed_agent.go | 3 --- internal/agentdeployer/kubernetes.go | 4 ---- 3 files changed, 32 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 00b40de5fc..b078bd97b9 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -281,31 +281,6 @@ func CreateServiceLogsDir(elasticPackagePath *locations.LocationManager, name st return dirPath, nil } -// Signal sends a signal to the agent. -func (s *dockerComposeDeployedAgent) Signal(ctx context.Context, signal string) error { - p, err := compose.NewProject(s.project, s.ymlPaths...) - if err != nil { - return fmt.Errorf("could not create Docker Compose project for service: %w", err) - } - - opts := compose.CommandOptions{ - Env: append( - s.env, - s.variant.Env..., - ), - ExtraArgs: []string{"-s", signal}, - } - if s.agentInfo.Name != "" { - opts.Services = append(opts.Services, s.agentInfo.Name) - } - - err = p.Kill(ctx, opts) - if err != nil { - return fmt.Errorf("could not send %q signal: %w", signal, err) - } - return nil -} - // ExitCode returns true if the agent is exited and its exit code. func (s *dockerComposeDeployedAgent) ExitCode(ctx context.Context, agent string) (bool, int, error) { p, err := compose.NewProject(s.project, s.ymlPaths...) diff --git a/internal/agentdeployer/deployed_agent.go b/internal/agentdeployer/deployed_agent.go index ec372b4cdf..7af4b37838 100644 --- a/internal/agentdeployer/deployed_agent.go +++ b/internal/agentdeployer/deployed_agent.go @@ -17,9 +17,6 @@ type DeployedAgent interface { // TearDown implements the logic for tearing down an agent. TearDown(ctx context.Context) error - // Signal sends a signal to the agent. - Signal(ctx context.Context, signal string) error - // Info returns the current information from the agent. Info() AgentInfo diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index c6fd352386..ad368ae19b 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -64,10 +64,6 @@ func (s kubernetesDeployedAgent) TearDown(ctx context.Context) error { return nil } -func (s kubernetesDeployedAgent) Signal(ctx context.Context, _ string) error { - return ErrNotSupported -} - func (s kubernetesDeployedAgent) ExitCode(ctx context.Context, _ string) (bool, int, error) { return false, -1, ErrNotSupported } From 447482c6f52fbdbb96ee60ef9dc897a06cc2995b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 3 Apr 2024 18:00:49 +0200 Subject: [PATCH 084/114] Use agent name from object in ExitCode --- internal/agentdeployer/agent.go | 4 ++-- internal/agentdeployer/deployed_agent.go | 4 ++-- internal/agentdeployer/kubernetes.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index b078bd97b9..52026f9130 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -282,7 +282,7 @@ func CreateServiceLogsDir(elasticPackagePath *locations.LocationManager, name st } // ExitCode returns true if the agent is exited and its exit code. -func (s *dockerComposeDeployedAgent) ExitCode(ctx context.Context, agent string) (bool, int, error) { +func (s *dockerComposeDeployedAgent) ExitCode(ctx context.Context) (bool, int, error) { p, err := compose.NewProject(s.project, s.ymlPaths...) if err != nil { return false, -1, fmt.Errorf("could not create Docker Compose project for agent: %w", err) @@ -295,7 +295,7 @@ func (s *dockerComposeDeployedAgent) ExitCode(ctx context.Context, agent string) ), } - return p.ServiceExitCode(ctx, agent, opts) + return p.ServiceExitCode(ctx, s.agentInfo.Name, opts) } // Logs returns the logs from the agent starting at the given time diff --git a/internal/agentdeployer/deployed_agent.go b/internal/agentdeployer/deployed_agent.go index 7af4b37838..6c49aef23b 100644 --- a/internal/agentdeployer/deployed_agent.go +++ b/internal/agentdeployer/deployed_agent.go @@ -23,8 +23,8 @@ type DeployedAgent interface { // SetInfo sets the current information about the agent. SetInfo(AgentInfo) - // ExitCode returns true if the service is exited and its exit code. - ExitCode(ctx context.Context, service string) (bool, int, error) + // ExitCode returns true if the agent is exited and its exit code. + ExitCode(ctx context.Context) (bool, int, error) // Logs returns the logs from the agent starting at the given time Logs(ctx context.Context, t time.Time) ([]byte, error) diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index ad368ae19b..d3951d3261 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -64,7 +64,7 @@ func (s kubernetesDeployedAgent) TearDown(ctx context.Context) error { return nil } -func (s kubernetesDeployedAgent) ExitCode(ctx context.Context, _ string) (bool, int, error) { +func (s kubernetesDeployedAgent) ExitCode(ctx context.Context) (bool, int, error) { return false, -1, ErrNotSupported } From 33a8eb9021c236b9f6107a263b4e520c0b4d7736 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 3 Apr 2024 18:35:04 +0200 Subject: [PATCH 085/114] Document NetworkName attribute --- internal/agentdeployer/info.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go index f40837cdf3..92ef6d5819 100644 --- a/internal/agentdeployer/info.go +++ b/internal/agentdeployer/info.go @@ -24,6 +24,8 @@ type AgentInfo struct { // the Agent container. Hostname string + // NetworkName is the name of the docker network created for the agent, + // required to connect the Service with the agent. NetworkName string // Ports is a list of ports that the service listens on, as addressable From c920bcef1f2eb415dbbbe806136b5b25d87c3a61 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 3 Apr 2024 19:09:11 +0200 Subject: [PATCH 086/114] Add new filter agents method for independent agents --- internal/testrunner/runners/system/runner.go | 69 +++++++++++++------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 50f67f395d..7518832337 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1383,7 +1383,11 @@ func checkEnrolledAgents(ctx context.Context, client *kibana.Client, agentInfo a return false, fmt.Errorf("could not list agents: %w", err) } - agents = filterAgents(allAgents, agentInfo, threshold, svcInfo, runIndependentElasticAgent) + if runIndependentElasticAgent { + agents = filterIndependentAgents(allAgents, agentInfo, threshold, svcInfo) + } else { + agents = filterAgents(allAgents, svcInfo) + } logger.Debugf("found %d enrolled agent(s)", len(agents)) if len(agents) == 0 { return false, nil // selected agents are unavailable yet @@ -1786,7 +1790,39 @@ func deleteDataStreamDocs(ctx context.Context, api *elasticsearch.API, dataStrea return nil } -func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, threshold time.Time, svcInfo servicedeployer.ServiceInfo, runIndependentElasticAgent bool) []kibana.Agent { +func filterAgents(allAgents []kibana.Agent, svcInfo servicedeployer.ServiceInfo) []kibana.Agent { + if svcInfo.Agent.Host.NamePrefix != "" { + logger.Debugf("filter agents using service criteria: NamePrefix=%s", svcInfo.Agent.Host.NamePrefix) + } + + // filtered list of agents must contain all agents started by the stack + // they could be assigned the default policy (elastic-agent-managed-ep) or the test policy (ep-test-system-*) + var filtered []kibana.Agent + for _, agent := range allAgents { + if agent.PolicyRevision == 0 { + continue // For some reason Kibana doesn't always return a valid policy revision (eventually it will be present and valid) + } + + // It cannot filtered by "elastic-agent-managed-ep" , since this is the default + // policy assigned to the agents when they first enroll + if agent.PolicyID == "fleet-server-policy" { + continue + } + + if agent.Status != "online" { + continue + } + + hasServicePrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, svcInfo.Agent.Host.NamePrefix) + if svcInfo.Agent.Host.NamePrefix != "" && !hasServicePrefix { + continue + } + filtered = append(filtered, agent) + } + return filtered +} + +func filterIndependentAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, threshold time.Time, svcInfo servicedeployer.ServiceInfo) []kibana.Agent { if agentInfo.Agent.Host.NamePrefix != "" { logger.Debugf("filter agents using agent criteria: NamePrefix=%s", agentInfo.Agent.Host.NamePrefix) } @@ -1823,12 +1859,12 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t continue } - if runIndependentElasticAgent && agent.EnrolledAt.Before(threshold) { + if agent.EnrolledAt.Before(threshold) { logger.Debugf("filtered agent (enrolling time) %q", agent.ID) // TODO: remove continue } - if (runIndependentElasticAgent && agent.LocalMetadata.Host.Name != "kind-control-plane") || svcInfo.Agent.Host.NamePrefix == "docker-custom-agent" { + if agent.LocalMetadata.Host.Name != "kind-control-plane" { // All agents created except the ones from Kubernetes should have a different hostname logger.Debugf("Checking hostname %s in agent hostname %s", expectedHostname, agent.LocalMetadata.Host.Name) if expectedHostname != agent.LocalMetadata.Host.Name { @@ -1840,7 +1876,7 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t // Tags are available starting in 8.3 // Kubernetes Agent cannot set a different hostname // Using tags allow us to detect the right agent from the list - if runIndependentElasticAgent && len(expectedTags) > 0 && len(agent.Tags) > 0 { + if len(expectedTags) > 0 && len(agent.Tags) > 0 { logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) foundAllTags := true for _, tag := range expectedTags { @@ -1855,23 +1891,12 @@ func filterAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, t } } - if runIndependentElasticAgent { - // FIXME: check for package and data stream name too ? - // Current verson could be returning an unexpected agent if tests are parallelized - hasAgentPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) - // if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix && !hasServicePrefix { - if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix { - logger.Debugf("filtered agent (prefix) %q", agent.ID) // TODO: remove - continue - } - } else { - // TODO: required for custom agents triggered from service deployers ? - hasServicePrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, svcInfo.Agent.Host.NamePrefix) - if svcInfo.Agent.Host.NamePrefix != "" && !hasServicePrefix { - logger.Debugf("filtered agent (prefix) %q", agent.ID) // TODO: remove - continue - } - + // FIXME: check for package and data stream name too ? + // Current version could be returning an unexpected agent if tests are parallelized + hasAgentPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) + if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix { + logger.Debugf("filtered agent (prefix) %q", agent.ID) // TODO: remove + continue } filtered = append(filtered, agent) } From 4e363a226e1626d816d8b9797451a172a73f9230 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 12:08:29 +0200 Subject: [PATCH 087/114] Use test agent policy to enroll agents --- internal/agentdeployer/agent.go | 7 +- internal/agentdeployer/factory.go | 3 + internal/agentdeployer/info.go | 3 + internal/agentdeployer/kubernetes.go | 19 +- internal/kibana/agents.go | 17 ++ internal/testrunner/runners/system/runner.go | 173 +++++++++---------- 6 files changed, 119 insertions(+), 103 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 52026f9130..0e412c3c28 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -40,7 +40,8 @@ type DockerComposeAgentDeployer struct { dockerComposeFile string stackVersion string - variant AgentVariant + variant AgentVariant + policyName string agentRunID string @@ -56,6 +57,7 @@ type DockerComposeAgentDeployerOptions struct { DockerComposeFile string StackVersion string Variant AgentVariant + PolicyName string PackageName string DataStream string @@ -85,6 +87,7 @@ func NewCustomAgentDeployer(options DockerComposeAgentDeployerOptions) (*DockerC stackVersion: options.StackVersion, packageName: options.PackageName, dataStream: options.DataStream, + policyName: options.PolicyName, variant: options.Variant, runTearDown: options.RunTearDown, runTestsOnly: options.RunTestsOnly, @@ -108,7 +111,7 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI // Local Elastic stacks have a default Agent Policy created, // but Cloud or Serverless Projects could not have one - agentPolicyName := defaultAgentPolicyName + agentPolicyName := d.policyName if strings.HasPrefix(d.stackVersion, "7.") { // Local Elastic stacks 7.* have an Agent Policy that is set as default // No need to set an Agent Policy Name diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go index 570d31cfaa..50ce1143c1 100644 --- a/internal/agentdeployer/factory.go +++ b/internal/agentdeployer/factory.go @@ -28,6 +28,7 @@ type FactoryOptions struct { DevDeployDir string Type string StackVersion string + PolicyName string PackageName string DataStream string @@ -75,6 +76,7 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { Variant: variant, StackVersion: options.StackVersion, PackageName: options.PackageName, + PolicyName: options.PolicyName, DataStream: options.DataStream, RunTearDown: options.RunTearDown, RunTestsOnly: options.RunTestsOnly, @@ -112,6 +114,7 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { Profile: options.Profile, DefinitionsDir: agentDeployerPath, StackVersion: options.StackVersion, + PolicyName: options.PolicyName, RunSetup: options.RunSetup, RunTestsOnly: options.RunTestsOnly, RunTearDown: options.RunTearDown, diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go index 92ef6d5819..f1748e7b71 100644 --- a/internal/agentdeployer/info.go +++ b/internal/agentdeployer/info.go @@ -28,6 +28,9 @@ type AgentInfo struct { // required to connect the Service with the agent. NetworkName string + PolicyName string + PolicyID string + // Ports is a list of ports that the service listens on, as addressable // from the Agent container. Ports []int diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index d3951d3261..9fd3ae0566 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -28,6 +28,7 @@ type KubernetesAgentDeployer struct { profile *profile.Profile definitionsDir string stackVersion string + policyName string runSetup bool runTestsOnly bool @@ -38,6 +39,7 @@ type KubernetesAgentDeployerOptions struct { Profile *profile.Profile DefinitionsDir string StackVersion string + PolicyName string RunSetup bool RunTestsOnly bool @@ -53,7 +55,7 @@ type kubernetesDeployedAgent struct { } func (s kubernetesDeployedAgent) TearDown(ctx context.Context) error { - elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion, s.agentInfo.Tags) + elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion, s.agentInfo.PolicyName, s.agentInfo.Tags) if err != nil { return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) } @@ -89,6 +91,7 @@ func NewKubernetesAgentDeployer(opts KubernetesAgentDeployerOptions) (*Kubernete profile: opts.Profile, definitionsDir: opts.DefinitionsDir, stackVersion: opts.StackVersion, + policyName: opts.PolicyName, runSetup: opts.RunSetup, runTestsOnly: opts.RunTestsOnly, runTearDown: opts.RunTearDown, @@ -115,7 +118,7 @@ func (ksd KubernetesAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInf if ksd.runTearDown || ksd.runTestsOnly { logger.Debug("Skip install Elastic Agent in cluster") } else { - err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion, agentInfo.Tags) + err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion, agentInfo.PolicyName, agentInfo.Tags) if err != nil { return nil, fmt.Errorf("can't install Elastic-Agent in the Kubernetes cluster: %w", err) } @@ -136,10 +139,10 @@ func (ksd KubernetesAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInf var _ AgentDeployer = new(KubernetesAgentDeployer) -func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, stackVersion string, tags []string) error { +func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, stackVersion, policyName string, tags []string) error { logger.Debug("install Elastic Agent in the Kubernetes cluster") - elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion, tags) + elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion, policyName, tags) if err != nil { return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) } @@ -157,7 +160,7 @@ func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, //go:embed _static/elastic-agent-managed.yaml.tmpl var elasticAgentManagedYamlTmpl string -func getElasticAgentYAML(profile *profile.Profile, stackVersion string, tags []string) ([]byte, error) { +func getElasticAgentYAML(profile *profile.Profile, stackVersion, policyName string, tags []string) ([]byte, error) { logger.Debugf("Prepare YAML definition for Elastic Agent running in stack v%s", stackVersion) appConfig, err := install.Configuration() @@ -178,7 +181,7 @@ func getElasticAgentYAML(profile *profile.Profile, stackVersion string, tags []s "kibanaURL": "https://kibana:5601", "caCertPem": caCert, "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, - "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion), + "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion, policyName), "elasticAgentTags": strings.Join(tags, ","), }) if err != nil { @@ -204,9 +207,9 @@ func readCACertBase64(profile *profile.Profile) (string, error) { // getTokenPolicyName function returns the policy name for the 8.x Elastic stack. The agent's policy // is predefined in the Kibana configuration file. The logic is not present in older stacks. -func getTokenPolicyName(stackVersion string) string { +func getTokenPolicyName(stackVersion, policyName string) string { if strings.HasPrefix(stackVersion, "8.") { - return "Elastic-Agent (elastic-package)" + return policyName } return "" } diff --git a/internal/kibana/agents.go b/internal/kibana/agents.go index 979003dad9..de3210d0a0 100644 --- a/internal/kibana/agents.go +++ b/internal/kibana/agents.go @@ -92,6 +92,23 @@ func (c *Client) AssignPolicyToAgent(ctx context.Context, a Agent, p Policy) err return nil } +// UnenrollAgent unenrolls the given agent +func (c *Client) UnenrollAgent(ctx context.Context, a Agent) error { + reqBody := `{ "revoke": true, "force": true }` + + path := fmt.Sprintf("%s/agents/%s/unenroll", FleetAPI, a.ID) + statusCode, respBody, err := c.post(ctx, path, []byte(reqBody)) + if err != nil { + return fmt.Errorf("could not enroll agent: %w", err) + } + + if statusCode != http.StatusOK { + return fmt.Errorf("could not enroll agent; API status code = %d; response body = %s", statusCode, respBody) + } + + return nil +} + func (c *Client) waitUntilPolicyAssigned(ctx context.Context, a Agent, p Policy) error { ctx, cancel := context.WithTimeout(ctx, waitForPolicyAssignedTimeout) defer cancel() diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 7518832337..385911fe15 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -111,6 +111,7 @@ type runner struct { serviceStateFilePath string // Execution order of following handlers is defined in runner.TearDown() method. + unenrollAgentHandler func(context.Context) error deleteTestPolicyHandler func(context.Context) error deletePackageHandler func(context.Context) error resetAgentPolicyHandler func(context.Context) error @@ -263,7 +264,7 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes return result.WithSuccess() } -func (r *runner) createAgentOptions(variantName string) agentdeployer.FactoryOptions { +func (r *runner) createAgentOptions(variantName, policyName string) agentdeployer.FactoryOptions { return agentdeployer.FactoryOptions{ Profile: r.options.Profile, PackageRootPath: r.options.PackageRootPath, @@ -273,6 +274,7 @@ func (r *runner) createAgentOptions(variantName string) agentdeployer.FactoryOpt StackVersion: r.stackVersion.Version(), PackageName: r.options.TestFolder.Package, DataStream: r.options.TestFolder.DataStream, + PolicyName: policyName, Variant: variantName, RunTearDown: r.options.RunTearDown, RunTestsOnly: r.options.RunTestsOnly, @@ -298,7 +300,7 @@ func (r *runner) createServiceOptions(variantName string) servicedeployer.Factor } } -func (r *runner) createAgentInfo() (agentdeployer.AgentInfo, error) { +func (r *runner) createAgentInfo(policy *kibana.Policy) (agentdeployer.AgentInfo, error) { var info agentdeployer.AgentInfo info.Name = r.options.TestFolder.Package @@ -319,6 +321,9 @@ func (r *runner) createAgentInfo() (agentdeployer.AgentInfo, error) { } info.Logs.Folder.Local = dirPath + info.PolicyName = policy.Name + info.PolicyID = policy.ID + return info, nil } @@ -373,6 +378,13 @@ func (r *runner) tearDownTest(ctx context.Context) error { r.resetAgentLogLevelHandler = nil } + if r.unenrollAgentHandler != nil { + if err := r.unenrollAgentHandler(cleanupCtx); err != nil { + return err + } + r.unenrollAgentHandler = nil + } + if r.deleteTestPolicyHandler != nil { if err := r.deleteTestPolicyHandler(cleanupCtx); err != nil { return err @@ -750,9 +762,41 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf if err != nil { return nil, fmt.Errorf("failed to find the selected policy_template: %w", err) } + + // Configure package (single data stream) via Fleet APIs. + var policy *kibana.Policy + if r.options.RunTearDown || r.options.RunTestsOnly { + policy = &serviceStateData.CurrentPolicy + logger.Debugf("Got policy from file: %q - %q", policy.Name, policy.ID) + } else { + logger.Debug("creating test policy...") + testTime := time.Now().Format("20060102T15:04:05Z") + + p := kibana.Policy{ + Name: fmt.Sprintf("ep-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime), + Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream), + Namespace: "ep", + } + // Assign the data_output_id to the agent policy to configure the output to logstash. The value is inferred from stack/_static/kibana.yml.tmpl + if r.options.Profile.Config("stack.logstash_enabled", "false") == "true" { + p.DataOutputID = "fleet-logstash-output" + } + policy, err = r.options.KibanaClient.CreatePolicy(ctx, p) + if err != nil { + return nil, fmt.Errorf("could not create test policy: %w", err) + } + } + r.deleteTestPolicyHandler = func(ctx context.Context) error { + logger.Debug("deleting test policy...") + if err := r.options.KibanaClient.DeletePolicy(ctx, *policy); err != nil { + return fmt.Errorf("error cleaning up test policy: %w", err) + } + return nil + } + // Setup agent logger.Debug("setting up agent...") - agentInfo, err := r.createAgentInfo() + agentInfo, err := r.createAgentInfo(policy) if err != nil { return nil, err } @@ -852,37 +896,6 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf } r.deletePackageHandler = deletePackageHandler - // Configure package (single data stream) via Fleet APIs. - var policy *kibana.Policy - if r.options.RunTearDown || r.options.RunTestsOnly { - policy = &serviceStateData.CurrentPolicy - logger.Debugf("Got policy from file: %q - %q", policy.Name, policy.ID) - } else { - logger.Debug("creating test policy...") - testTime := time.Now().Format("20060102T15:04:05Z") - - p := kibana.Policy{ - Name: fmt.Sprintf("ep-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime), - Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream), - Namespace: "ep", - } - // Assign the data_output_id to the agent policy to configure the output to logstash. The value is inferred from stack/_static/kibana.yml.tmpl - if r.options.Profile.Config("stack.logstash_enabled", "false") == "true" { - p.DataOutputID = "fleet-logstash-output" - } - policy, err = r.options.KibanaClient.CreatePolicy(ctx, p) - if err != nil { - return nil, fmt.Errorf("could not create test policy: %w", err) - } - } - r.deleteTestPolicyHandler = func(ctx context.Context) error { - logger.Debug("deleting test policy...") - if err := r.options.KibanaClient.DeletePolicy(ctx, *policy); err != nil { - return fmt.Errorf("error cleaning up test policy: %w", err) - } - return nil - } - logger.Debug("adding package data stream to test policy...") ds := createPackageDatastream(*policy, *scenario.pkgManifest, policyTemplate, *scenario.dataStreamManifest, *config) if r.options.RunTearDown || r.options.RunTestsOnly { @@ -944,13 +957,35 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf logger.Debugf("JSON Agents:\n%s", string(data)) agent := agents[0] + r.unenrollAgentHandler = func(ctx context.Context) error { + if !r.options.RunIndependentElasticAgent { + return nil + } + logger.Debug("unenrolling agent...") + err := r.options.KibanaClient.UnenrollAgent(ctx, agent) + if err != nil { + return fmt.Errorf("failed to unenroll agent %q: %w", agent.ID, err) + } + return nil + } + if r.options.RunTearDown || r.options.RunTestsOnly { origPolicy = serviceStateData.OrigPolicy logger.Debugf("Got orig policy from file: %q - %q", origPolicy.Name, origPolicy.ID) } else { - origPolicy = kibana.Policy{ - ID: agent.PolicyID, - Revision: agent.PolicyRevision, + if r.options.RunIndependentElasticAgent { + // Get default policy to assign agents after testing is finished + defaultPolicy, err := r.options.KibanaClient.GetPolicy(ctx, "elastic-agent-managed-ep") + if err != nil { + return nil, fmt.Errorf("failed to get default policy") + } + origPolicy = *defaultPolicy + } else { + // Store previous agent policy assigned to the agent + origPolicy = kibana.Policy{ + ID: agent.PolicyID, + Revision: agent.PolicyRevision, + } } } // Assign policy to agent @@ -1103,7 +1138,9 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp // will have a network alias "elastic-agent" that services can use // Docker custom agents would have another alias "docker-custom-agent" svcInfo.AgentHostname = "elastic-agent" - if r.options.RunIndependentElasticAgent { + + // Set the right folder for logs execpt for custom agents that are still deployed using "servicedeployer" + if r.options.RunIndependentElasticAgent && agentDeployed != nil { svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local } if config.Service != "" { @@ -1129,7 +1166,7 @@ func (r *runner) setupAgent(ctx context.Context, variant string, agentInfo agent if !r.options.RunIndependentElasticAgent { return nil, agentInfo, nil } - agentOptions := r.createAgentOptions(variant) + agentOptions := r.createAgentOptions(variant, agentInfo.PolicyName) agentDeployer, err := agentdeployer.Factory(agentOptions) if err != nil { return nil, agentInfo, fmt.Errorf("could not create agent runner: %w", err) @@ -1384,7 +1421,7 @@ func checkEnrolledAgents(ctx context.Context, client *kibana.Client, agentInfo a } if runIndependentElasticAgent { - agents = filterIndependentAgents(allAgents, agentInfo, threshold, svcInfo) + agents = filterIndependentAgents(allAgents, agentInfo, svcInfo) } else { agents = filterAgents(allAgents, svcInfo) } @@ -1822,23 +1859,7 @@ func filterAgents(allAgents []kibana.Agent, svcInfo servicedeployer.ServiceInfo) return filtered } -func filterIndependentAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, threshold time.Time, svcInfo servicedeployer.ServiceInfo) []kibana.Agent { - if agentInfo.Agent.Host.NamePrefix != "" { - logger.Debugf("filter agents using agent criteria: NamePrefix=%s", agentInfo.Agent.Host.NamePrefix) - } - - if svcInfo.Agent.Host.NamePrefix != "" { - logger.Debugf("filter agents using service criteria: NamePrefix=%s", svcInfo.Agent.Host.NamePrefix) - } - - expectedHostname := agentInfo.Hostname - expectedTags := agentInfo.Tags - if svcInfo.Agent.Host.NamePrefix == "docker-custom-agent" { - // custom agents are still started from servicedeployer, they are defined in the same docker compose scenario - expectedHostname = svcInfo.AgentHostname - expectedTags = svcInfo.Tags - } - +func filterIndependentAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, svcInfo servicedeployer.ServiceInfo) []kibana.Agent { // filtered list of agents must contain all agents started by the stack // they could be assigned the default policy (elastic-agent-managed-ep) or the test policy (ep-test-system-*) var filtered []kibana.Agent @@ -1859,45 +1880,11 @@ func filterIndependentAgents(allAgents []kibana.Agent, agentInfo agentdeployer.A continue } - if agent.EnrolledAt.Before(threshold) { - logger.Debugf("filtered agent (enrolling time) %q", agent.ID) // TODO: remove + if agent.PolicyID != agentInfo.PolicyID { + logger.Debugf("filtered agent (not same policy) %q", agent.ID) continue } - if agent.LocalMetadata.Host.Name != "kind-control-plane" { - // All agents created except the ones from Kubernetes should have a different hostname - logger.Debugf("Checking hostname %s in agent hostname %s", expectedHostname, agent.LocalMetadata.Host.Name) - if expectedHostname != agent.LocalMetadata.Host.Name { - logger.Debugf("filtered agent (invalid hostname suffix) %s - %q: %s", expectedHostname, agent.LocalMetadata.Host.Name, agent.ID) // TODO: remove - continue - } - } - - // Tags are available starting in 8.3 - // Kubernetes Agent cannot set a different hostname - // Using tags allow us to detect the right agent from the list - if len(expectedTags) > 0 && len(agent.Tags) > 0 { - logger.Debugf("Checking tags %s vs %s", strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) - foundAllTags := true - for _, tag := range expectedTags { - if !slices.Contains(agent.Tags, tag) { - logger.Debugf("filtered agent (invalid tag found) %s - %q vs %q", tag, strings.Join(agent.Tags, ","), strings.Join(expectedTags, ",")) // TODO: remove - foundAllTags = false - break - } - } - if !foundAllTags { - continue - } - } - - // FIXME: check for package and data stream name too ? - // Current version could be returning an unexpected agent if tests are parallelized - hasAgentPrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, agentInfo.Agent.Host.NamePrefix) - if agentInfo.Agent.Host.NamePrefix != "" && !hasAgentPrefix { - logger.Debugf("filtered agent (prefix) %q", agent.ID) // TODO: remove - continue - } filtered = append(filtered, agent) } return filtered From 53ce8155961a9946bcc591f68fd0aed9d84cc81e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 12:20:13 +0200 Subject: [PATCH 088/114] Support to set agent policy in custom agents (servicedeployer) --- .../_static/docker-custom-agent-base.yml | 1 + internal/servicedeployer/custom_agent.go | 4 ++++ internal/servicedeployer/factory.go | 5 +++++ internal/servicedeployer/info.go | 3 +++ internal/servicedeployer/kubernetes.go | 6 +++--- internal/testrunner/runners/system/runner.go | 17 ++++++++++++----- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/internal/servicedeployer/_static/docker-custom-agent-base.yml b/internal/servicedeployer/_static/docker-custom-agent-base.yml index f0f4dbc54e..c39507787a 100644 --- a/internal/servicedeployer/_static/docker-custom-agent-base.yml +++ b/internal/servicedeployer/_static/docker-custom-agent-base.yml @@ -12,6 +12,7 @@ services: - FLEET_URL=https://fleet-server:8220 - KIBANA_HOST=https://kibana:5601 - ELASTIC_AGENT_TAGS=${ELASTIC_AGENT_TAGS} + - FLEET_TOKEN_POLICY_NAME=${FLEET_TOKEN_POLICY_NAME} volumes: - ${SERVICE_LOGS_DIR}:/tmp/service_logs/ - ${LOCAL_CA_CERT}:/etc/ssl/certs/elastic-package.pem diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index a5e8eb998f..1190e51f38 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -40,6 +40,7 @@ type CustomAgentDeployer struct { variant ServiceVariant packageName string dataStream string + policyName string agentRunID string @@ -54,6 +55,7 @@ type CustomAgentDeployerOptions struct { Variant ServiceVariant PackageName string DataStream string + PolicyName string RunTearDown bool RunTestsOnly bool @@ -69,6 +71,7 @@ func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDep stackVersion: options.StackVersion, variant: options.Variant, packageName: options.PackageName, + policyName: options.PolicyName, dataStream: options.DataStream, runTearDown: options.RunTearDown, runTestsOnly: options.RunTestsOnly, @@ -102,6 +105,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D fmt.Sprintf("%s=%s", serviceLogsDirEnv, svcInfo.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), fmt.Sprintf("%s=%s", agentHostnameEnv, svcInfo.AgentHostname), + fmt.Sprintf("%s=%s", fleetPolicyEnv, d.policyName), fmt.Sprintf("%s=%s", elasticAgentTagsEnv, strings.Join(svcInfo.Tags, ",")), ) diff --git a/internal/servicedeployer/factory.go b/internal/servicedeployer/factory.go index 45f4dd2fb0..e510e6a5ab 100644 --- a/internal/servicedeployer/factory.go +++ b/internal/servicedeployer/factory.go @@ -29,6 +29,8 @@ type FactoryOptions struct { StackVersion string DeployIndependentAgent bool + PolicyName string + PackageName string DataStream string @@ -98,6 +100,8 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { if err != nil { return nil, fmt.Errorf("can't use service variant: %w", err) } + policyName := getTokenPolicyName(options.StackVersion, options.PolicyName) + opts := CustomAgentDeployerOptions{ Profile: options.Profile, DockerComposeFile: customAgentCfgYMLPath, @@ -105,6 +109,7 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { Variant: sv, PackageName: options.PackageName, DataStream: options.DataStream, + PolicyName: policyName, RunTearDown: options.RunTearDown, RunTestsOnly: options.RunTestsOnly, diff --git a/internal/servicedeployer/info.go b/internal/servicedeployer/info.go index c73dfdf11f..cf97b08143 100644 --- a/internal/servicedeployer/info.go +++ b/internal/servicedeployer/info.go @@ -10,6 +10,9 @@ const ( testRunIDEnv = "TEST_RUN_ID" agentHostnameEnv = "AGENT_HOSTNAME" elasticAgentTagsEnv = "ELASTIC_AGENT_TAGS" + fleetPolicyEnv = "FLEET_TOKEN_POLICY_NAME" + + defaulFleetTokenPolicyName = "Elastic-Agent (elastic-package)" ) // ServiceInfo encapsulates context that is both available to a ServiceDeployer and diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index f5e82363ed..fc48f6eddf 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -222,7 +222,7 @@ func getElasticAgentYAML(profile *profile.Profile, stackVersion string) ([]byte, "kibanaURL": "https://kibana:5601", "caCertPem": caCert, "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, - "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion), + "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion, defaulFleetTokenPolicyName), }) if err != nil { return nil, fmt.Errorf("can't generate elastic agent manifest: %w", err) @@ -247,9 +247,9 @@ func readCACertBase64(profile *profile.Profile) (string, error) { // getTokenPolicyName function returns the policy name for the 8.x Elastic stack. The agent's policy // is predefined in the Kibana configuration file. The logic is not present in older stacks. -func getTokenPolicyName(stackVersion string) string { +func getTokenPolicyName(stackVersion, policyName string) string { if strings.HasPrefix(stackVersion, "8.") { - return "Elastic-Agent (elastic-package)" + return policyName } return "" } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 385911fe15..fa30d86ddb 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1129,11 +1129,6 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOptions servicedeployer.FactoryOptions, svcInfo servicedeployer.ServiceInfo, agentInfo agentdeployer.AgentInfo, agentDeployed agentdeployer.DeployedAgent) (servicedeployer.DeployedService, servicedeployer.ServiceInfo, error) { logger.Debug("setting up service...") - serviceDeployer, err := servicedeployer.Factory(serviceOptions) - if err != nil { - return nil, svcInfo, fmt.Errorf("could not create service runner: %w", err) - } - // Elastic Agent from stack and Elastic Agents started independently // will have a network alias "elastic-agent" that services can use // Docker custom agents would have another alias "docker-custom-agent" @@ -1143,9 +1138,21 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp if r.options.RunIndependentElasticAgent && agentDeployed != nil { svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local } + + // In case of custom agent, update serviceOptions to include test policy too + if r.options.RunIndependentElasticAgent { + serviceOptions.PolicyName = agentInfo.PolicyName + } + if config.Service != "" { svcInfo.Name = config.Service } + + serviceDeployer, err := servicedeployer.Factory(serviceOptions) + if err != nil { + return nil, svcInfo, fmt.Errorf("could not create service runner: %w", err) + } + service, err := serviceDeployer.SetUp(ctx, svcInfo) if err != nil { return nil, svcInfo, fmt.Errorf("could not setup service: %w", err) From 89fe149e0153e37944860a649f191f93f9760b2f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 12:24:28 +0200 Subject: [PATCH 089/114] Remove unused params --- internal/testrunner/runners/system/runner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index fa30d86ddb..5dfc45d013 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -949,7 +949,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf // FIXME: running per stages does not work when multiple agents are created var origPolicy kibana.Policy - agents, err := checkEnrolledAgents(ctx, r.options.KibanaClient, agentInfo, enrollingTime, svcInfo, r.options.RunIndependentElasticAgent) + agents, err := checkEnrolledAgents(ctx, r.options.KibanaClient, agentInfo, svcInfo, r.options.RunIndependentElasticAgent) if err != nil { return nil, fmt.Errorf("can't check enrolled agents: %w", err) } @@ -1418,7 +1418,7 @@ func (r *runner) runTest(ctx context.Context, config *testConfig, svcInfo servic return r.validateTestScenario(ctx, result, scenario, config, serviceOptions) } -func checkEnrolledAgents(ctx context.Context, client *kibana.Client, agentInfo agentdeployer.AgentInfo, threshold time.Time, svcInfo servicedeployer.ServiceInfo, runIndependentElasticAgent bool) ([]kibana.Agent, error) { +func checkEnrolledAgents(ctx context.Context, client *kibana.Client, agentInfo agentdeployer.AgentInfo, svcInfo servicedeployer.ServiceInfo, runIndependentElasticAgent bool) ([]kibana.Agent, error) { var agents []kibana.Agent enrolled, err := wait.UntilTrue(ctx, func(ctx context.Context) (bool, error) { @@ -1428,7 +1428,7 @@ func checkEnrolledAgents(ctx context.Context, client *kibana.Client, agentInfo a } if runIndependentElasticAgent { - agents = filterIndependentAgents(allAgents, agentInfo, svcInfo) + agents = filterIndependentAgents(allAgents, agentInfo) } else { agents = filterAgents(allAgents, svcInfo) } @@ -1866,7 +1866,7 @@ func filterAgents(allAgents []kibana.Agent, svcInfo servicedeployer.ServiceInfo) return filtered } -func filterIndependentAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo, svcInfo servicedeployer.ServiceInfo) []kibana.Agent { +func filterIndependentAgents(allAgents []kibana.Agent, agentInfo agentdeployer.AgentInfo) []kibana.Agent { // filtered list of agents must contain all agents started by the stack // they could be assigned the default policy (elastic-agent-managed-ep) or the test policy (ep-test-system-*) var filtered []kibana.Agent From e6283fbcc34cb423d1a74ab447ee3ff50b43ed5a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 12:53:27 +0200 Subject: [PATCH 090/114] Remove tags from agents --- internal/agentdeployer/_static/docker-agent-base.yml | 1 - .../_static/elastic-agent-managed.yaml.tmpl | 2 -- internal/agentdeployer/agent.go | 1 - internal/agentdeployer/info.go | 3 --- internal/agentdeployer/kubernetes.go | 11 +++++------ .../_static/docker-custom-agent-base.yml | 1 - internal/servicedeployer/custom_agent.go | 2 -- internal/servicedeployer/info.go | 3 --- internal/testrunner/runners/system/runner.go | 10 ---------- 9 files changed, 5 insertions(+), 29 deletions(-) diff --git a/internal/agentdeployer/_static/docker-agent-base.yml b/internal/agentdeployer/_static/docker-agent-base.yml index 2fa4a5b2c4..c16f94c3ed 100644 --- a/internal/agentdeployer/_static/docker-agent-base.yml +++ b/internal/agentdeployer/_static/docker-agent-base.yml @@ -12,7 +12,6 @@ services: - FLEET_URL=https://fleet-server:8220 - KIBANA_HOST=https://kibana:5601 - FLEET_TOKEN_POLICY_NAME=${FLEET_TOKEN_POLICY_NAME} - - ELASTIC_AGENT_TAGS=${ELASTIC_AGENT_TAGS} volumes: - ${SERVICE_LOGS_DIR}:/tmp/service_logs/ - ${LOCAL_CA_CERT}:/etc/ssl/certs/elastic-package.pem diff --git a/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl b/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl index d868087f6a..c2ce38713a 100644 --- a/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl +++ b/internal/agentdeployer/_static/elastic-agent-managed.yaml.tmpl @@ -63,8 +63,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - - name: ELASTIC_AGENT_TAGS - value: "{{ .elasticAgentTags }}" securityContext: runAsUser: 0 resources: diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 0e412c3c28..cd9d6a4c89 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -124,7 +124,6 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), fmt.Sprintf("%s=%s", fleetPolicyEnv, agentPolicyName), fmt.Sprintf("%s=%s", agentHostnameEnv, d.agentHostname()), - fmt.Sprintf("%s=%s", elasticAgentTagsEnv, strings.Join(agentInfo.Tags, ",")), ) configDir, err := d.installDockerfile() diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go index f1748e7b71..d033b41772 100644 --- a/internal/agentdeployer/info.go +++ b/internal/agentdeployer/info.go @@ -79,7 +79,4 @@ type AgentInfo struct { // Service token Token string - - // Tags set in the agent - Tags []string } diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index 9fd3ae0566..8fc7f4187b 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -55,7 +55,7 @@ type kubernetesDeployedAgent struct { } func (s kubernetesDeployedAgent) TearDown(ctx context.Context) error { - elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion, s.agentInfo.PolicyName, s.agentInfo.Tags) + elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion, s.agentInfo.PolicyName) if err != nil { return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) } @@ -118,7 +118,7 @@ func (ksd KubernetesAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInf if ksd.runTearDown || ksd.runTestsOnly { logger.Debug("Skip install Elastic Agent in cluster") } else { - err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion, agentInfo.PolicyName, agentInfo.Tags) + err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion, agentInfo.PolicyName) if err != nil { return nil, fmt.Errorf("can't install Elastic-Agent in the Kubernetes cluster: %w", err) } @@ -139,10 +139,10 @@ func (ksd KubernetesAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInf var _ AgentDeployer = new(KubernetesAgentDeployer) -func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, stackVersion, policyName string, tags []string) error { +func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, stackVersion, policyName string) error { logger.Debug("install Elastic Agent in the Kubernetes cluster") - elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion, policyName, tags) + elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion, policyName) if err != nil { return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) } @@ -160,7 +160,7 @@ func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, //go:embed _static/elastic-agent-managed.yaml.tmpl var elasticAgentManagedYamlTmpl string -func getElasticAgentYAML(profile *profile.Profile, stackVersion, policyName string, tags []string) ([]byte, error) { +func getElasticAgentYAML(profile *profile.Profile, stackVersion, policyName string) ([]byte, error) { logger.Debugf("Prepare YAML definition for Elastic Agent running in stack v%s", stackVersion) appConfig, err := install.Configuration() @@ -182,7 +182,6 @@ func getElasticAgentYAML(profile *profile.Profile, stackVersion, policyName stri "caCertPem": caCert, "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion, policyName), - "elasticAgentTags": strings.Join(tags, ","), }) if err != nil { return nil, fmt.Errorf("can't generate elastic agent manifest: %w", err) diff --git a/internal/servicedeployer/_static/docker-custom-agent-base.yml b/internal/servicedeployer/_static/docker-custom-agent-base.yml index c39507787a..1077bf38c6 100644 --- a/internal/servicedeployer/_static/docker-custom-agent-base.yml +++ b/internal/servicedeployer/_static/docker-custom-agent-base.yml @@ -11,7 +11,6 @@ services: - FLEET_ENROLL=1 - FLEET_URL=https://fleet-server:8220 - KIBANA_HOST=https://kibana:5601 - - ELASTIC_AGENT_TAGS=${ELASTIC_AGENT_TAGS} - FLEET_TOKEN_POLICY_NAME=${FLEET_TOKEN_POLICY_NAME} volumes: - ${SERVICE_LOGS_DIR}:/tmp/service_logs/ diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index 1190e51f38..71dcc79f1e 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -10,7 +10,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/configuration/locations" @@ -106,7 +105,6 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), fmt.Sprintf("%s=%s", agentHostnameEnv, svcInfo.AgentHostname), fmt.Sprintf("%s=%s", fleetPolicyEnv, d.policyName), - fmt.Sprintf("%s=%s", elasticAgentTagsEnv, strings.Join(svcInfo.Tags, ",")), ) configDir, err := d.installDockerfile() diff --git a/internal/servicedeployer/info.go b/internal/servicedeployer/info.go index cf97b08143..144877b169 100644 --- a/internal/servicedeployer/info.go +++ b/internal/servicedeployer/info.go @@ -74,9 +74,6 @@ type ServiceInfo struct { // Directory to store any outputs generated OutputDir string - - // Tags assigned to the agent - Tags []string } // Aliases method returned aliases to properties of the service context. diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 5dfc45d013..964b59df90 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -307,12 +307,10 @@ func (r *runner) createAgentInfo(policy *kibana.Policy) (agentdeployer.AgentInfo info.Logs.Folder.Agent = ServiceLogsAgentDir info.Test.RunID = createTestRunID() - info.Tags = append(info.Tags, "test", "system", info.Test.RunID, r.options.TestFolder.Package) folderName := fmt.Sprintf("agent-%s", r.options.TestFolder.Package) if r.options.TestFolder.DataStream != "" { folderName = fmt.Sprintf("%s-%s", folderName, r.options.TestFolder.DataStream) - info.Tags = append(info.Tags, r.options.TestFolder.DataStream) } dirPath, err := agentdeployer.CreateServiceLogsDir(r.locationManager, folderName) @@ -340,10 +338,6 @@ func (r *runner) createServiceInfo() (servicedeployer.ServiceInfo, error) { } svcInfo.OutputDir = outputDir - svcInfo.Tags = append(svcInfo.Tags, "test", "system", svcInfo.Test.RunID, r.options.TestFolder.Package) - if r.options.TestFolder.DataStream != "" { - svcInfo.Tags = append(svcInfo.Tags, r.options.TestFolder.DataStream) - } return svcInfo, nil } @@ -805,11 +799,9 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf if r.options.RunTearDown || r.options.RunTestsOnly { enrollingTime = serviceStateData.EnrollingAgentTime - agentInfo.Tags = serviceStateData.Agent.Tags agentInfo.Test.RunID = serviceStateData.AgentRunID agentInfo.Hostname = serviceStateData.AgentHostname - svcInfo.Tags = serviceStateData.Agent.Tags svcInfo.Test.RunID = serviceStateData.ServiceRunID svcInfo.AgentHostname = serviceStateData.ServiceAgentHostname svcInfo.Hostname = serviceStateData.ServiceAgentHostname @@ -1237,8 +1229,6 @@ type ServiceState struct { ConfigFilePath string `json:"config_file_path"` VariantName string `json:"variant_name"` EnrollingAgentTime time.Time `json:"enrolling_agent_time"` - ServiceInfoTags []string `json:"service_info_tags,omitempty"` - AgentInfoTags []string `json:"agent_info_tags,omitempty"` ServiceRunID string `json:"service_info_run_id"` AgentRunID string `json:"agent_info_run_id"` AgentHostname string `json:"agent_hostname"` From 2039ad879589bb1b808fbd3bc0f6878939eb07bf Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 12:57:18 +0200 Subject: [PATCH 091/114] Restore filterAgents function as previously --- internal/testrunner/runners/system/runner.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 964b59df90..c51c4c6dc0 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1826,27 +1826,15 @@ func deleteDataStreamDocs(ctx context.Context, api *elasticsearch.API, dataStrea func filterAgents(allAgents []kibana.Agent, svcInfo servicedeployer.ServiceInfo) []kibana.Agent { if svcInfo.Agent.Host.NamePrefix != "" { - logger.Debugf("filter agents using service criteria: NamePrefix=%s", svcInfo.Agent.Host.NamePrefix) + logger.Debugf("filter agents using criteria: NamePrefix=%s", svcInfo.Agent.Host.NamePrefix) } - // filtered list of agents must contain all agents started by the stack - // they could be assigned the default policy (elastic-agent-managed-ep) or the test policy (ep-test-system-*) var filtered []kibana.Agent for _, agent := range allAgents { if agent.PolicyRevision == 0 { continue // For some reason Kibana doesn't always return a valid policy revision (eventually it will be present and valid) } - // It cannot filtered by "elastic-agent-managed-ep" , since this is the default - // policy assigned to the agents when they first enroll - if agent.PolicyID == "fleet-server-policy" { - continue - } - - if agent.Status != "online" { - continue - } - hasServicePrefix := strings.HasPrefix(agent.LocalMetadata.Host.Name, svcInfo.Agent.Host.NamePrefix) if svcInfo.Agent.Host.NamePrefix != "" && !hasServicePrefix { continue From 9f0f305fd5b8ebdf594231c2692e4924689ee5f3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 13:22:17 +0200 Subject: [PATCH 092/114] Skip reasigning original policy if run agents independenlty --- internal/testrunner/runners/system/runner.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index c51c4c6dc0..9bd8e15720 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -965,23 +965,17 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf origPolicy = serviceStateData.OrigPolicy logger.Debugf("Got orig policy from file: %q - %q", origPolicy.Name, origPolicy.ID) } else { - if r.options.RunIndependentElasticAgent { - // Get default policy to assign agents after testing is finished - defaultPolicy, err := r.options.KibanaClient.GetPolicy(ctx, "elastic-agent-managed-ep") - if err != nil { - return nil, fmt.Errorf("failed to get default policy") - } - origPolicy = *defaultPolicy - } else { - // Store previous agent policy assigned to the agent - origPolicy = kibana.Policy{ - ID: agent.PolicyID, - Revision: agent.PolicyRevision, - } + // Store previous agent policy assigned to the agent + origPolicy = kibana.Policy{ + ID: agent.PolicyID, + Revision: agent.PolicyRevision, } } // Assign policy to agent r.resetAgentPolicyHandler = func(ctx context.Context) error { + if r.options.RunIndependentElasticAgent { + return nil + } logger.Debug("reassigning original policy back to agent...") if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil { return fmt.Errorf("error reassigning original policy to agent: %w", err) From e95d536048a56514a0f8feb211a7777c613e3031 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 13:22:33 +0200 Subject: [PATCH 093/114] Remove test debug messages --- internal/testrunner/runners/system/runner.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 9bd8e15720..f7ebdb72e5 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -945,9 +945,8 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf if err != nil { return nil, fmt.Errorf("can't check enrolled agents: %w", err) } - data, _ := json.Marshal(agents) - logger.Debugf("JSON Agents:\n%s", string(data)) agent := agents[0] + logger.Debugf("Selected enrolled agent %q", agent.ID) r.unenrollAgentHandler = func(ctx context.Context) error { if !r.options.RunIndependentElasticAgent { @@ -1850,17 +1849,14 @@ func filterIndependentAgents(allAgents []kibana.Agent, agentInfo agentdeployer.A // It cannot filtered by "elastic-agent-managed-ep" , since this is the default // policy assigned to the agents when they first enroll if agent.PolicyID == "fleet-server-policy" { - logger.Debugf("filtered agent (policy id) %q", agent.ID) // TODO: remove continue } if agent.Status != "online" { - logger.Debugf("filtered agent (not online) %q", agent.ID) // TODO: remove continue } if agent.PolicyID != agentInfo.PolicyID { - logger.Debugf("filtered agent (not same policy) %q", agent.ID) continue } From e9856a8c7b3074596467a0ecabe247ec3823eb3b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 13:36:36 +0200 Subject: [PATCH 094/114] Fix env. name to enable independent agent --- .buildkite/pipeline.trigger.integration.tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index abdba7203a..5da18fc06d 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -44,7 +44,7 @@ for test in "${CHECK_PACKAGES_TESTS[@]}"; do echo " agents:" echo " provider: \"gcp\"" echo " env:" - echo " ELASTIC_PACKAGE_INDEPENDENT_AGENT: ${independent_agent}" + echo " ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/elastic-stack-dump/check-*/logs/*.log" From ca6e041eaccb2e16872d03602903dff494d807b3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 15:54:15 +0200 Subject: [PATCH 095/114] Reverse if condition to connect network --- internal/servicedeployer/compose.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/servicedeployer/compose.go b/internal/servicedeployer/compose.go index dd75605407..ec3a9da4e4 100644 --- a/internal/servicedeployer/compose.go +++ b/internal/servicedeployer/compose.go @@ -79,7 +79,7 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic variant: d.variant, env: []string{ fmt.Sprintf("%s=%s", serviceLogsDirEnv, svcInfo.Logs.Folder.Local), - // Hostname envioronment varible is required since some packages require to run + // Hostname environment varible is required since some packages require to run // queries to the elastic-agent container (e.g. ti_anomali) fmt.Sprintf("%s=%s", agentHostnameEnv, svcInfo.AgentHostname), }, @@ -142,18 +142,18 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic if d.runTearDown || d.runTestsOnly { logger.Debug("Skipping connect container to network (non setup steps)") } else { - if !d.deployIndependentAgent { - // Connect service network with stack network (for the purpose of metrics collection) - err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network(d.profile)) - if err != nil { - return nil, fmt.Errorf("can't attach service container to the stack network: %w", err) - } - } else { + if d.deployIndependentAgent { // Connect service network with agent network err = docker.ConnectToNetwork(p.ContainerName(serviceName), svcInfo.AgentNetworkName) if err != nil { return nil, fmt.Errorf("can't attach service container to the agent network: %w", err) } + } else { + // Connect service network with stack network (for the purpose of metrics collection) + err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network(d.profile)) + if err != nil { + return nil, fmt.Errorf("can't attach service container to the stack network: %w", err) + } } } From f94566cd6e770e1251f666ce35609ccdfd1ce5a1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 16:24:46 +0200 Subject: [PATCH 096/114] Set just one hostname for customagent (servicedeployer) --- internal/agentdeployer/factory.go | 2 +- internal/servicedeployer/custom_agent.go | 34 +++--------------------- internal/servicedeployer/factory.go | 7 ----- 3 files changed, 5 insertions(+), 38 deletions(-) diff --git a/internal/agentdeployer/factory.go b/internal/agentdeployer/factory.go index 50ce1143c1..1ef47d16fe 100644 --- a/internal/agentdeployer/factory.go +++ b/internal/agentdeployer/factory.go @@ -72,7 +72,7 @@ func Factory(options FactoryOptions) (AgentDeployer, error) { } opts := DockerComposeAgentDeployerOptions{ Profile: options.Profile, - DockerComposeFile: "", + DockerComposeFile: "", // TODO: Allow other docker-compose files to apply overrides Variant: variant, StackVersion: options.StackVersion, PackageName: options.PackageName, diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index 71dcc79f1e..f4ddd747e8 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -36,13 +36,8 @@ type CustomAgentDeployer struct { profile *profile.Profile dockerComposeFile string stackVersion string - variant ServiceVariant - packageName string - dataStream string policyName string - agentRunID string - runTearDown bool runTestsOnly bool } @@ -51,9 +46,6 @@ type CustomAgentDeployerOptions struct { Profile *profile.Profile DockerComposeFile string StackVersion string - Variant ServiceVariant - PackageName string - DataStream string PolicyName string RunTearDown bool @@ -68,10 +60,7 @@ func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDep profile: options.Profile, dockerComposeFile: options.DockerComposeFile, stackVersion: options.StackVersion, - variant: options.Variant, - packageName: options.PackageName, policyName: options.PolicyName, - dataStream: options.DataStream, runTearDown: options.RunTearDown, runTestsOnly: options.RunTestsOnly, }, nil @@ -81,8 +70,6 @@ func NewCustomAgentDeployer(options CustomAgentDeployerOptions) (*CustomAgentDep func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (DeployedService, error) { logger.Debug("setting up service using Docker Compose service deployer") - d.agentRunID = svcInfo.Test.RunID - appConfig, err := install.Configuration() if err != nil { return nil, fmt.Errorf("can't read application configuration: %w", err) @@ -96,8 +83,10 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D // Build service container name // FIXME: Currently, this service deployer starts a new agent on its own and // it cannot use directly the `svcInfo.AgentHostname` value - svcInfo.Hostname = d.agentHostname() - svcInfo.AgentHostname = dockerCustomAgentName // Alias for custom agent + + // Set alias for custom agent + svcInfo.Hostname = dockerCustomAgentName + svcInfo.AgentHostname = dockerCustomAgentName env := append( appConfig.StackImageRefs(d.stackVersion).AsEnv(), @@ -207,21 +196,6 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D return &service, nil } -func (d *CustomAgentDeployer) agentName() string { - name := d.packageName - if d.variant.Name != "" { - name = fmt.Sprintf("%s-%s", name, d.variant.Name) - } - if d.dataStream != "" && d.dataStream != "." { - name = fmt.Sprintf("%s-%s", name, d.dataStream) - } - return name -} - -func (d *CustomAgentDeployer) agentHostname() string { - return fmt.Sprintf("%s-%s-%s", dockerCustomAgentName, d.agentName(), d.agentRunID) -} - // installDockerfile creates the files needed to run the custom elastic agent and returns // the directory with these files. func (d *CustomAgentDeployer) installDockerfile() (string, error) { diff --git a/internal/servicedeployer/factory.go b/internal/servicedeployer/factory.go index e510e6a5ab..a25c54cb4c 100644 --- a/internal/servicedeployer/factory.go +++ b/internal/servicedeployer/factory.go @@ -96,19 +96,12 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { if _, err := os.Stat(customAgentCfgYMLPath); err != nil { return nil, fmt.Errorf("can't find expected file custom-agent.yml: %w", err) } - sv, err := useServiceVariant(devDeployPath, options.Variant) - if err != nil { - return nil, fmt.Errorf("can't use service variant: %w", err) - } policyName := getTokenPolicyName(options.StackVersion, options.PolicyName) opts := CustomAgentDeployerOptions{ Profile: options.Profile, DockerComposeFile: customAgentCfgYMLPath, StackVersion: options.StackVersion, - Variant: sv, - PackageName: options.PackageName, - DataStream: options.DataStream, PolicyName: policyName, RunTearDown: options.RunTearDown, From eb06c8f30e897e054399226e9420b5043b1ad63d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 16:32:19 +0200 Subject: [PATCH 097/114] Use policy name also for 7.* stack (agentdeployer) --- internal/agentdeployer/agent.go | 12 +----------- internal/testrunner/runners/system/runner.go | 1 - 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index cd9d6a4c89..2ac5211bcc 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -10,7 +10,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "time" "github.com/elastic/elastic-package/internal/compose" @@ -109,20 +108,11 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI return nil, fmt.Errorf("can't locate CA certificate: %w", err) } - // Local Elastic stacks have a default Agent Policy created, - // but Cloud or Serverless Projects could not have one - agentPolicyName := d.policyName - if strings.HasPrefix(d.stackVersion, "7.") { - // Local Elastic stacks 7.* have an Agent Policy that is set as default - // No need to set an Agent Policy Name - agentPolicyName = "" - } - env := append( appConfig.StackImageRefs(d.stackVersion).AsEnv(), fmt.Sprintf("%s=%s", serviceLogsDirEnv, agentInfo.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), - fmt.Sprintf("%s=%s", fleetPolicyEnv, agentPolicyName), + fmt.Sprintf("%s=%s", fleetPolicyEnv, d.policyName), fmt.Sprintf("%s=%s", agentHostnameEnv, d.agentHostname()), ) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index f7ebdb72e5..27d00465d3 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -308,7 +308,6 @@ func (r *runner) createAgentInfo(policy *kibana.Policy) (agentdeployer.AgentInfo info.Test.RunID = createTestRunID() folderName := fmt.Sprintf("agent-%s", r.options.TestFolder.Package) - if r.options.TestFolder.DataStream != "" { folderName = fmt.Sprintf("%s-%s", folderName, r.options.TestFolder.DataStream) } From 13af1572e5e5ea2869570233eaba4d9e2c920c34 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 16:35:52 +0200 Subject: [PATCH 098/114] Remove unnecessary check in filterIndependentAgents --- internal/testrunner/runners/system/runner.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 27d00465d3..66b157629a 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1845,12 +1845,6 @@ func filterIndependentAgents(allAgents []kibana.Agent, agentInfo agentdeployer.A continue // For some reason Kibana doesn't always return a valid policy revision (eventually it will be present and valid) } - // It cannot filtered by "elastic-agent-managed-ep" , since this is the default - // policy assigned to the agents when they first enroll - if agent.PolicyID == "fleet-server-policy" { - continue - } - if agent.Status != "online" { continue } From 6b3e976206c3333182c20c2ea27de59343e5ac9b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 16:36:04 +0200 Subject: [PATCH 099/114] Remove unused fields in Agent response --- internal/kibana/agents.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/kibana/agents.go b/internal/kibana/agents.go index de3210d0a0..df1bb627ca 100644 --- a/internal/kibana/agents.go +++ b/internal/kibana/agents.go @@ -35,9 +35,7 @@ type Agent struct { } `json:"agent"` } `json:"elastic"` } `json:"local_metadata"` - Status string `json:"status"` - EnrolledAt time.Time `json:"enrolled_at"` - Tags []string `json:"tags,omitempty"` + Status string `json:"status"` } // String method returns string representation of an agent. From 0424d80d2e350ee1fb7965bc8bd1a66cee6c389a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 16:49:10 +0200 Subject: [PATCH 100/114] Rename policy fields in AgentInfo struct --- internal/agentdeployer/info.go | 12 +++++++----- internal/agentdeployer/kubernetes.go | 4 ++-- internal/testrunner/runners/system/runner.go | 10 +++++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go index d033b41772..8344236a2d 100644 --- a/internal/agentdeployer/info.go +++ b/internal/agentdeployer/info.go @@ -28,8 +28,13 @@ type AgentInfo struct { // required to connect the Service with the agent. NetworkName string - PolicyName string - PolicyID string + // Agent Policy related properties + Policy struct { + // Name is the name of the test Agent Policy created for the given agent + Name string + // ID is the name of the test Agent Policy created for the given agent + ID string + } // Ports is a list of ports that the service listens on, as addressable // from the Agent container. @@ -76,7 +81,4 @@ type AgentInfo struct { // Directory to store agent configuration files ConfigDir string - - // Service token - Token string } diff --git a/internal/agentdeployer/kubernetes.go b/internal/agentdeployer/kubernetes.go index 8fc7f4187b..01aa7052be 100644 --- a/internal/agentdeployer/kubernetes.go +++ b/internal/agentdeployer/kubernetes.go @@ -55,7 +55,7 @@ type kubernetesDeployedAgent struct { } func (s kubernetesDeployedAgent) TearDown(ctx context.Context) error { - elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion, s.agentInfo.PolicyName) + elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion, s.agentInfo.Policy.Name) if err != nil { return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) } @@ -118,7 +118,7 @@ func (ksd KubernetesAgentDeployer) SetUp(ctx context.Context, agentInfo AgentInf if ksd.runTearDown || ksd.runTestsOnly { logger.Debug("Skip install Elastic Agent in cluster") } else { - err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion, agentInfo.PolicyName) + err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion, agentInfo.Policy.Name) if err != nil { return nil, fmt.Errorf("can't install Elastic-Agent in the Kubernetes cluster: %w", err) } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 66b157629a..a48345649b 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -318,8 +318,8 @@ func (r *runner) createAgentInfo(policy *kibana.Policy) (agentdeployer.AgentInfo } info.Logs.Folder.Local = dirPath - info.PolicyName = policy.Name - info.PolicyID = policy.ID + info.Policy.Name = policy.Name + info.Policy.ID = policy.ID return info, nil } @@ -1125,7 +1125,7 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp // In case of custom agent, update serviceOptions to include test policy too if r.options.RunIndependentElasticAgent { - serviceOptions.PolicyName = agentInfo.PolicyName + serviceOptions.PolicyName = agentInfo.Policy.Name } if config.Service != "" { @@ -1157,7 +1157,7 @@ func (r *runner) setupAgent(ctx context.Context, variant string, agentInfo agent if !r.options.RunIndependentElasticAgent { return nil, agentInfo, nil } - agentOptions := r.createAgentOptions(variant, agentInfo.PolicyName) + agentOptions := r.createAgentOptions(variant, agentInfo.Policy.Name) agentDeployer, err := agentdeployer.Factory(agentOptions) if err != nil { return nil, agentInfo, fmt.Errorf("could not create agent runner: %w", err) @@ -1849,7 +1849,7 @@ func filterIndependentAgents(allAgents []kibana.Agent, agentInfo agentdeployer.A continue } - if agent.PolicyID != agentInfo.PolicyID { + if agent.PolicyID != agentInfo.Policy.ID { continue } From 6ce4ed25351bcb750ddcdabc0c5481fff895a200 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 17:39:45 +0200 Subject: [PATCH 101/114] Move code to setupAgent and setupService --- internal/testrunner/runners/system/runner.go | 53 +++++++++++--------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index a48345649b..cde45ce66b 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -787,41 +787,21 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf return nil } - // Setup agent - logger.Debug("setting up agent...") - agentInfo, err := r.createAgentInfo(policy) - if err != nil { - return nil, err - } - enrollingTime := time.Now() if r.options.RunTearDown || r.options.RunTestsOnly { enrollingTime = serviceStateData.EnrollingAgentTime - - agentInfo.Test.RunID = serviceStateData.AgentRunID - agentInfo.Hostname = serviceStateData.AgentHostname - - svcInfo.Test.RunID = serviceStateData.ServiceRunID - svcInfo.AgentHostname = serviceStateData.ServiceAgentHostname - svcInfo.Hostname = serviceStateData.ServiceAgentHostname - // By default using agent running in the Elastic stack - svcInfo.AgentNetworkName = stack.Network(r.options.Profile) } - agentDeployed, agentInfo, err := r.setupAgent(ctx, serviceOptions.Variant, agentInfo) + agentDeployed, agentInfo, err := r.setupAgent(ctx, serviceOptions.Variant, serviceStateData, policy) if err != nil { return nil, err } - if agentDeployed != nil { - agentInfo = agentDeployed.Info() - svcInfo.AgentNetworkName = agentInfo.NetworkName - } scenario.enrollingTime = enrollingTime scenario.agent = agentDeployed // Setup service. - service, svcInfo, err := r.setupService(ctx, config, serviceOptions, svcInfo, agentInfo, agentDeployed) + service, svcInfo, err := r.setupService(ctx, config, serviceOptions, svcInfo, agentInfo, agentDeployed, serviceStateData) if err != nil { return nil, err } @@ -1111,13 +1091,25 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf return &scenario, nil } -func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOptions servicedeployer.FactoryOptions, svcInfo servicedeployer.ServiceInfo, agentInfo agentdeployer.AgentInfo, agentDeployed agentdeployer.DeployedAgent) (servicedeployer.DeployedService, servicedeployer.ServiceInfo, error) { +func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOptions servicedeployer.FactoryOptions, svcInfo servicedeployer.ServiceInfo, agentInfo agentdeployer.AgentInfo, agentDeployed agentdeployer.DeployedAgent, state ServiceState) (servicedeployer.DeployedService, servicedeployer.ServiceInfo, error) { logger.Debug("setting up service...") + if r.options.RunTearDown || r.options.RunTestsOnly { + svcInfo.Test.RunID = state.ServiceRunID + svcInfo.AgentHostname = state.ServiceAgentHostname + svcInfo.Hostname = state.ServiceAgentHostname + } + // Elastic Agent from stack and Elastic Agents started independently // will have a network alias "elastic-agent" that services can use // Docker custom agents would have another alias "docker-custom-agent" svcInfo.AgentHostname = "elastic-agent" + // By default using agent running in the Elastic stack + svcInfo.AgentNetworkName = stack.Network(r.options.Profile) + if agentDeployed != nil { + svcInfo.AgentNetworkName = agentInfo.NetworkName + } + // Set the right folder for logs execpt for custom agents that are still deployed using "servicedeployer" if r.options.RunIndependentElasticAgent && agentDeployed != nil { svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local @@ -1153,10 +1145,20 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp return service, service.Info(), nil } -func (r *runner) setupAgent(ctx context.Context, variant string, agentInfo agentdeployer.AgentInfo) (agentdeployer.DeployedAgent, agentdeployer.AgentInfo, error) { +func (r *runner) setupAgent(ctx context.Context, variant string, state ServiceState, policy *kibana.Policy) (agentdeployer.DeployedAgent, agentdeployer.AgentInfo, error) { if !r.options.RunIndependentElasticAgent { - return nil, agentInfo, nil + return nil, agentdeployer.AgentInfo{}, nil } + logger.Debug("setting up agent...") + agentInfo, err := r.createAgentInfo(policy) + if err != nil { + return nil, agentdeployer.AgentInfo{}, err + } + if r.options.RunTearDown || r.options.RunTestsOnly { + agentInfo.Test.RunID = state.AgentRunID + agentInfo.Hostname = state.AgentHostname + } + agentOptions := r.createAgentOptions(variant, agentInfo.Policy.Name) agentDeployer, err := agentdeployer.Factory(agentOptions) if err != nil { @@ -1165,6 +1167,7 @@ func (r *runner) setupAgent(ctx context.Context, variant string, agentInfo agent if agentDeployer == nil { return nil, agentInfo, nil } + agentDeployed, err := agentDeployer.SetUp(ctx, agentInfo) if err != nil { return nil, agentInfo, fmt.Errorf("could not setup agent: %w", err) From e697f342c3017a19ad9113fcb196b6b546abffbb Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 18:14:23 +0200 Subject: [PATCH 102/114] Remove AGENT_HOSTNAME from servicedeployer --- .../_static/docker-custom-agent-base.yml | 2 +- .../_static/terraform_deployer.yml | 1 - internal/servicedeployer/compose.go | 3 -- internal/servicedeployer/custom_agent.go | 2 - internal/servicedeployer/info.go | 6 +-- internal/servicedeployer/terraform_env.go | 1 - internal/testrunner/runners/system/runner.go | 45 ++++++++----------- 7 files changed, 20 insertions(+), 40 deletions(-) diff --git a/internal/servicedeployer/_static/docker-custom-agent-base.yml b/internal/servicedeployer/_static/docker-custom-agent-base.yml index 1077bf38c6..84a1fcefac 100644 --- a/internal/servicedeployer/_static/docker-custom-agent-base.yml +++ b/internal/servicedeployer/_static/docker-custom-agent-base.yml @@ -6,7 +6,7 @@ services: test: "elastic-agent status" retries: 180 interval: 1s - hostname: ${AGENT_HOSTNAME} + hostname: docker-custom-agent environment: - FLEET_ENROLL=1 - FLEET_URL=https://fleet-server:8220 diff --git a/internal/servicedeployer/_static/terraform_deployer.yml b/internal/servicedeployer/_static/terraform_deployer.yml index 6c22f7644c..794446d1ba 100644 --- a/internal/servicedeployer/_static/terraform_deployer.yml +++ b/internal/servicedeployer/_static/terraform_deployer.yml @@ -11,7 +11,6 @@ services: - TF_VAR_BUILD_ID=${BUILD_ID:-unknown} - TF_VAR_ENVIRONMENT=${ENVIRONMENT:-unknown} - TF_VAR_REPO=${REPO:-unknown} - - AGENT_HOSTNAME=${AGENT_HOSTNAME:-elastic-agent} volumes: - ${TF_DIR}:/stage - ${TF_OUTPUT_DIR}:/output diff --git a/internal/servicedeployer/compose.go b/internal/servicedeployer/compose.go index ec3a9da4e4..1e11b0fcee 100644 --- a/internal/servicedeployer/compose.go +++ b/internal/servicedeployer/compose.go @@ -79,9 +79,6 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic variant: d.variant, env: []string{ fmt.Sprintf("%s=%s", serviceLogsDirEnv, svcInfo.Logs.Folder.Local), - // Hostname environment varible is required since some packages require to run - // queries to the elastic-agent container (e.g. ti_anomali) - fmt.Sprintf("%s=%s", agentHostnameEnv, svcInfo.AgentHostname), }, } diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go index f4ddd747e8..f6cc5e2594 100644 --- a/internal/servicedeployer/custom_agent.go +++ b/internal/servicedeployer/custom_agent.go @@ -86,13 +86,11 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D // Set alias for custom agent svcInfo.Hostname = dockerCustomAgentName - svcInfo.AgentHostname = dockerCustomAgentName env := append( appConfig.StackImageRefs(d.stackVersion).AsEnv(), fmt.Sprintf("%s=%s", serviceLogsDirEnv, svcInfo.Logs.Folder.Local), fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), - fmt.Sprintf("%s=%s", agentHostnameEnv, svcInfo.AgentHostname), fmt.Sprintf("%s=%s", fleetPolicyEnv, d.policyName), ) diff --git a/internal/servicedeployer/info.go b/internal/servicedeployer/info.go index 144877b169..252e743798 100644 --- a/internal/servicedeployer/info.go +++ b/internal/servicedeployer/info.go @@ -8,7 +8,6 @@ const ( localCACertEnv = "LOCAL_CA_CERT" serviceLogsDirEnv = "SERVICE_LOGS_DIR" testRunIDEnv = "TEST_RUN_ID" - agentHostnameEnv = "AGENT_HOSTNAME" elasticAgentTagsEnv = "ELASTIC_AGENT_TAGS" fleetPolicyEnv = "FLEET_TOKEN_POLICY_NAME" @@ -26,10 +25,7 @@ type ServiceInfo struct { // the Agent container. Hostname string - // AgentHostname is the host name of the agent, as addressable from - // the Service container . - AgentHostname string - + // AgentNetworkName is the network name where the agent is running. AgentNetworkName string // Ports is a list of ports that the service listens on, as addressable diff --git a/internal/servicedeployer/terraform_env.go b/internal/servicedeployer/terraform_env.go index b99c8705a4..428aefe352 100644 --- a/internal/servicedeployer/terraform_env.go +++ b/internal/servicedeployer/terraform_env.go @@ -26,7 +26,6 @@ func (tsd TerraformServiceDeployer) buildTerraformExecutorEnvironment(info Servi vars[tfTestRunID] = info.Test.RunID vars[tfDir] = tsd.definitionsDir vars[tfOutputDir] = info.OutputDir - vars[agentHostnameEnv] = info.AgentHostname var pairs []string for k, v := range vars { diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index cde45ce66b..833fb2bfd8 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1095,15 +1095,8 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp logger.Debug("setting up service...") if r.options.RunTearDown || r.options.RunTestsOnly { svcInfo.Test.RunID = state.ServiceRunID - svcInfo.AgentHostname = state.ServiceAgentHostname - svcInfo.Hostname = state.ServiceAgentHostname } - // Elastic Agent from stack and Elastic Agents started independently - // will have a network alias "elastic-agent" that services can use - // Docker custom agents would have another alias "docker-custom-agent" - svcInfo.AgentHostname = "elastic-agent" - // By default using agent running in the Elastic stack svcInfo.AgentNetworkName = stack.Network(r.options.Profile) if agentDeployed != nil { @@ -1218,16 +1211,15 @@ func (r *runner) readServiceStateData() (ServiceState, error) { } type ServiceState struct { - OrigPolicy kibana.Policy `json:"orig_policy"` - CurrentPolicy kibana.Policy `json:"current_policy"` - Agent kibana.Agent `json:"agent"` - ConfigFilePath string `json:"config_file_path"` - VariantName string `json:"variant_name"` - EnrollingAgentTime time.Time `json:"enrolling_agent_time"` - ServiceRunID string `json:"service_info_run_id"` - AgentRunID string `json:"agent_info_run_id"` - AgentHostname string `json:"agent_hostname"` - ServiceAgentHostname string `json:"service_agent_hostname"` + OrigPolicy kibana.Policy `json:"orig_policy"` + CurrentPolicy kibana.Policy `json:"current_policy"` + Agent kibana.Agent `json:"agent"` + ConfigFilePath string `json:"config_file_path"` + VariantName string `json:"variant_name"` + EnrollingAgentTime time.Time `json:"enrolling_agent_time"` + ServiceRunID string `json:"service_info_run_id"` + AgentRunID string `json:"agent_info_run_id"` + AgentHostname string `json:"agent_hostname"` } type scenarioStateOpts struct { @@ -1242,16 +1234,15 @@ type scenarioStateOpts struct { func (r *runner) writeScenarioState(opts scenarioStateOpts) error { data := ServiceState{ - OrigPolicy: *opts.origPolicy, - CurrentPolicy: *opts.currentPolicy, - Agent: opts.agent, - ConfigFilePath: opts.config.Path, - VariantName: opts.config.ServiceVariantName, - EnrollingAgentTime: opts.enrollingTime, - ServiceRunID: opts.svcInfo.Test.RunID, - AgentRunID: opts.agentInfo.Test.RunID, - AgentHostname: opts.agentInfo.Hostname, - ServiceAgentHostname: opts.svcInfo.AgentHostname, + OrigPolicy: *opts.origPolicy, + CurrentPolicy: *opts.currentPolicy, + Agent: opts.agent, + ConfigFilePath: opts.config.Path, + VariantName: opts.config.ServiceVariantName, + EnrollingAgentTime: opts.enrollingTime, + ServiceRunID: opts.svcInfo.Test.RunID, + AgentRunID: opts.agentInfo.Test.RunID, + AgentHostname: opts.agentInfo.Hostname, } dataBytes, err := json.Marshal(data) if err != nil { From 595eccd775740115b62806305e48f9c4d05d5e7e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 18:22:23 +0200 Subject: [PATCH 103/114] Remove unused field --- internal/testrunner/runners/system/runner.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 833fb2bfd8..ddf960101f 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1149,7 +1149,6 @@ func (r *runner) setupAgent(ctx context.Context, variant string, state ServiceSt } if r.options.RunTearDown || r.options.RunTestsOnly { agentInfo.Test.RunID = state.AgentRunID - agentInfo.Hostname = state.AgentHostname } agentOptions := r.createAgentOptions(variant, agentInfo.Policy.Name) @@ -1219,7 +1218,6 @@ type ServiceState struct { EnrollingAgentTime time.Time `json:"enrolling_agent_time"` ServiceRunID string `json:"service_info_run_id"` AgentRunID string `json:"agent_info_run_id"` - AgentHostname string `json:"agent_hostname"` } type scenarioStateOpts struct { @@ -1242,7 +1240,6 @@ func (r *runner) writeScenarioState(opts scenarioStateOpts) error { EnrollingAgentTime: opts.enrollingTime, ServiceRunID: opts.svcInfo.Test.RunID, AgentRunID: opts.agentInfo.Test.RunID, - AgentHostname: opts.agentInfo.Hostname, } dataBytes, err := json.Marshal(data) if err != nil { From e35732e17a3790e89149b3eb24de1d500c90cee3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 18:46:59 +0200 Subject: [PATCH 104/114] Ensure output dir is just created once - services --- internal/testrunner/runners/system/runner.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index ddf960101f..adac2bc706 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -331,11 +331,15 @@ func (r *runner) createServiceInfo() (servicedeployer.ServiceInfo, error) { svcInfo.Logs.Folder.Agent = ServiceLogsAgentDir svcInfo.Test.RunID = createTestRunID() - outputDir, err := servicedeployer.CreateOutputDir(r.locationManager, svcInfo.Test.RunID) - if err != nil { - return servicedeployer.ServiceInfo{}, fmt.Errorf("could not create output dir for terraform deployer %w", err) + if r.options.RunTearDown || r.options.RunTestsOnly { + logger.Debug("Skip creating output directory") + } else { + outputDir, err := servicedeployer.CreateOutputDir(r.locationManager, svcInfo.Test.RunID) + if err != nil { + return servicedeployer.ServiceInfo{}, fmt.Errorf("could not create output dir for terraform deployer %w", err) + } + svcInfo.OutputDir = outputDir } - svcInfo.OutputDir = outputDir return svcInfo, nil } @@ -1095,6 +1099,7 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp logger.Debug("setting up service...") if r.options.RunTearDown || r.options.RunTestsOnly { svcInfo.Test.RunID = state.ServiceRunID + svcInfo.OutputDir = state.ServiceOutputDir } // By default using agent running in the Elastic stack @@ -1218,6 +1223,7 @@ type ServiceState struct { EnrollingAgentTime time.Time `json:"enrolling_agent_time"` ServiceRunID string `json:"service_info_run_id"` AgentRunID string `json:"agent_info_run_id"` + ServiceOutputDir string `json:"service_output_dir"` } type scenarioStateOpts struct { @@ -1240,6 +1246,7 @@ func (r *runner) writeScenarioState(opts scenarioStateOpts) error { EnrollingAgentTime: opts.enrollingTime, ServiceRunID: opts.svcInfo.Test.RunID, AgentRunID: opts.agentInfo.Test.RunID, + ServiceOutputDir: opts.svcInfo.OutputDir, } dataBytes, err := json.Marshal(data) if err != nil { From fbb8f4c6acd39c96079f52f6a7163c381510444a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 18:47:15 +0200 Subject: [PATCH 105/114] Remove unnecessary field in agentInfo --- internal/agentdeployer/agent.go | 4 ---- internal/agentdeployer/info.go | 3 --- 2 files changed, 7 deletions(-) diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 2ac5211bcc..dc05e9b074 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -315,10 +315,6 @@ func (s *dockerComposeDeployedAgent) TearDown(ctx context.Context) error { if err != nil { logger.Errorf("could not remove the agent logs (path: %s)", s.agentInfo.Logs.Folder.Local) } - // Remove the outputs generated by the service container - if err = os.RemoveAll(s.agentInfo.OutputDir); err != nil { - logger.Errorf("could not remove the temporary output files %w", err) - } // Remove the configuration dir (e.g. compose scenario files) if err = os.RemoveAll(s.agentInfo.ConfigDir); err != nil { diff --git a/internal/agentdeployer/info.go b/internal/agentdeployer/info.go index 8344236a2d..4529286338 100644 --- a/internal/agentdeployer/info.go +++ b/internal/agentdeployer/info.go @@ -76,9 +76,6 @@ type AgentInfo struct { // CustomProperties store additional data used to boot up the service, e.g. AWS credentials. CustomProperties map[string]interface{} - // Directory to store any outputs generated - OutputDir string - // Directory to store agent configuration files ConfigDir string } From f4b65681f205f7d2e1f802fddeb030677e000dfd Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 19:00:39 +0200 Subject: [PATCH 106/114] Remove unused field --- internal/servicedeployer/terraform.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/servicedeployer/terraform.go b/internal/servicedeployer/terraform.go index 575cd6c129..386f9c0bc9 100644 --- a/internal/servicedeployer/terraform.go +++ b/internal/servicedeployer/terraform.go @@ -43,12 +43,10 @@ var terraformDeployerDockerfileContent string // TerraformServiceDeployer is responsible for deploying infrastructure described with Terraform definitions. type TerraformServiceDeployer struct { definitionsDir string - deployAgent bool } type TerraformServiceDeployerOptions struct { DefinitionsDir string - DeployAgent bool } // addTerraformOutputs method reads the terraform outputs generated in the json format and @@ -92,7 +90,6 @@ func addTerraformOutputs(svcInfo ServiceInfo) error { func NewTerraformServiceDeployer(opts TerraformServiceDeployerOptions) (*TerraformServiceDeployer, error) { return &TerraformServiceDeployer{ definitionsDir: opts.DefinitionsDir, - deployAgent: opts.DeployAgent, }, nil } From 7bf8e90b2a3727d8e8953becfca1d72ec6508df9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 19:08:38 +0200 Subject: [PATCH 107/114] Avoid creating a temporal variable for svcInfo --- internal/servicedeployer/terraform.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/servicedeployer/terraform.go b/internal/servicedeployer/terraform.go index 386f9c0bc9..c35f7673ab 100644 --- a/internal/servicedeployer/terraform.go +++ b/internal/servicedeployer/terraform.go @@ -117,7 +117,6 @@ func (tsd TerraformServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceIn env: tfEnvironment, shutdownTimeout: 300 * time.Second, } - outCtxt := svcInfo p, err := compose.NewProject(service.project, service.ymlPaths...) if err != nil { @@ -125,7 +124,7 @@ func (tsd TerraformServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceIn } // Clean service logs - err = files.RemoveContent(outCtxt.Logs.Folder.Local) + err = files.RemoveContent(svcInfo.Logs.Folder.Local) if err != nil { return nil, fmt.Errorf("removing service logs failed: %w", err) } @@ -138,7 +137,7 @@ func (tsd TerraformServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceIn if err != nil { return nil, fmt.Errorf("could not get Docker Compose configuration for service: %w", err) } - outCtxt.CustomProperties, err = buildTerraformAliases(serviceComposeConfig) + svcInfo.CustomProperties, err = buildTerraformAliases(serviceComposeConfig) if err != nil { return nil, fmt.Errorf("can't build Terraform aliases: %w", err) } @@ -157,18 +156,18 @@ func (tsd TerraformServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceIn if err != nil { processServiceContainerLogs(ctx, p, compose.CommandOptions{ Env: opts.Env, - }, outCtxt.Name) + }, svcInfo.Name) //lint:ignore ST1005 error starting with product name can be capitalized return nil, fmt.Errorf("Terraform deployer is unhealthy: %w", err) } - outCtxt.Agent.Host.NamePrefix = "docker-fleet-agent" + svcInfo.Agent.Host.NamePrefix = "docker-fleet-agent" - err = addTerraformOutputs(outCtxt) + err = addTerraformOutputs(svcInfo) if err != nil { return nil, fmt.Errorf("could not handle terraform output: %w", err) } - service.svcInfo = outCtxt + service.svcInfo = svcInfo return &service, nil } From b63b4254c7ca5c9f7df414262820131766000549 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 19:54:18 +0200 Subject: [PATCH 108/114] Use reference to add terraform outputs --- internal/servicedeployer/terraform.go | 4 +-- internal/servicedeployer/terraform_test.go | 30 ++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/internal/servicedeployer/terraform.go b/internal/servicedeployer/terraform.go index c35f7673ab..ef309e0d3e 100644 --- a/internal/servicedeployer/terraform.go +++ b/internal/servicedeployer/terraform.go @@ -52,7 +52,7 @@ type TerraformServiceDeployerOptions struct { // addTerraformOutputs method reads the terraform outputs generated in the json format and // adds them to the custom properties of ServiceInfo and can be used in the handlebars template // like `{{TF_OUTPUT_queue_url}}` where `queue_url` is the output configured -func addTerraformOutputs(svcInfo ServiceInfo) error { +func addTerraformOutputs(svcInfo *ServiceInfo) error { // Read the `output.json` file where terraform outputs are generated outputFile := filepath.Join(svcInfo.OutputDir, terraformOutputJSONFile) content, err := os.ReadFile(outputFile) @@ -163,7 +163,7 @@ func (tsd TerraformServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceIn svcInfo.Agent.Host.NamePrefix = "docker-fleet-agent" - err = addTerraformOutputs(svcInfo) + err = addTerraformOutputs(&svcInfo) if err != nil { return nil, fmt.Errorf("could not handle terraform output: %w", err) } diff --git a/internal/servicedeployer/terraform_test.go b/internal/servicedeployer/terraform_test.go index 3b06c8a977..d768856cd3 100644 --- a/internal/servicedeployer/terraform_test.go +++ b/internal/servicedeployer/terraform_test.go @@ -64,6 +64,29 @@ func TestAddTerraformOutputs(t *testing.T) { "TF_OUTPUT_queue_url": "https://sqs.us-east-1.amazonaws.com/1234654/elastic-package-aws-logs-queue-someId", }, }, + { + testName: "add_single_value_output", + runId: "99999", + svcInfo: ServiceInfo{ + Test: struct{ RunID string }{"99999"}, + CustomProperties: map[string]interface{}{ + "TF_foo": "bar", + }, + }, + content: []byte( + `{ + "queue_url": { + "sensitive": false, + "type": "string", + "value": "https://sqs.us-east-1.amazonaws.com/1234654/elastic-package-aws-logs-queue-someId" + } + }`, + ), + expectedProps: map[string]interface{}{ + "TF_OUTPUT_queue_url": "https://sqs.us-east-1.amazonaws.com/1234654/elastic-package-aws-logs-queue-someId", + "TF_foo": "bar", + }, + }, { testName: "multiple_value_output", runId: "23465", @@ -138,7 +161,10 @@ func TestAddTerraformOutputs(t *testing.T) { for _, tc := range testCases { t.Run(tc.testName, func(t *testing.T) { - tc.svcInfo.CustomProperties = make(map[string]interface{}) + if tc.svcInfo.CustomProperties == nil { + tc.svcInfo.CustomProperties = make(map[string]interface{}) + } + tc.svcInfo.OutputDir = t.TempDir() if err := os.WriteFile(tc.svcInfo.OutputDir+"/tfOutputValues.json", tc.content, 0777); err != nil { @@ -146,7 +172,7 @@ func TestAddTerraformOutputs(t *testing.T) { } // Test that the terraform output values are generated correctly - err := addTerraformOutputs(tc.svcInfo) + err := addTerraformOutputs(&tc.svcInfo) if tc.expectedError { require.Error(t, err) return From 148668a8ce9f0e854514e5790e7bc4b17fb82ed7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Apr 2024 20:10:02 +0200 Subject: [PATCH 109/114] Remove map initialization in test --- internal/servicedeployer/terraform_test.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/internal/servicedeployer/terraform_test.go b/internal/servicedeployer/terraform_test.go index d768856cd3..a1c5b2f291 100644 --- a/internal/servicedeployer/terraform_test.go +++ b/internal/servicedeployer/terraform_test.go @@ -31,7 +31,7 @@ func TestAddTerraformOutputs(t *testing.T) { content: []byte( ``, ), - expectedProps: map[string]interface{}{}, + expectedProps: nil, expectedError: true, }, { @@ -43,7 +43,7 @@ func TestAddTerraformOutputs(t *testing.T) { content: []byte( `{}`, ), - expectedProps: map[string]interface{}{}, + expectedProps: nil, }, { testName: "single_value_output", @@ -161,10 +161,6 @@ func TestAddTerraformOutputs(t *testing.T) { for _, tc := range testCases { t.Run(tc.testName, func(t *testing.T) { - if tc.svcInfo.CustomProperties == nil { - tc.svcInfo.CustomProperties = make(map[string]interface{}) - } - tc.svcInfo.OutputDir = t.TempDir() if err := os.WriteFile(tc.svcInfo.OutputDir+"/tfOutputValues.json", tc.content, 0777); err != nil { From 8d5fce3cc2dbddd313a8a1e96804b83a065aa319 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 Apr 2024 12:16:28 +0200 Subject: [PATCH 110/114] Replace some switch statements by ifs --- internal/testrunner/runners/system/runner.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index adac2bc706..0997a5c8f6 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -804,7 +804,6 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf scenario.enrollingTime = enrollingTime scenario.agent = agentDeployed - // Setup service. service, svcInfo, err := r.setupService(ctx, config, serviceOptions, svcInfo, agentInfo, agentDeployed, serviceStateData) if err != nil { return nil, err @@ -967,12 +966,10 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf origAgent := agent origLogLevel := "" - switch { - case r.options.RunTearDown: - logger.Debug("Skip assiging log level debut to agent") - logger.Debugf("Got agent from file: %q - %q", serviceStateData.Agent.ID, serviceStateData.Agent.LocalMetadata.Elastic.Agent.LogLevel) + if r.options.RunTearDown { + logger.Debug("Skip assiging log level debug to agent") origLogLevel = serviceStateData.Agent.LocalMetadata.Elastic.Agent.LogLevel - default: + } else { logger.Debug("Set Debug log level to agent") origLogLevel = agent.LocalMetadata.Elastic.Agent.LogLevel err = r.options.KibanaClient.SetAgentLogLevel(ctx, agent.ID, "debug") @@ -989,10 +986,9 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf return nil } - switch { - case r.options.RunTearDown || r.options.RunTestsOnly: + if r.options.RunTearDown || r.options.RunTestsOnly { logger.Debug("Skip assiging package data stream to agent") - default: + } else { policyWithDataStream, err := r.options.KibanaClient.GetPolicy(ctx, policy.ID) if err != nil { return nil, fmt.Errorf("could not read the policy with data stream: %w", err) @@ -1113,7 +1109,7 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local } - // In case of custom agent, update serviceOptions to include test policy too + // In case of custom agent (servicedeployer), update serviceOptions to include test policy too if r.options.RunIndependentElasticAgent { serviceOptions.PolicyName = agentInfo.Policy.Name } From 93fb9761479576a8b81422ce5b7c52726e69cb15 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 Apr 2024 12:37:14 +0200 Subject: [PATCH 111/114] Rephrased comment --- internal/testrunner/runners/system/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 0997a5c8f6..5d8021b4b0 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1109,7 +1109,7 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local } - // In case of custom agent (servicedeployer), update serviceOptions to include test policy too + // In case of custom agent (servicedeployer) enabling independent agents, update serviceOptions to include test policy too if r.options.RunIndependentElasticAgent { serviceOptions.PolicyName = agentInfo.Policy.Name } From 229b23f49f98aae3646ae77870d7167dcce8124e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 Apr 2024 12:58:10 +0200 Subject: [PATCH 112/114] Return default Agent policy if it is empty --- internal/servicedeployer/kubernetes.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index fc48f6eddf..537836e098 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -248,6 +248,9 @@ func readCACertBase64(profile *profile.Profile) (string, error) { // getTokenPolicyName function returns the policy name for the 8.x Elastic stack. The agent's policy // is predefined in the Kibana configuration file. The logic is not present in older stacks. func getTokenPolicyName(stackVersion, policyName string) string { + if policyName == "" { + policyName = defaulFleetTokenPolicyName + } if strings.HasPrefix(stackVersion, "8.") { return policyName } From bda3a2c32b5ce407d9ee44a215efa978dc5fc2ee Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 9 Apr 2024 13:38:53 +0200 Subject: [PATCH 113/114] Rename function to remove agent --- internal/kibana/agents.go | 4 ++-- internal/testrunner/runners/system/runner.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/kibana/agents.go b/internal/kibana/agents.go index df1bb627ca..5014a852fa 100644 --- a/internal/kibana/agents.go +++ b/internal/kibana/agents.go @@ -90,8 +90,8 @@ func (c *Client) AssignPolicyToAgent(ctx context.Context, a Agent, p Policy) err return nil } -// UnenrollAgent unenrolls the given agent -func (c *Client) UnenrollAgent(ctx context.Context, a Agent) error { +// RemoveAgent unenrolls the given agent +func (c *Client) RemoveAgent(ctx context.Context, a Agent) error { reqBody := `{ "revoke": true, "force": true }` path := fmt.Sprintf("%s/agents/%s/unenroll", FleetAPI, a.ID) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 5d8021b4b0..9f4b085622 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -935,7 +935,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf return nil } logger.Debug("unenrolling agent...") - err := r.options.KibanaClient.UnenrollAgent(ctx, agent) + err := r.options.KibanaClient.RemoveAgent(ctx, agent) if err != nil { return fmt.Errorf("failed to unenroll agent %q: %w", agent.ID, err) } From ead4d7ce1add5eac44384cbb2d3642055e9cce95 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 9 Apr 2024 16:14:10 +0200 Subject: [PATCH 114/114] Add technical preview message --- internal/testrunner/runners/system/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 9f4b085622..5c08b13ce9 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -1143,7 +1143,7 @@ func (r *runner) setupAgent(ctx context.Context, variant string, state ServiceSt if !r.options.RunIndependentElasticAgent { return nil, agentdeployer.AgentInfo{}, nil } - logger.Debug("setting up agent...") + logger.Warn("setting up agent (technical preview)...") agentInfo, err := r.createAgentInfo(policy) if err != nil { return nil, agentdeployer.AgentInfo{}, err