From 5601bced18da342b7f01a7db560f26c93e589f76 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 3 Aug 2021 12:10:47 +0300 Subject: [PATCH 01/13] Revert "Revert: Get elastic-agent-managed-daemonset.yaml from upstream 7.x instead of using a local static file (#459)" This reverts commit 65313621f9bd6b947f27b39b7bde3bdfbdbf6d22. --- internal/configuration/locations/locations.go | 8 +-- internal/install/install.go | 26 --------- internal/kubectl/kubectl.go | 18 ++++++ internal/kubectl/kubectl_apply.go | 16 ++++++ .../system/servicedeployer/kubernetes.go | 57 +++++++++++++++++-- test/packages/kubernetes/_dev/build/build.yml | 3 + .../kubernetes/data_stream/pod/fields/ecs.yml | 4 ++ 7 files changed, 95 insertions(+), 37 deletions(-) create mode 100644 test/packages/kubernetes/_dev/build/build.yml diff --git a/internal/configuration/locations/locations.go b/internal/configuration/locations/locations.go index a6c7930f12..0fdcf35891 100644 --- a/internal/configuration/locations/locations.go +++ b/internal/configuration/locations/locations.go @@ -23,8 +23,7 @@ const ( fieldsCachedDir = "cache/fields" - kubernetesDeployerElasticAgentYmlFile = "elastic-agent.yml" - terraformDeployerYmlFile = "terraform-deployer.yml" + terraformDeployerYmlFile = "terraform-deployer.yml" ) var ( @@ -84,11 +83,6 @@ func (loc LocationManager) KubernetesDeployerDir() string { return filepath.Join(loc.stackPath, kubernetesDeployerDir) } -// KubernetesDeployerAgentYml returns the Kubernetes Deployer Elastic Agent yml -func (loc LocationManager) KubernetesDeployerAgentYml() string { - return filepath.Join(loc.stackPath, kubernetesDeployerDir, kubernetesDeployerElasticAgentYmlFile) -} - // TerraformDeployerDir returns the Terraform Directory func (loc LocationManager) TerraformDeployerDir() string { return filepath.Join(loc.stackPath, terraformDeployerDir) diff --git a/internal/install/install.go b/internal/install/install.go index ff28697520..4b3e038b0a 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "time" "github.com/pkg/errors" @@ -60,11 +59,6 @@ func EnsureInstalled() error { return errors.Wrap(err, "writing stack resources failed") } - err = writeKubernetesDeployerResources(elasticPackagePath) - if err != nil { - return errors.Wrap(err, "writing Kubernetes deployer resources failed") - } - err = writeTerraformDeployerResources(elasticPackagePath) if err != nil { return errors.Wrap(err, "writing Terraform deployer resources failed") @@ -172,26 +166,6 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error { } -func writeKubernetesDeployerResources(elasticPackagePath *locations.LocationManager) error { - err := os.MkdirAll(elasticPackagePath.KubernetesDeployerDir(), 0755) - if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.KubernetesDeployerDir()) - } - - appConfig, err := Configuration() - if err != nil { - return errors.Wrap(err, "can't read application configuration") - } - - err = writeStaticResource(err, elasticPackagePath.KubernetesDeployerAgentYml(), - strings.ReplaceAll(kubernetesDeployerElasticAgentYml, "{{ ELASTIC_AGENT_IMAGE_REF }}", - appConfig.DefaultStackImageRefs().ElasticAgent)) - if err != nil { - return errors.Wrap(err, "writing static resource failed") - } - return nil -} - func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManager) error { terraformDeployer := elasticPackagePath.TerraformDeployerDir() err := os.MkdirAll(terraformDeployer, 0755) diff --git a/internal/kubectl/kubectl.go b/internal/kubectl/kubectl.go index 40a3ea2722..ac40d07d78 100644 --- a/internal/kubectl/kubectl.go +++ b/internal/kubectl/kubectl.go @@ -49,3 +49,21 @@ func modifyKubernetesResources(action string, definitionPaths ...string) ([]byte } return output, nil } + +// applyKubernetesResourcesStdin applies a Kubernetes manifest provided as stdin. +// It returns the resources created as output and an error +func applyKubernetesResourcesStdin(input []byte) ([]byte, error) { + // create kubectl apply command + kubectlCmd := exec.Command("kubectl", "apply", "-f", "-", "-o", "yaml") + //Stdin of kubectl command is the manifest provided + kubectlCmd.Stdin = bytes.NewReader(input) + errOutput := new(bytes.Buffer) + kubectlCmd.Stderr = errOutput + + logger.Debugf("run command: %s", kubectlCmd) + output, err := kubectlCmd.Output() + if err != nil { + return nil, errors.Wrapf(err, "kubectl apply failed (stderr=%q)", errOutput.String()) + } + return output, nil +} diff --git a/internal/kubectl/kubectl_apply.go b/internal/kubectl/kubectl_apply.go index 1bb2c7be8b..c49ef1ff12 100644 --- a/internal/kubectl/kubectl_apply.go +++ b/internal/kubectl/kubectl_apply.go @@ -84,6 +84,22 @@ func Apply(definitionPaths ...string) error { return nil } +// ApplyStdin function adds resources to the Kubernetes cluster based on provided stdin. +func ApplyStdin(input []byte) error { + logger.Debugf("Apply Kubernetes stdin") + out, err := applyKubernetesResourcesStdin(input) + if err != nil { + return errors.Wrap(err, "can't modify Kubernetes resources (apply stdin)") + } + + logger.Debugf("Handle \"apply\" command output") + err = handleApplyCommandOutput(out) + if err != nil { + return errors.Wrap(err, "can't handle command output") + } + return nil +} + func handleApplyCommandOutput(out []byte) error { logger.Debugf("Extract resources from command output") resources, err := extractResources(out) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index c8e45c1c09..ccc53a4e7c 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -5,18 +5,23 @@ package servicedeployer import ( + "io" + "net/http" "os" "path/filepath" + "regexp" "strings" "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/configuration/locations" + "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kind" "github.com/elastic/elastic-package/internal/kubectl" "github.com/elastic/elastic-package/internal/logger" ) +const elasticAgentManagedYamlURL = "https://raw.githubusercontent.com/elastic/beats/7.x/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml" + // KubernetesServiceDeployer is responsible for deploying resources in the Kubernetes cluster. type KubernetesServiceDeployer struct { definitionsDir string @@ -144,14 +149,58 @@ func findKubernetesDefinitions(definitionsDir string) ([]string, error) { func installElasticAgentInCluster() error { logger.Debug("install Elastic Agent in the Kubernetes cluster") - locationManager, err := locations.NewLocationManager() + elasticAgentManagedYaml, err := getElasticAgentYAML() + logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml)) if err != nil { - return errors.Wrap(err, "can't locate Kubernetes file for Elastic Agent in ") + return errors.Wrap(err, "can't retrieve Kubernetes file for Elastic Agent") } - err = kubectl.Apply(locationManager.KubernetesDeployerAgentYml()) + err = kubectl.ApplyStdin(elasticAgentManagedYaml) if err != nil { return errors.Wrap(err, "can't install Elastic-Agent in Kubernetes cluster") } return nil } + +// downloadElasticAgentManagedYAML will download a url from a path and return the response body. +func downloadElasticAgentManagedYAML(url string) ([]byte, error) { + // Get the data + resp, err := http.Get(url) + if err != nil { + return nil, errors.Wrapf(err, "failed to get file from URL %s", url) + } + defer resp.Body.Close() + + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "failed to read response body") + } + return b, nil +} + +// getElasticAgentYAML retrieves elastic-agent-managed.yaml from upstream and modifies the file as needed +// to run locally. +func getElasticAgentYAML() ([]byte, error) { + appConfig, err := install.Configuration() + if err != nil { + return nil, errors.Wrap(err, "can't read application configuration") + } + + logger.Debugf("downloading elastic-agent-managed-kubernetes.yaml from %s", elasticAgentManagedYamlURL) + elasticAgentManagedYaml, err := downloadElasticAgentManagedYAML(elasticAgentManagedYamlURL) + if err != nil { + return nil, errors.Wrapf(err, "downloading failed for file from source %s", elasticAgentManagedYamlURL) + } + + // Set regex to match fleet url from yaml file + fleetURLRegex := regexp.MustCompile("http(s){0,1}:\\/\\/fleet-server:(\\d+)") + // Replace fleet url + elasticAgentManagedYaml = fleetURLRegex.ReplaceAll(elasticAgentManagedYaml, []byte("http://fleet-server:8220")) + + // Set regex to match image name from yaml file + imageRegex := regexp.MustCompile("docker.elastic.co/beats/elastic-agent:\\d.+") + // Replace image name + elasticAgentManagedYaml = imageRegex.ReplaceAll(elasticAgentManagedYaml, []byte(appConfig.DefaultStackImageRefs().ElasticAgent)) + + return elasticAgentManagedYaml, nil +} diff --git a/test/packages/kubernetes/_dev/build/build.yml b/test/packages/kubernetes/_dev/build/build.yml new file mode 100644 index 0000000000..002aa15659 --- /dev/null +++ b/test/packages/kubernetes/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@1.10 diff --git a/test/packages/kubernetes/data_stream/pod/fields/ecs.yml b/test/packages/kubernetes/data_stream/pod/fields/ecs.yml index d7ce11b7c1..adbab0f791 100644 --- a/test/packages/kubernetes/data_stream/pod/fields/ecs.yml +++ b/test/packages/kubernetes/data_stream/pod/fields/ecs.yml @@ -7,3 +7,7 @@ - name: service.type type: keyword description: Service type +- name: orchestrator.cluster.name + external: ecs +- name: orchestrator.cluster.url + external: ecs From 36871550d66929b7d9fee7df6cc49c270f8985d0 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 3 Aug 2021 11:23:08 +0200 Subject: [PATCH 02/13] Revert "Revert: Get elastic-agent-managed-daemonset.yaml from upstream 7.x instead of using a local static file (#459)" This reverts commit 65313621f9bd6b947f27b39b7bde3bdfbdbf6d22. --- internal/configuration/locations/locations.go | 8 +-- internal/install/install.go | 26 --------- internal/kubectl/kubectl.go | 18 ++++++ internal/kubectl/kubectl_apply.go | 16 ++++++ .../system/servicedeployer/kubernetes.go | 57 +++++++++++++++++-- test/packages/kubernetes/_dev/build/build.yml | 3 + .../kubernetes/data_stream/pod/fields/ecs.yml | 4 ++ 7 files changed, 95 insertions(+), 37 deletions(-) create mode 100644 test/packages/kubernetes/_dev/build/build.yml diff --git a/internal/configuration/locations/locations.go b/internal/configuration/locations/locations.go index a6c7930f12..0fdcf35891 100644 --- a/internal/configuration/locations/locations.go +++ b/internal/configuration/locations/locations.go @@ -23,8 +23,7 @@ const ( fieldsCachedDir = "cache/fields" - kubernetesDeployerElasticAgentYmlFile = "elastic-agent.yml" - terraformDeployerYmlFile = "terraform-deployer.yml" + terraformDeployerYmlFile = "terraform-deployer.yml" ) var ( @@ -84,11 +83,6 @@ func (loc LocationManager) KubernetesDeployerDir() string { return filepath.Join(loc.stackPath, kubernetesDeployerDir) } -// KubernetesDeployerAgentYml returns the Kubernetes Deployer Elastic Agent yml -func (loc LocationManager) KubernetesDeployerAgentYml() string { - return filepath.Join(loc.stackPath, kubernetesDeployerDir, kubernetesDeployerElasticAgentYmlFile) -} - // TerraformDeployerDir returns the Terraform Directory func (loc LocationManager) TerraformDeployerDir() string { return filepath.Join(loc.stackPath, terraformDeployerDir) diff --git a/internal/install/install.go b/internal/install/install.go index ff28697520..4b3e038b0a 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "time" "github.com/pkg/errors" @@ -60,11 +59,6 @@ func EnsureInstalled() error { return errors.Wrap(err, "writing stack resources failed") } - err = writeKubernetesDeployerResources(elasticPackagePath) - if err != nil { - return errors.Wrap(err, "writing Kubernetes deployer resources failed") - } - err = writeTerraformDeployerResources(elasticPackagePath) if err != nil { return errors.Wrap(err, "writing Terraform deployer resources failed") @@ -172,26 +166,6 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error { } -func writeKubernetesDeployerResources(elasticPackagePath *locations.LocationManager) error { - err := os.MkdirAll(elasticPackagePath.KubernetesDeployerDir(), 0755) - if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.KubernetesDeployerDir()) - } - - appConfig, err := Configuration() - if err != nil { - return errors.Wrap(err, "can't read application configuration") - } - - err = writeStaticResource(err, elasticPackagePath.KubernetesDeployerAgentYml(), - strings.ReplaceAll(kubernetesDeployerElasticAgentYml, "{{ ELASTIC_AGENT_IMAGE_REF }}", - appConfig.DefaultStackImageRefs().ElasticAgent)) - if err != nil { - return errors.Wrap(err, "writing static resource failed") - } - return nil -} - func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManager) error { terraformDeployer := elasticPackagePath.TerraformDeployerDir() err := os.MkdirAll(terraformDeployer, 0755) diff --git a/internal/kubectl/kubectl.go b/internal/kubectl/kubectl.go index 40a3ea2722..ac40d07d78 100644 --- a/internal/kubectl/kubectl.go +++ b/internal/kubectl/kubectl.go @@ -49,3 +49,21 @@ func modifyKubernetesResources(action string, definitionPaths ...string) ([]byte } return output, nil } + +// applyKubernetesResourcesStdin applies a Kubernetes manifest provided as stdin. +// It returns the resources created as output and an error +func applyKubernetesResourcesStdin(input []byte) ([]byte, error) { + // create kubectl apply command + kubectlCmd := exec.Command("kubectl", "apply", "-f", "-", "-o", "yaml") + //Stdin of kubectl command is the manifest provided + kubectlCmd.Stdin = bytes.NewReader(input) + errOutput := new(bytes.Buffer) + kubectlCmd.Stderr = errOutput + + logger.Debugf("run command: %s", kubectlCmd) + output, err := kubectlCmd.Output() + if err != nil { + return nil, errors.Wrapf(err, "kubectl apply failed (stderr=%q)", errOutput.String()) + } + return output, nil +} diff --git a/internal/kubectl/kubectl_apply.go b/internal/kubectl/kubectl_apply.go index 1bb2c7be8b..c49ef1ff12 100644 --- a/internal/kubectl/kubectl_apply.go +++ b/internal/kubectl/kubectl_apply.go @@ -84,6 +84,22 @@ func Apply(definitionPaths ...string) error { return nil } +// ApplyStdin function adds resources to the Kubernetes cluster based on provided stdin. +func ApplyStdin(input []byte) error { + logger.Debugf("Apply Kubernetes stdin") + out, err := applyKubernetesResourcesStdin(input) + if err != nil { + return errors.Wrap(err, "can't modify Kubernetes resources (apply stdin)") + } + + logger.Debugf("Handle \"apply\" command output") + err = handleApplyCommandOutput(out) + if err != nil { + return errors.Wrap(err, "can't handle command output") + } + return nil +} + func handleApplyCommandOutput(out []byte) error { logger.Debugf("Extract resources from command output") resources, err := extractResources(out) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index c8e45c1c09..ccc53a4e7c 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -5,18 +5,23 @@ package servicedeployer import ( + "io" + "net/http" "os" "path/filepath" + "regexp" "strings" "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/configuration/locations" + "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kind" "github.com/elastic/elastic-package/internal/kubectl" "github.com/elastic/elastic-package/internal/logger" ) +const elasticAgentManagedYamlURL = "https://raw.githubusercontent.com/elastic/beats/7.x/deploy/kubernetes/elastic-agent-managed-kubernetes.yaml" + // KubernetesServiceDeployer is responsible for deploying resources in the Kubernetes cluster. type KubernetesServiceDeployer struct { definitionsDir string @@ -144,14 +149,58 @@ func findKubernetesDefinitions(definitionsDir string) ([]string, error) { func installElasticAgentInCluster() error { logger.Debug("install Elastic Agent in the Kubernetes cluster") - locationManager, err := locations.NewLocationManager() + elasticAgentManagedYaml, err := getElasticAgentYAML() + logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml)) if err != nil { - return errors.Wrap(err, "can't locate Kubernetes file for Elastic Agent in ") + return errors.Wrap(err, "can't retrieve Kubernetes file for Elastic Agent") } - err = kubectl.Apply(locationManager.KubernetesDeployerAgentYml()) + err = kubectl.ApplyStdin(elasticAgentManagedYaml) if err != nil { return errors.Wrap(err, "can't install Elastic-Agent in Kubernetes cluster") } return nil } + +// downloadElasticAgentManagedYAML will download a url from a path and return the response body. +func downloadElasticAgentManagedYAML(url string) ([]byte, error) { + // Get the data + resp, err := http.Get(url) + if err != nil { + return nil, errors.Wrapf(err, "failed to get file from URL %s", url) + } + defer resp.Body.Close() + + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "failed to read response body") + } + return b, nil +} + +// getElasticAgentYAML retrieves elastic-agent-managed.yaml from upstream and modifies the file as needed +// to run locally. +func getElasticAgentYAML() ([]byte, error) { + appConfig, err := install.Configuration() + if err != nil { + return nil, errors.Wrap(err, "can't read application configuration") + } + + logger.Debugf("downloading elastic-agent-managed-kubernetes.yaml from %s", elasticAgentManagedYamlURL) + elasticAgentManagedYaml, err := downloadElasticAgentManagedYAML(elasticAgentManagedYamlURL) + if err != nil { + return nil, errors.Wrapf(err, "downloading failed for file from source %s", elasticAgentManagedYamlURL) + } + + // Set regex to match fleet url from yaml file + fleetURLRegex := regexp.MustCompile("http(s){0,1}:\\/\\/fleet-server:(\\d+)") + // Replace fleet url + elasticAgentManagedYaml = fleetURLRegex.ReplaceAll(elasticAgentManagedYaml, []byte("http://fleet-server:8220")) + + // Set regex to match image name from yaml file + imageRegex := regexp.MustCompile("docker.elastic.co/beats/elastic-agent:\\d.+") + // Replace image name + elasticAgentManagedYaml = imageRegex.ReplaceAll(elasticAgentManagedYaml, []byte(appConfig.DefaultStackImageRefs().ElasticAgent)) + + return elasticAgentManagedYaml, nil +} diff --git a/test/packages/kubernetes/_dev/build/build.yml b/test/packages/kubernetes/_dev/build/build.yml new file mode 100644 index 0000000000..002aa15659 --- /dev/null +++ b/test/packages/kubernetes/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@1.10 diff --git a/test/packages/kubernetes/data_stream/pod/fields/ecs.yml b/test/packages/kubernetes/data_stream/pod/fields/ecs.yml index d7ce11b7c1..adbab0f791 100644 --- a/test/packages/kubernetes/data_stream/pod/fields/ecs.yml +++ b/test/packages/kubernetes/data_stream/pod/fields/ecs.yml @@ -7,3 +7,7 @@ - name: service.type type: keyword description: Service type +- name: orchestrator.cluster.name + external: ecs +- name: orchestrator.cluster.url + external: ecs From ab9128f08371214301bdc7ee797eddedbfeffe02 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 3 Aug 2021 14:19:54 +0300 Subject: [PATCH 03/13] Retry when downloading fails --- .../system/servicedeployer/kubernetes.go | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index ccc53a4e7c..5480f8b1c0 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -5,12 +5,14 @@ package servicedeployer import ( + "fmt" "io" "net/http" "os" "path/filepath" "regexp" "strings" + "time" "github.com/pkg/errors" @@ -150,7 +152,6 @@ func installElasticAgentInCluster() error { logger.Debug("install Elastic Agent in the Kubernetes cluster") elasticAgentManagedYaml, err := getElasticAgentYAML() - logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml)) if err != nil { return errors.Wrap(err, "can't retrieve Kubernetes file for Elastic Agent") } @@ -170,7 +171,7 @@ func downloadElasticAgentManagedYAML(url string) ([]byte, error) { return nil, errors.Wrapf(err, "failed to get file from URL %s", url) } defer resp.Body.Close() - + logger.Debugf("Status code when downloading elastic-agent-managed-kubernetes.yaml is %d", resp.StatusCode) b, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "failed to read response body") @@ -187,10 +188,11 @@ func getElasticAgentYAML() ([]byte, error) { } logger.Debugf("downloading elastic-agent-managed-kubernetes.yaml from %s", elasticAgentManagedYamlURL) - elasticAgentManagedYaml, err := downloadElasticAgentManagedYAML(elasticAgentManagedYamlURL) + elasticAgentManagedYaml, err := retryDownloadElasticAgentManagedYAML(elasticAgentManagedYamlURL, 5, 10, downloadElasticAgentManagedYAML) if err != nil { return nil, errors.Wrapf(err, "downloading failed for file from source %s", elasticAgentManagedYamlURL) } + logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml)) // Set regex to match fleet url from yaml file fleetURLRegex := regexp.MustCompile("http(s){0,1}:\\/\\/fleet-server:(\\d+)") @@ -204,3 +206,23 @@ func getElasticAgentYAML() ([]byte, error) { return elasticAgentManagedYaml, nil } + +func retryDownloadElasticAgentManagedYAML(url string, attempts int, sleep time.Duration, f func(string) ([]byte, error)) ([]byte, error) { + var err error + for i := 0; i < attempts; i++ { + if i > 0 { + logger.Debugf("retrying after error: %s", err) + time.Sleep(sleep) + sleep *= 2 + } + elasticAgentManagedYaml, err := f(url) + if err == nil { + logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml)) + if len(elasticAgentManagedYaml) > 1000 { + return elasticAgentManagedYaml, nil + } + err = fmt.Errorf("bytes downloaded are too low: %d", len(elasticAgentManagedYaml)) + } + } + return nil, fmt.Errorf("after %d attempts, last error: %s", attempts, err) +} From 5f6cccf2f6bb48a20e4f75412c9fc921b6c9f54c Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 3 Aug 2021 14:25:13 +0300 Subject: [PATCH 04/13] Remove unused debug --- internal/testrunner/runners/system/servicedeployer/kubernetes.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index 5480f8b1c0..79ef357379 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -192,7 +192,6 @@ func getElasticAgentYAML() ([]byte, error) { if err != nil { return nil, errors.Wrapf(err, "downloading failed for file from source %s", elasticAgentManagedYamlURL) } - logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml)) // Set regex to match fleet url from yaml file fleetURLRegex := regexp.MustCompile("http(s){0,1}:\\/\\/fleet-server:(\\d+)") From 5bfd3d4b89a7dfd6819346a4feb5d7fd513e87ab Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 3 Aug 2021 14:27:14 +0300 Subject: [PATCH 05/13] Add debug --- .../testrunner/runners/system/servicedeployer/kubernetes.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index 79ef357379..9f41acc38a 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -221,6 +221,8 @@ func retryDownloadElasticAgentManagedYAML(url string, attempts int, sleep time.D return elasticAgentManagedYaml, nil } err = fmt.Errorf("bytes downloaded are too low: %d", len(elasticAgentManagedYaml)) + logger.Debugf("Failed because of %s", err) + logger.Debugf("manifest downloaded is %s", string(elasticAgentManagedYaml)) } } return nil, fmt.Errorf("after %d attempts, last error: %s", attempts, err) From 6b59a7e0ea48cf0c25465db1203be1c662c476bf Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 3 Aug 2021 16:04:21 +0300 Subject: [PATCH 06/13] Simplify debug messages --- .../runners/system/servicedeployer/kubernetes.go | 14 +++++++++----- scripts/test-check-packages.sh | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index 9f41acc38a..e89dd453a0 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -172,6 +172,9 @@ func downloadElasticAgentManagedYAML(url string) ([]byte, error) { } defer resp.Body.Close() logger.Debugf("Status code when downloading elastic-agent-managed-kubernetes.yaml is %d", resp.StatusCode) + if resp.StatusCode != 200 { + return nil, errors.Wrapf(err, "downloading fail due to status code %d", resp.StatusCode) + } b, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "failed to read response body") @@ -206,24 +209,25 @@ func getElasticAgentYAML() ([]byte, error) { return elasticAgentManagedYaml, nil } +// retryDownloadElasticAgentManagedYAML retries downloading elastic agent managed manifest for x attempts +// until there is no error and bytes of the file are more than 2000 func retryDownloadElasticAgentManagedYAML(url string, attempts int, sleep time.Duration, f func(string) ([]byte, error)) ([]byte, error) { var err error for i := 0; i < attempts; i++ { if i > 0 { - logger.Debugf("retrying after error: %s", err) + logger.Debugf("retrying download after error: %s", err) time.Sleep(sleep) - sleep *= 2 } elasticAgentManagedYaml, err := f(url) if err == nil { logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml)) - if len(elasticAgentManagedYaml) > 1000 { + if len(elasticAgentManagedYaml) > 2000 { return elasticAgentManagedYaml, nil } err = fmt.Errorf("bytes downloaded are too low: %d", len(elasticAgentManagedYaml)) logger.Debugf("Failed because of %s", err) - logger.Debugf("manifest downloaded is %s", string(elasticAgentManagedYaml)) + logger.Debugf("File downloaded is %s", string(elasticAgentManagedYaml)) } } - return nil, fmt.Errorf("after %d attempts, last error: %s", attempts, err) + return nil, errors.Wrapf(err, "after %d attempts of downloading elastic-agent-managed-kubernetes.yaml", attempts) } diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 7c86ed8e2d..7806ba8b86 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -1,4 +1,4 @@ -#!/bin/bash + #!/bin/bash set -euxo pipefail @@ -10,7 +10,7 @@ cleanup() { # Dump kubectl details kubectl describe pods --all-namespaces > build/kubectl-dump.txt - kubectl logs -l app=kind-fleet-agent-clusterscope -n kube-system >> build/kubectl-dump.txt + kubectl logs -l app=elastic-agent -n kube-system >> build/kubectl-dump.txt # Take down the kind cluster kind delete cluster From 2866387ff1680d6be1c9ffb38f8ad897af8f4a4f Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 3 Aug 2021 16:13:23 +0300 Subject: [PATCH 07/13] Add comment --- internal/testrunner/runners/system/servicedeployer/kubernetes.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index e89dd453a0..c5285ac78c 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -191,6 +191,7 @@ func getElasticAgentYAML() ([]byte, error) { } logger.Debugf("downloading elastic-agent-managed-kubernetes.yaml from %s", elasticAgentManagedYamlURL) + // retry downloading elastic agent manifest for 5 times (sleep 10 seconds between each try) in case of error elasticAgentManagedYaml, err := retryDownloadElasticAgentManagedYAML(elasticAgentManagedYamlURL, 5, 10, downloadElasticAgentManagedYAML) if err != nil { return nil, errors.Wrapf(err, "downloading failed for file from source %s", elasticAgentManagedYamlURL) From d533e8d833430386a0c3f9316bfda49ebab27064 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 3 Aug 2021 16:25:42 +0300 Subject: [PATCH 08/13] Fix error --- scripts/test-check-packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 7806ba8b86..2a9740dc88 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -1,4 +1,4 @@ - #!/bin/bash +#!/bin/bash set -euxo pipefail From 7adee3587e0aaa7db2378d6c444f23b1fc8f8e13 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 3 Aug 2021 16:35:38 +0300 Subject: [PATCH 09/13] Refactor debug messages --- .../runners/system/servicedeployer/kubernetes.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index c5285ac78c..8116974d52 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -171,9 +171,9 @@ func downloadElasticAgentManagedYAML(url string) ([]byte, error) { return nil, errors.Wrapf(err, "failed to get file from URL %s", url) } defer resp.Body.Close() - logger.Debugf("Status code when downloading elastic-agent-managed-kubernetes.yaml is %d", resp.StatusCode) + logger.Debugf("status code when downloading elastic-agent-managed-kubernetes.yaml is %d", resp.StatusCode) if resp.StatusCode != 200 { - return nil, errors.Wrapf(err, "downloading fail due to status code %d", resp.StatusCode) + return nil, errors.Wrapf(err, "downloading failed due to status code %d", resp.StatusCode) } b, err := io.ReadAll(resp.Body) if err != nil { @@ -225,10 +225,10 @@ func retryDownloadElasticAgentManagedYAML(url string, attempts int, sleep time.D if len(elasticAgentManagedYaml) > 2000 { return elasticAgentManagedYaml, nil } - err = fmt.Errorf("bytes downloaded are too low: %d", len(elasticAgentManagedYaml)) - logger.Debugf("Failed because of %s", err) - logger.Debugf("File downloaded is %s", string(elasticAgentManagedYaml)) + err = fmt.Errorf("bytes downloaded should be more than 2000 but where: %d", len(elasticAgentManagedYaml)) + logger.Debugf("failed because of %s", err) + logger.Debugf("file downloaded is %s", string(elasticAgentManagedYaml)) } } - return nil, errors.Wrapf(err, "after %d attempts of downloading elastic-agent-managed-kubernetes.yaml", attempts) + return nil, errors.Wrapf(err, "failing after %d unsuccessful attempts of downloading elastic-agent-managed-kubernetes.yaml", attempts) } From ac82a52fc5e2ae0a9d16f343a9a4b5f0f1b093b2 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 3 Aug 2021 16:35:57 +0300 Subject: [PATCH 10/13] Refactor debug messages --- .../testrunner/runners/system/servicedeployer/kubernetes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index 8116974d52..b9f50fa79e 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -211,7 +211,7 @@ func getElasticAgentYAML() ([]byte, error) { } // retryDownloadElasticAgentManagedYAML retries downloading elastic agent managed manifest for x attempts -// until there is no error and bytes of the file are more than 2000 +// until there is no error and bytes of the file are more than 2000. func retryDownloadElasticAgentManagedYAML(url string, attempts int, sleep time.Duration, f func(string) ([]byte, error)) ([]byte, error) { var err error for i := 0; i < attempts; i++ { From 9a26d5c883d9f09bdf6ccf925ba7952c1d3bc6a4 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Tue, 3 Aug 2021 17:30:57 +0300 Subject: [PATCH 11/13] Updated error handling --- .../system/servicedeployer/kubernetes.go | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/kubernetes.go b/internal/testrunner/runners/system/servicedeployer/kubernetes.go index b9f50fa79e..646234fd39 100644 --- a/internal/testrunner/runners/system/servicedeployer/kubernetes.go +++ b/internal/testrunner/runners/system/servicedeployer/kubernetes.go @@ -171,14 +171,16 @@ func downloadElasticAgentManagedYAML(url string) ([]byte, error) { return nil, errors.Wrapf(err, "failed to get file from URL %s", url) } defer resp.Body.Close() - logger.Debugf("status code when downloading elastic-agent-managed-kubernetes.yaml is %d", resp.StatusCode) - if resp.StatusCode != 200 { - return nil, errors.Wrapf(err, "downloading failed due to status code %d", resp.StatusCode) - } + b, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, "failed to read response body") } + + logger.Debugf("status code when downloading elastic-agent-managed-kubernetes.yaml is %d", resp.StatusCode) + if resp.StatusCode != 200 { + return nil, fmt.Errorf("downloading failed due to status code %d, resp body: %s", resp.StatusCode, string(b)) + } return b, nil } @@ -192,7 +194,8 @@ func getElasticAgentYAML() ([]byte, error) { logger.Debugf("downloading elastic-agent-managed-kubernetes.yaml from %s", elasticAgentManagedYamlURL) // retry downloading elastic agent manifest for 5 times (sleep 10 seconds between each try) in case of error - elasticAgentManagedYaml, err := retryDownloadElasticAgentManagedYAML(elasticAgentManagedYamlURL, 5, 10, downloadElasticAgentManagedYAML) + elasticAgentManagedYaml, err := retryDownloadElasticAgentManagedYAML(elasticAgentManagedYamlURL, 5, 10, + downloadElasticAgentManagedYAML) if err != nil { return nil, errors.Wrapf(err, "downloading failed for file from source %s", elasticAgentManagedYamlURL) } @@ -212,23 +215,23 @@ func getElasticAgentYAML() ([]byte, error) { // retryDownloadElasticAgentManagedYAML retries downloading elastic agent managed manifest for x attempts // until there is no error and bytes of the file are more than 2000. -func retryDownloadElasticAgentManagedYAML(url string, attempts int, sleep time.Duration, f func(string) ([]byte, error)) ([]byte, error) { - var err error +func retryDownloadElasticAgentManagedYAML(url string, attempts int, sleep time.Duration, f func(string) ([]byte, error)) ( + elasticAgentManagedYaml []byte, err error) { for i := 0; i < attempts; i++ { if i > 0 { - logger.Debugf("retrying download after error: %s", err) - time.Sleep(sleep) + logger.Debugf("retrying download attempt %d", i+1) + time.Sleep(sleep * time.Second) } - elasticAgentManagedYaml, err := f(url) + elasticAgentManagedYaml, err = f(url) if err == nil { logger.Debugf("downloaded %d bytes", len(elasticAgentManagedYaml)) if len(elasticAgentManagedYaml) > 2000 { return elasticAgentManagedYaml, nil } err = fmt.Errorf("bytes downloaded should be more than 2000 but where: %d", len(elasticAgentManagedYaml)) - logger.Debugf("failed because of %s", err) - logger.Debugf("file downloaded is %s", string(elasticAgentManagedYaml)) + logger.Debugf("failed because %s", err) } } - return nil, errors.Wrapf(err, "failing after %d unsuccessful attempts of downloading elastic-agent-managed-kubernetes.yaml", attempts) + return nil, + errors.Wrapf(err, "failed after %d unsuccessful attempts of downloading elastic-agent-managed-kubernetes.yaml", attempts) } From f4c8b4645376c1835f82789f73a40f3946ec643e Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Wed, 4 Aug 2021 13:07:54 +0300 Subject: [PATCH 12/13] Add comment about wait of daemonset not working --- internal/kubectl/kubectl_apply.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/kubectl/kubectl_apply.go b/internal/kubectl/kubectl_apply.go index c49ef1ff12..5f139de589 100644 --- a/internal/kubectl/kubectl_apply.go +++ b/internal/kubectl/kubectl_apply.go @@ -129,6 +129,11 @@ func waitForReadyResources(resources []resource) error { kubeClient.Log = func(s string, i ...interface{}) { logger.Debugf(s, i...) } + // In case of elastic-agent daemonset Wait will not work as expected + // because in single node clusters one pod of the daemonset can always + // be unavailable (DaemonSet.spec.updateStrategy.rollingUpdate.maxUnavailable defaults to 1). + // daemonSetReady will return true regardless of the pod not being ready yet. + // Can be solved with multi-node clusters. err := kubeClient.Wait(resList, readinessTimeout) if err != nil { return errors.Wrap(err, "waiter failed") From 127f5874228d5f275d13969eb1aada9bdd7cfcf6 Mon Sep 17 00:00:00 2001 From: MichaelKatsoulis Date: Wed, 4 Aug 2021 13:19:59 +0300 Subject: [PATCH 13/13] Delete old static agent file --- .../static_kubernetes_elastic_agent_yml.go | 173 ------------------ 1 file changed, 173 deletions(-) delete mode 100644 internal/install/static_kubernetes_elastic_agent_yml.go diff --git a/internal/install/static_kubernetes_elastic_agent_yml.go b/internal/install/static_kubernetes_elastic_agent_yml.go deleted file mode 100644 index f8ad6fd966..0000000000 --- a/internal/install/static_kubernetes_elastic_agent_yml.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package install - -const kubernetesDeployerElasticAgentYml = `--- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kind-fleet-agent-clusterscope - namespace: kube-system - labels: - app: kind-fleet-agent-clusterscope - group: fleet -spec: - selector: - matchLabels: - app: kind-fleet-agent-clusterscope - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - template: - metadata: - labels: - app: kind-fleet-agent-clusterscope - group: fleet - spec: - serviceAccountName: kind-fleet-agent - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: kind-fleet-agent-clusterscope - image: {{ ELASTIC_AGENT_IMAGE_REF }} - env: - - name: FLEET_ENROLL - value: "1" - - name: FLEET_INSECURE - value: "1" - - name: FLEET_URL - value: "http://fleet-server:8220" - - name: STATE_PATH - value: /usr/share/elastic-agent - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - resources: - limits: - memory: 400Mi - requests: - cpu: 200m - memory: 400Mi - startupProbe: - exec: - command: - - elastic-agent - - status ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: kind-fleet-agent-clusterscope - namespace: kube-system - labels: - group: fleet -data: - elastic-agent.yml: |- - management: - mode: "fleet" - grpc: - # listen address for the GRPC server that spawned processes connect back to. - address: localhost - # port for the GRPC server that spawned processes connect back to. - port: 6789 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kind-fleet-agent -subjects: - - kind: ServiceAccount - name: kind-fleet-agent - namespace: kube-system -roleRef: - kind: ClusterRole - name: kind-fleet-agent - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: kind-fleet-agent - labels: - k8s-app: kind-fleet-agent -rules: - - apiGroups: [""] - resources: - - nodes - - namespaces - - events - - pods - - secrets - - services - verbs: ["get", "list", "watch"] - - apiGroups: ["extensions"] - resources: - - replicasets - verbs: ["get", "list", "watch"] - - apiGroups: ["apps"] - resources: - - statefulsets - - deployments - - replicasets - verbs: ["get", "list", "watch"] - - apiGroups: [ "batch" ] - resources: - - jobs - verbs: [ "get", "list", "watch" ] - - apiGroups: - - "" - resources: - - nodes/stats - verbs: - - get - # required for apiserver - - nonResourceURLs: - - "/metrics" - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - namespace: kube-system - name: kind-fleet-agent -subjects: - - kind: ServiceAccount - name: kind-fleet-agent - namespace: kube-system -roleRef: - kind: Role - name: kind-fleet-agent - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: kind-fleet-agent - namespace: kube-system - labels: - k8s-app: kind-fleet-agent -rules: - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: ["get", "create", "update"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kind-fleet-agent - namespace: kube-system - labels: - k8s-app: kind-fleet-agent ---- -`