From 8fe6892c739279056f68b4a67bbe1819d2e4f84c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 27 Jun 2023 14:47:40 +0200 Subject: [PATCH 1/6] Add docs to use custom images --- cmd/stack.go | 6 +- docs/howto/custom_images.md | 127 ++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 docs/howto/custom_images.md diff --git a/cmd/stack.go b/cmd/stack.go index 03e20bc55d..0e07dbb442 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -26,7 +26,7 @@ var availableServices = map[string]struct{}{ "package-registry": {}, } -const stackLongDescription = `Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . +const stackLongDescription = `Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . In case, you need to run your own custom images (e.g. Elasticsearch, Kibana or elastic-agent), see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. @@ -34,11 +34,11 @@ For details on how to connect the service with the Elastic stack, see the [servi const stackUpLongDescription = `Use this command to boot up the stack locally. -By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . +By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . In case, you need to run your own custom images (e.g. Elasticsearch, Kibana or elastic-agent), see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. -To expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. +For Elastic stack versions prior to 8.7, to expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service).` diff --git a/docs/howto/custom_images.md b/docs/howto/custom_images.md new file mode 100644 index 0000000000..88ea8f7751 --- /dev/null +++ b/docs/howto/custom_images.md @@ -0,0 +1,127 @@ +# Use your own custom images + +By default, `elastic-package stack up` command starts the required docker images for each service. + +Setting `--version` flag, you can start the Elastic stack using different versions, for instance SNAPSHOT version (e.g. 8.8.2-SNAPSHOT). +If this flag is not set, the default version used is by elastic-package is defined [here](../../internal/install/stack_version.go). + +There could be cases where you need to use your own custom images for development or debugging purposes. +If you need to use your own custom docker images for your service (e.g. elastic-agent), this could be +achieved in two different ways: +- [defining environment variables](#using-environment-variables) +- [updating elastic-package configuration file](#using-the-configuration-file) + +If both way are used, environment variables have preference to set the custom image. + +The current images that could be overwritten are: + +| Service | Environment Variable | Entry configuration file | +| --- | --- | --- | +| Elasticsearch | ELASTICSEARCH_IMAGE_REF_OVERRIDE | elasticsearch | +| Kibana | KIBANA_IMAGE_REF_OVERRIDE | kibana | +| Elastic Agent | ELASTIC_AGENT_IMAGE_REF_OVERRIDE | elastic-agent | + + +For the following two examples, it will be used as example overwriting elastic-agent image. +Here, there are the steps to follow to create the images for the examples ([docs link](https://github.com/elastic/elastic-agent#packaging)): + +```bash +# Go to elastic-agent repository and create the packages for docker +git checkout 8.8 # or the required tag like v8.8.* +DEV=true SNAPSHOT=true EXTERNAL=true PLATFORMS=linux/amd64 PACKAGES=docker mage -v package + +docker tag docker.elastic.co/beats/elastic-agent-complete:8.8.2-SNAPSHOT docker.elastic.co/beats/elastic-agent-complete:8.8.2-SNAPSHOT-test + +git checkout v8.8.1 +DEV=true SNAPSHOT=true EXTERNAL=true PLATFORMS=linux/amd64 PACKAGES=docker mage -v package + +docker tag docker.elastic.co/beats/elastic-agent-complete:8.8.1-SNAPSHOT docker.elastic.co/beats/elastic-agent-complete:8.8.1-SNAPSHOT-test +docker tag docker.elastic.co/beats/elastic-agent-complete:8.8.1-SNAPSHOT docker.elastic.co/beats/elastic-agent-complete:8.8.1-SNAPSHOT-foo +``` + +## Using Environment variables + +Depending on the service you need to use a custom image, you need to set a different environment variable. + +Following the example of elastic-agent, it is required to define the `ELASTICSEARCH_IMAGE_REF_OVERRIDE` environment variable before running `elastic-package stack up`: +``` +export ELASTIC_AGENT_IMAGE_REF="docker.elastic.co/beats/elastic-agent-complete:8.8.2-SNAPSHOT-test" +elastic-package stack up -v -d --version 8.8.0-SNAPSHOT +``` + +Once the Elastic stack is running, you can check that the docker image used for the elastic-agent image is now the one it was built: + +```bash + $ docker ps --format "{{.Names}} {{.Image}}" +elastic-package-stack-elastic-agent-1 docker.elastic.co/beats/elastic-agent-complete:8.8.2-SNAPSHOT-test +elastic-package-stack-fleet-server-1 docker.elastic.co/beats/elastic-agent-complete:8.8.2-SNAPSHOT-test +elastic-package-stack-kibana-1 docker.elastic.co/kibana/kibana:8.8.2-SNAPSHOT +elastic-package-stack-elasticsearch-1 docker.elastic.co/elasticsearch/elasticsearch:8.8.2-SNAPSHOT +elastic-package-stack-package-registry-1 elastic-package-stack-package-registry +``` + +## Using the configuration file + +These custom version or images can also be defined in the configuration file that is located at +`~/.elastic-package/config.yml`. + +```yaml +stack: + image_ref_overrides: {} +profile: + current: default +``` + +To define the same elastic-agent image as in the previous section without any environment variable, +it should be added a map for each Elastic stack version (e.g. 8.8.2-SNAPSHOT) under `image_ref_overrides` key. +Each Elastic stack map would contain the required custom images. + +It is important to note that Elastic stack version set in this configuration file must match with the version +set in the `--version` flag. + +For instance, the following configuration file sets a custom image for Elastic stack versions `8.8.2-SNAPSHOT` and `8.8.1-SNAPSHOT`: + +```yaml +stack: + image_ref_overrides: + 8.8.2-SNAPSHOT: + elastic-agent: docker.elastic.co/beats/elastic-agnet-complete:8.8-2-SNAPSHOT-test + # elasticsearch: + # kibana: + 8.8.1-SNAPSHOT: + elastic-agent: docker.elastic.co/beats/elastic-agnet-complete:8.8-1-SNAPSHOT-test + # elasticsearch: + # kibana: +profile: + current: default +``` + +If Elastic stack version `8.8.1` is started, then there would not be any custom image spun up. + +Some examples with the given configuration file: + +```bash + $ elastic-stack up -v --version 8.8.1-SNAPSHOT + $ docker ps --format "{{.Names}} {{.Image}}" +elastic-package-stack-elastic-agent-1 docker.elastic.co/beats/elastic-agent-complete:8.8.1-SNAPSHOT-test +elastic-package-stack-fleet-server-1 docker.elastic.co/beats/elastic-agent-complete:8.8.1-SNAPSHOT-test +elastic-package-stack-kibana-1 docker.elastic.co/kibana/kibana:8.8.1-SNAPSHOT +elastic-package-stack-elasticsearch-1 docker.elastic.co/elasticsearch/elasticsearch:8.8.1-SNAPSHOT +elastic-package-stack-package-registry-1 elastic-package-stack-package-registry + + $ elastic-stack up -v --version 8.8.2-SNAPSHOT + $ docker ps --format "{{.Names}} {{.Image}}" +elastic-package-stack-elastic-agent-1 docker.elastic.co/beats/elastic-agent-complete:8.8.2-SNAPSHOT-test +elastic-package-stack-fleet-server-1 docker.elastic.co/beats/elastic-agent-complete:8.8.2-SNAPSHOT-test +elastic-package-stack-kibana-1 docker.elastic.co/kibana/kibana:8.8.2-SNAPSHOT +elastic-package-stack-elasticsearch-1 docker.elastic.co/elasticsearch/elasticsearch:8.8.2-SNAPSHOT +elastic-package-stack-package-registry-1 elastic-package-stack-package-registry + + $ elastic-stack up -v --version 8.8.1 + $ docker ps --format "{{.Names}} {{.Image}}" +elastic-package-stack-elastic-agent-1 docker.elastic.co/elastic-agent/elastic-agent-complete:8.8.1 +elastic-package-stack-fleet-server-1 docker.elastic.co/elastic-agent/elastic-agent-complete:8.8.1 +elastic-package-stack-kibana-1 docker.elastic.co/kibana/kibana:8.8.1 +elastic-package-stack-elasticsearch-1 docker.elastic.co/elasticsearch/elasticsearch:8.8.1 +elastic-package-stack-package-registry-1 elastic-package-stack-package-registry +``` From 133966c5b10fb9a5d53c3c7f4743494bd57e3448 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 27 Jun 2023 16:04:04 +0200 Subject: [PATCH 2/6] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e6bbe418b1..aa960b2b7e 100644 --- a/README.md +++ b/README.md @@ -360,7 +360,7 @@ Boot up the stack. _Context: global_ -Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . +Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . In case, you need to run your own custom images (e.g. Elasticsearch, Kibana or elastic-agent), see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. @@ -396,11 +396,11 @@ _Context: global_ Use this command to boot up the stack locally. -By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . +By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . In case, you need to run your own custom images (e.g. Elasticsearch, Kibana or elastic-agent), see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. -To expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. +For Elastic stack versions prior to 8.7, to expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service). From 7da737a70ba794a3a7196c49bb9662df6060297c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 27 Jun 2023 22:33:00 +0200 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Jaime Soriano Pastor --- docs/howto/custom_images.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/howto/custom_images.md b/docs/howto/custom_images.md index 88ea8f7751..fef71777fa 100644 --- a/docs/howto/custom_images.md +++ b/docs/howto/custom_images.md @@ -11,7 +11,7 @@ achieved in two different ways: - [defining environment variables](#using-environment-variables) - [updating elastic-package configuration file](#using-the-configuration-file) -If both way are used, environment variables have preference to set the custom image. +If both ways are used, environment variables have preference to set the custom image. The current images that could be overwritten are: @@ -23,7 +23,7 @@ The current images that could be overwritten are: For the following two examples, it will be used as example overwriting elastic-agent image. -Here, there are the steps to follow to create the images for the examples ([docs link](https://github.com/elastic/elastic-agent#packaging)): +You can find here the instructions to create the docker images for the examples ([docs link](https://github.com/elastic/elastic-agent#packaging)): ```bash # Go to elastic-agent repository and create the packages for docker From 58966733aba259b9590393954be48141029e08e2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 27 Jun 2023 22:35:22 +0200 Subject: [PATCH 4/6] Update readme --- README.md | 8 ++++++-- cmd/stack.go | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index aa960b2b7e..e896a1d7ea 100644 --- a/README.md +++ b/README.md @@ -360,7 +360,9 @@ Boot up the stack. _Context: global_ -Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . In case, you need to run your own custom images (e.g. Elasticsearch, Kibana or elastic-agent), see [this document](./docs/howto/custom_images.md). +Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . + +You can run your own custom images for Elasticsearch, Kibana or elastic-agent, see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. @@ -396,7 +398,9 @@ _Context: global_ Use this command to boot up the stack locally. -By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . In case, you need to run your own custom images (e.g. Elasticsearch, Kibana or elastic-agent), see [this document](./docs/howto/custom_images.md). +By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . + +You can run your own custom images for Elasticsearch, Kibana or elastic-agent, see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. diff --git a/cmd/stack.go b/cmd/stack.go index 0e07dbb442..9086d331cc 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -26,7 +26,9 @@ var availableServices = map[string]struct{}{ "package-registry": {}, } -const stackLongDescription = `Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . In case, you need to run your own custom images (e.g. Elasticsearch, Kibana or elastic-agent), see [this document](./docs/howto/custom_images.md). +const stackLongDescription = `Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . + +You can run your own custom images for Elasticsearch, Kibana or elastic-agent, see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. @@ -34,7 +36,9 @@ For details on how to connect the service with the Elastic stack, see the [servi const stackUpLongDescription = `Use this command to boot up the stack locally. -By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . In case, you need to run your own custom images (e.g. Elasticsearch, Kibana or elastic-agent), see [this document](./docs/howto/custom_images.md). +By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . + +You can run your own custom images for Elasticsearch, Kibana or elastic-agent, see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. From 2a58163533ae4aa21e0d988cc1c504fe1518a092 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 27 Jun 2023 22:40:34 +0200 Subject: [PATCH 5/6] Rephrase README --- README.md | 2 +- cmd/stack.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e896a1d7ea..0ca70813a8 100644 --- a/README.md +++ b/README.md @@ -404,7 +404,7 @@ You can run your own custom images for Elasticsearch, Kibana or elastic-agent, s Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. -For Elastic stack versions prior to 8.7, to expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. +To expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. Starting with Elastic stack version >= 8.7.0, it is not mandatory to be available local packages in the Package Registry to run the tests. For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service). diff --git a/cmd/stack.go b/cmd/stack.go index 9086d331cc..0e538f106b 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -42,7 +42,7 @@ You can run your own custom images for Elasticsearch, Kibana or elastic-agent, s Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. -For Elastic stack versions prior to 8.7, to expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. +To expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. Starting with Elastic stack version >= 8.7.0, it is not mandatory to be available local packages in the Package Registry to run the tests. For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service).` From 0d615ed1ada685ef6df860332a9cd0d9c4c0227b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 28 Jun 2023 09:19:10 +0200 Subject: [PATCH 6/6] Use uppercased names --- README.md | 4 ++-- cmd/stack.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0ca70813a8..27fc3226d9 100644 --- a/README.md +++ b/README.md @@ -362,7 +362,7 @@ _Context: global_ Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . -You can run your own custom images for Elasticsearch, Kibana or elastic-agent, see [this document](./docs/howto/custom_images.md). +You can run your own custom images for Elasticsearch, Kibana or Elastic Agent, see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. @@ -400,7 +400,7 @@ Use this command to boot up the stack locally. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . -You can run your own custom images for Elasticsearch, Kibana or elastic-agent, see [this document](./docs/howto/custom_images.md). +You can run your own custom images for Elasticsearch, Kibana or Elastic Agent, see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. diff --git a/cmd/stack.go b/cmd/stack.go index 0e538f106b..a2a755fe43 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -28,7 +28,7 @@ var availableServices = map[string]struct{}{ const stackLongDescription = `Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . -You can run your own custom images for Elasticsearch, Kibana or elastic-agent, see [this document](./docs/howto/custom_images.md). +You can run your own custom images for Elasticsearch, Kibana or Elastic Agent, see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. @@ -38,7 +38,7 @@ const stackUpLongDescription = `Use this command to boot up the stack locally. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . -You can run your own custom images for Elasticsearch, Kibana or elastic-agent, see [this document](./docs/howto/custom_images.md). +You can run your own custom images for Elasticsearch, Kibana or Elastic Agent, see [this document](./docs/howto/custom_images.md). Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold.