Inherit all environment variables from ES container in initContainers (…

…#5962)

docs/operating-eck/upgrading-eck.asciidoc

@@ -82,6 +82,7 @@ Upgrading the operator results in a one-time update to existing managed resource
 * 2.1
 * 2.2
 * 2.4
+* 2.5
 
 If you have a very large Elasticsearch cluster or multiple Elastic Stack deployments, this rolling restart might be disruptive or inconvenient. To have more control over when the pods belonging to a particular deployment should be restarted, you can <<{p}-exclude-resource,add an annotation>> to the corresponding resources to temporarily exclude them from being managed by the operator. When the time is convenient, you can remove the annotation and let the rolling restart go through.
 

docs/orchestrating-elastic-stack-applications/elasticsearch/advanced-node-scheduling.asciidoc

@@ -238,14 +238,6 @@ spec:
       cluster.routing.allocation.awareness.attributes: k8s_node_name,zone
     podTemplate:
       spec:
-        # The initContainers section is only required if using secureSettings in conjunction with zone awareness.
-        # initContainers:
-        # - name: elastic-internal-init-keystore
-        #   env:
-        #   - name: ZONE
-        #     valueFrom:
-        #       fieldRef:
-        #         fieldPath: metadata.annotations['topology.kubernetes.io/zone']
         containers:
         - name: elasticsearch
           env:

docs/orchestrating-elastic-stack-applications/elasticsearch/es-secure-settings.asciidoc

@@ -71,5 +71,3 @@ data:
 ----
 
 Check <<{p}-snapshots,How to create automated snapshots>> for an example use case.
-
-WARNING: If using <<{p}-advanced-node-scheduling>> in conjunction with secure settings, you have to add an `initContainers` section to the `podTemplate` to ensure all the required environment variables exist for the initialization of the keystore.  Refer to <<{p}-availability-zone-awareness-example,Zone Awareness>> for a complete example.

pkg/controller/common/defaults/pod_template.go

@@ -57,8 +57,8 @@ func NewPodTemplateBuilder(base corev1.PodTemplateSpec, containerName string) *P
 	return builder.setDefaults()
 }
 
-// getContainer retrieves the main Container from the pod template
-func (b *PodTemplateBuilder) getContainer() *corev1.Container {
+// MainContainer retrieves the main Container from the pod template or nil if not found.
+func (b *PodTemplateBuilder) MainContainer() *corev1.Container {
 	for i, c := range b.PodTemplate.Spec.Containers {
 		if c.Name == b.containerName {
 			return &b.PodTemplate.Spec.Containers[i]
@@ -68,13 +68,13 @@ func (b *PodTemplateBuilder) getContainer() *corev1.Container {
 }
 
 func (b *PodTemplateBuilder) setContainerDefaulter() {
-	b.containerDefaulter = container.NewDefaulter(b.getContainer())
+	b.containerDefaulter = container.NewDefaulter(b.MainContainer())
 }
 
 // setDefaults sets up a default Container in the pod template,
 // and disables service account token auto mount.
 func (b *PodTemplateBuilder) setDefaults() *PodTemplateBuilder {
-	userContainer := b.getContainer()
+	userContainer := b.MainContainer()
 	if userContainer == nil {
 		// create the default Container if not provided by the user
 		b.PodTemplate.Spec.Containers = append(b.PodTemplate.Spec.Containers, corev1.Container{Name: b.containerName})

pkg/controller/elasticsearch/nodespec/podspec.go

@@ -112,7 +112,8 @@ func BuildPodTemplateSpec(
 		WithVolumes(volumes...).
 		WithVolumeMounts(volumeMounts...).
 		WithInitContainers(initContainers...).
-		WithInitContainerDefaults(corev1.EnvVar{Name: settings.HeadlessServiceName, Value: headlessServiceName}).
+		// inherit all env vars from main containers to allow Elasticsearch tools that read ES config to work in initContainers
+		WithInitContainerDefaults(builder.MainContainer().Env...).
 		WithPreStopHook(*NewPreStopHook())
 
 	builder, err = stackmon.WithMonitoring(ctx, client, builder, es)

pkg/controller/elasticsearch/nodespec/podspec_test.go

@@ -6,6 +6,7 @@ package nodespec
 
 import (
 	"context"
+	"path"
 	"sort"
 	"testing"
 
@@ -23,6 +24,8 @@ import (
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/volume"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/initcontainer"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/settings"
+	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/user"
+	esvolume "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/volume"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/utils/pointer"
 )
@@ -241,11 +244,21 @@ func TestBuildPodTemplateSpec(t *testing.T) {
 	initContainers, err := initcontainer.NewInitContainers(transportCertificatesVolume(sampleES.Name), nil, nil)
 	require.NoError(t, err)
 	// init containers should be patched with volume and inherited env vars and image
-	headlessSvcEnvVar := corev1.EnvVar{Name: "HEADLESS_SERVICE_NAME", Value: "name-es-nodeset-1"}
+	// init container env vars come in a slightly different order than main container ones which is an artefact of how the pod template builder works
+	initContainerEnv := defaults.ExtendPodDownwardEnvVars(
+		[]corev1.EnvVar{
+			{Name: "my-env", Value: "my-value"},
+			{Name: settings.EnvProbePasswordPath, Value: path.Join(esvolume.ProbeUserSecretMountPath, user.ProbeUserName)},
+			{Name: settings.EnvProbeUsername, Value: user.ProbeUserName},
+			{Name: settings.EnvReadinessProbeProtocol, Value: sampleES.Spec.HTTP.Protocol()},
+			{Name: settings.HeadlessServiceName, Value: HeadlessServiceName(esv1.StatefulSet(sampleES.Name, nodeSet.Name))},
+			{Name: "NSS_SDB_USE_CACHE", Value: "no"},
+		}...,
+	)
 	esDockerImage := "docker.elastic.co/elasticsearch/elasticsearch:7.2.0"
 	for i := range initContainers {
 		initContainers[i].Image = esDockerImage
-		initContainers[i].Env = append(initContainers[i].Env, headlessSvcEnvVar)
+		initContainers[i].Env = initContainerEnv
 		initContainers[i].VolumeMounts = append(initContainers[i].VolumeMounts, volumeMounts...)
 		initContainers[i].Resources = DefaultResources
 	}
@@ -288,7 +301,7 @@ func TestBuildPodTemplateSpec(t *testing.T) {
 			InitContainers: append(initContainers, corev1.Container{
 				Name:         "additional-init-container",
 				Image:        esDockerImage,
-				Env:          defaults.ExtendPodDownwardEnvVars(headlessSvcEnvVar),
+				Env:          initContainerEnv,
 				VolumeMounts: volumeMounts,
 				Resources:    DefaultResources, // inherited from main container
 			}),