add an init container to stop the pod with bad revision

bindata/etcd/pod.yaml

@@ -10,6 +10,29 @@ metadata:
     revision: "REVISION"
 spec:
   initContainers:
+    - name: etcd-ensure-env-vars
+      image: ${IMAGE}
+      imagePullPolicy: IfNotPresent
+      terminationMessagePolicy: FallbackToLogsOnError
+      command:
+        - /bin/sh
+        - -c
+        - |
+          #!/bin/sh
+          set -euo pipefail
+
+          : "${NODE_NODE_ENVVAR_NAME_ETCD_URL_HOST?not set}"
+          : "${NODE_NODE_ENVVAR_NAME_ETCD_NAME?not set}"
+          : "${NODE_NODE_ENVVAR_NAME_IP?not set}"
+
+      resources:
+        requests:
+          memory: 60Mi
+          cpu: 30m
+      securityContext:
+        privileged: true
+      env:
+${COMPUTED_ENV_VARS}
     - name: etcd-resources-copy
       image: ${IMAGE}
       imagePullPolicy: IfNotPresent

pkg/operator/clustermembercontroller/clustermembercontroller.go

@@ -220,6 +220,14 @@ func (c *ClusterMemberController) getEtcdPodToAddToMembership() (*corev1.Pod, er
 			continue
 		}
 
+		if (pod.Status.InitContainerStatuses[1].State.Terminated == nil) ||
+			(pod.Status.InitContainerStatuses[1].State.Terminated != nil &&
+				pod.Status.InitContainerStatuses[1].State.Terminated.ExitCode != 0) {
+			// if pod is not initialized we dont know the env is correct,
+			// we will mark it ready and continue
+			continue
+		}
+
 		// check to see if this member is updating from 4.3
 		etcdMember, err := c.etcdClient.GetMember("etcd-member-" + pod.Spec.NodeName)
 		switch {

pkg/operator/clustermembercontroller/clustermembercontroller_test.go

@@ -69,6 +69,24 @@ func TestClusterMemberController_getEtcdPodToAddToMembership(t *testing.T) {
 					},
 					Status: corev1.PodStatus{
 						Phase: "Running",
+						InitContainerStatuses: []corev1.ContainerStatus{
+							{
+								Name: "etcd-ensure-env",
+								State: corev1.ContainerState{
+									Terminated: &corev1.ContainerStateTerminated{
+										ExitCode: 0,
+									},
+								},
+							},
+							{
+								Name: "etcd-resources-copy",
+								State: corev1.ContainerState{
+									Terminated: &corev1.ContainerStateTerminated{
+										ExitCode: 0,
+									},
+								},
+							},
+						},
 						ContainerStatuses: []corev1.ContainerStatus{
 							{
 								Name:  "etcd-member",
@@ -88,6 +106,24 @@ func TestClusterMemberController_getEtcdPodToAddToMembership(t *testing.T) {
 						},
 						Status: corev1.PodStatus{
 							Phase: "Running",
+							InitContainerStatuses: []corev1.ContainerStatus{
+								{
+									Name: "etcd-ensure-env",
+									State: corev1.ContainerState{
+										Terminated: &corev1.ContainerStateTerminated{
+											ExitCode: 0,
+										},
+									},
+								},
+								{
+									Name: "etcd-resources-copy",
+									State: corev1.ContainerState{
+										Terminated: &corev1.ContainerStateTerminated{
+											ExitCode: 0,
+										},
+									},
+								},
+							},
 							ContainerStatuses: []corev1.ContainerStatus{
 								{
 									Name:  "etcd",
@@ -116,6 +152,110 @@ func TestClusterMemberController_getEtcdPodToAddToMembership(t *testing.T) {
 			},
 			want: nil,
 		},
+		{
+			name: "test pods with init container failed",
+			fields: fields{
+				etcdClient: etcdcli.NewFakeEtcdClient([]*etcdserverpb.Member{
+					{
+						Name: "etcd-member-node-a",
+					},
+					{
+						Name: "etcd-member-node-b",
+					},
+					{
+						Name: "node-c",
+					},
+				}),
+				podLister: &fakePodLister{fake.NewSimpleClientset(&corev1.Pod{
+					ObjectMeta: metav1.ObjectMeta{
+						// this will be skipped
+						Name:      "etcd-member-node-a",
+						Namespace: "openshift-etcd",
+						Labels:    labels.Set{"app": "etcd"},
+					},
+					Status: corev1.PodStatus{
+						Phase: "Running",
+						InitContainerStatuses: []corev1.ContainerStatus{
+							{
+								Name: "etcd-ensure-env",
+								State: corev1.ContainerState{
+									Terminated: &corev1.ContainerStateTerminated{
+										ExitCode: 0,
+									},
+								},
+							},
+							{
+								Name: "etcd-resources-copy",
+								State: corev1.ContainerState{
+									Terminated: &corev1.ContainerStateTerminated{
+										ExitCode: 0,
+									},
+								},
+							},
+						},
+						ContainerStatuses: []corev1.ContainerStatus{
+							{
+								Name:  "etcd-member",
+								Ready: true,
+							},
+						},
+					},
+				},
+					&corev1.Pod{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "etcd-node-b",
+							Namespace: "openshift-etcd",
+							Labels:    labels.Set{"app": "etcd"},
+						},
+						Spec: corev1.PodSpec{
+							NodeName: "node-b",
+						},
+						Status: corev1.PodStatus{
+							Phase: "Running",
+							InitContainerStatuses: []corev1.ContainerStatus{
+								{
+									Name: "etcd-ensure-env",
+									State: corev1.ContainerState{
+										Terminated: &corev1.ContainerStateTerminated{
+											ExitCode: 1,
+										},
+									},
+								},
+								{
+									Name: "etcd-resources-copy",
+									State: corev1.ContainerState{
+										Terminated: nil,
+									},
+								},
+							},
+							ContainerStatuses: []corev1.ContainerStatus{
+								{
+									Name:  "etcd",
+									Ready: true,
+								},
+							},
+						},
+					},
+					&corev1.Pod{
+						ObjectMeta: metav1.ObjectMeta{
+							// this will be skipped
+							Name:      "etcd-node-c",
+							Namespace: "openshift-etcd",
+							Labels:    labels.Set{"app": "etcd"},
+						},
+						Status: corev1.PodStatus{
+							Phase: "Running",
+							ContainerStatuses: []corev1.ContainerStatus{
+								{
+									Name:  "etcd",
+									Ready: true,
+								},
+							},
+						},
+					}), "openshift-etcd"},
+			},
+			want: nil,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {