Check contents of persisted volume when dbserver is restarting

pkg/apis/deployment/v1alpha/member_status.go

@@ -44,6 +44,9 @@ type MemberStatus struct {
 	// RecentTerminatons holds the times when this member was recently terminated.
 	// First entry is the oldest. (do not add omitempty, since we want to be able to switch from a list to an empty list)
 	RecentTerminations []metav1.Time `json:"recent-terminations"`
+	// IsInitialized is set after the very first time a pod was created for this member.
+	// After that, DBServers must have a UUID field or fail.
+	IsInitialized bool `json:"initialized"`
 }
 
 // RemoveTerminationsBefore removes all recent terminations before the given timestamp.

pkg/deployment/images.go

@@ -166,7 +166,7 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima
 		"--server.authentication=false",
 		fmt.Sprintf("--server.endpoint=tcp://[::]:%d", k8sutil.ArangoPort),
 	}
-	if err := k8sutil.CreateArangodPod(ib.KubeCli, true, ib.APIObject, role, id, podName, "", image, ib.Spec.GetImagePullPolicy(), args, nil, nil, nil, "", ""); err != nil {
+	if err := k8sutil.CreateArangodPod(ib.KubeCli, true, ib.APIObject, role, id, podName, "", image, ib.Spec.GetImagePullPolicy(), "", false, args, nil, nil, nil, "", ""); err != nil {
 		log.Debug().Err(err).Msg("Failed to create image ID pod")
 		return true, maskAny(err)
 	}

pkg/deployment/resources/pod_creator.go

@@ -366,7 +366,10 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server
 				SecretKey:  constants.SecretKeyJWT,
 			}
 		}
-		if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, info.ImageID, spec.GetImagePullPolicy(), args, env, livenessProbe, readinessProbe, tlsKeyfileSecretName, rocksdbEncryptionSecretName); err != nil {
+		engine := string(spec.GetStorageEngine())
+		requireUUID := group == api.ServerGroupDBServers && m.IsInitialized
+		if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, info.ImageID, spec.GetImagePullPolicy(),
+			engine, requireUUID, args, env, livenessProbe, readinessProbe, tlsKeyfileSecretName, rocksdbEncryptionSecretName); err != nil {
 			return maskAny(err)
 		}
 		log.Debug().Str("pod-name", m.PodName).Msg("Created pod")

pkg/deployment/resources/pod_inspector.go

@@ -100,6 +100,7 @@ func (r *Resources) InspectPods() error {
 			// Pod is now ready
 			if memberStatus.Conditions.Update(api.ConditionTypeReady, true, "Pod Ready", "") {
 				log.Debug().Str("pod-name", p.GetName()).Msg("Updating member condition Ready to true")
+				memberStatus.IsInitialized = true // Require future pods for this member to have an existing UUID (in case of dbserver).
 				updateMemberStatusNeeded = true
 			}
 		} else {

pkg/util/k8sutil/pods.go

@@ -25,6 +25,7 @@ package k8sutil
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"k8s.io/api/core/v1"
@@ -170,13 +171,28 @@ func rocksdbEncryptionVolumeMounts() []v1.VolumeMount {
 
 // arangodInitContainer creates a container configured to
 // initalize a UUID file.
-func arangodInitContainer(name, id string) v1.Container {
+func arangodInitContainer(name, id, engine string, requireUUID bool) v1.Container {
 	uuidFile := filepath.Join(ArangodVolumeMountDir, "UUID")
+	engineFile := filepath.Join(ArangodVolumeMountDir, "ENGINE")
+	var command string
+	if requireUUID {
+		command = strings.Join([]string{
+			// Files must exist
+			fmt.Sprintf("test -f %s", uuidFile),
+			fmt.Sprintf("test -f %s", engineFile),
+			// Content must match
+			fmt.Sprintf("grep -q %s %s", id, uuidFile),
+			fmt.Sprintf("grep -q %s %s", engine, engineFile),
+		}, " && ")
+
+	} else {
+		command = fmt.Sprintf("test -f %s || echo '%s' > %s", uuidFile, id, uuidFile)
+	}
 	c := v1.Container{
 		Command: []string{
 			"/bin/sh",
 			"-c",
-			fmt.Sprintf("test -f %s || echo '%s' > %s", uuidFile, id, uuidFile),
+			command,
 		},
 		Name:         name,
 		Image:        alpineImage,
@@ -261,6 +277,7 @@ func newPod(deploymentName, ns, role, id, podName string) v1.Pod {
 // If another error occurs, that error is returned.
 func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject,
 	role, id, podName, pvcName, image string, imagePullPolicy v1.PullPolicy,
+	engine string, requireUUID bool,
 	args []string, env map[string]EnvValue,
 	livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig,
 	tlsKeyfileSecretName, rocksdbEncryptionSecretName string) error {
@@ -278,7 +295,7 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy
 	p.Spec.Containers = append(p.Spec.Containers, c)
 
 	// Add UUID init container
-	p.Spec.InitContainers = append(p.Spec.InitContainers, arangodInitContainer("uuid", id))
+	p.Spec.InitContainers = append(p.Spec.InitContainers, arangodInitContainer("uuid", id, engine, requireUUID))
 
 	// Add volume
 	if pvcName != "" {