[Bugfix] Fix backup flow

pkg/apis/deployment/v1/image_info.go

@@ -74,7 +74,9 @@ func (l *ImageInfoList) AddOrUpdate(info ImageInfo) {
 
 // Equal compares to ImageInfo
 func (i *ImageInfo) Equal(other *ImageInfo) bool {
-	if i == nil || other == nil {
+	if i == nil && other == nil {
+		return true
+	} else if i == nil || other == nil {
 		return false
 	} else if i == other {
 		return true

pkg/apis/deployment/v1/member_status.go

@@ -67,6 +67,8 @@ type MemberStatus struct {
 	ArangoVersion driver.Version `json:"arango-version,omitempty"`
 	// ImageId holds the members ArangoDB image ID
 	ImageID string `json:"image-id,omitempty"`
+	// Image holds image details
+	Image *ImageInfo `json:"image,omitempty"`
 }
 
 // Equal checks for equality
@@ -81,7 +83,8 @@ func (s MemberStatus) Equal(other MemberStatus) bool {
 		s.CleanoutJobID == other.CleanoutJobID &&
 		reflect.DeepEqual(s.SideCarSpecs, other.SideCarSpecs) &&
 		s.ArangoVersion == other.ArangoVersion &&
-		s.ImageID == other.ImageID
+		s.ImageID == other.ImageID &&
+		s.Image.Equal(other.Image)
 }
 
 // Age returns the duration since the creation timestamp of this member.

pkg/apis/deployment/v1/plan.go

@@ -81,6 +81,8 @@ const (
 	ActionTypeUpdateTLSSNI ActionType = "UpdateTLSSNI"
 	// ActionTypeSetCurrentImage causes status.CurrentImage to be updated to the image given in the action.
 	ActionTypeSetCurrentImage ActionType = "SetCurrentImage"
+	// ActionTypeSetCurrentImage replace image of member to current one.
+	ActionTypeSetMemberCurrentImage ActionType = "SetMemberCurrentImage"
 	// ActionTypeDisableClusterScaling turns off scaling DBservers and coordinators
 	ActionTypeDisableClusterScaling ActionType = "ScalingDisabled"
 	// ActionTypeEnableClusterScaling turns on scaling DBservers and coordinators

pkg/deployment/deployment_run_test.go

@@ -152,6 +152,9 @@ func runTestCase(t *testing.T, testCase testCaseStruct) {
 					require.Equal(t, false, exist)
 					_, exist = m.Conditions.Get(api.ConditionTypeAutoUpgrade)
 					require.Equal(t, false, exist)
+
+					require.NotNil(t, m.Image)
+					require.True(t, m.Image.Equal(d.apiObject.Status.CurrentImage))
 				}
 				return nil
 			}

pkg/deployment/deployment_suite_test.go

@@ -592,6 +592,13 @@ func (testCase *testCaseStruct) createTestPodData(deployment *Deployment, group
 
 	groupSpec := testCase.ArangoDeployment.Spec.GetServerGroupSpec(group)
 	testCase.ExpectedPod.Spec.Tolerations = deployment.resources.CreatePodTolerations(group, groupSpec)
+
+	// Add image info
+	if member, group, ok := deployment.apiObject.Status.Members.ElementByID(memberStatus.ID); ok {
+		member.Image = deployment.apiObject.Status.CurrentImage
+
+		deployment.apiObject.Status.Members.Update(member, group)
+	}
 }
 
 func testCreateExporterContainerWithPortAndSecureEndpoint(secure, exporterSecure bool, resources core.ResourceRequirements, port uint16) core.Container {

pkg/deployment/members.go

@@ -95,6 +95,7 @@ func createMember(log zerolog.Logger, status *api.DeploymentStatus, group api.Se
 			Phase:                     api.MemberPhaseNone,
 			PersistentVolumeClaimName: k8sutil.CreatePersistentVolumeClaimName(deploymentName, role, id),
 			PodName:                   "",
+			Image:                     apiObject.Status.CurrentImage,
 		}, group); err != nil {
 			return "", maskAny(err)
 		}
@@ -106,6 +107,7 @@ func createMember(log zerolog.Logger, status *api.DeploymentStatus, group api.Se
 			Phase:                     api.MemberPhaseNone,
 			PersistentVolumeClaimName: k8sutil.CreatePersistentVolumeClaimName(deploymentName, role, id),
 			PodName:                   "",
+			Image:                     apiObject.Status.CurrentImage,
 		}, group); err != nil {
 			return "", maskAny(err)
 		}
@@ -117,6 +119,7 @@ func createMember(log zerolog.Logger, status *api.DeploymentStatus, group api.Se
 			Phase:                     api.MemberPhaseNone,
 			PersistentVolumeClaimName: k8sutil.CreatePersistentVolumeClaimName(deploymentName, role, id),
 			PodName:                   "",
+			Image:                     apiObject.Status.CurrentImage,
 		}, group); err != nil {
 			return "", maskAny(err)
 		}
@@ -128,6 +131,7 @@ func createMember(log zerolog.Logger, status *api.DeploymentStatus, group api.Se
 			Phase:                     api.MemberPhaseNone,
 			PersistentVolumeClaimName: "",
 			PodName:                   "",
+			Image:                     apiObject.Status.CurrentImage,
 		}, group); err != nil {
 			return "", maskAny(err)
 		}
@@ -139,6 +143,7 @@ func createMember(log zerolog.Logger, status *api.DeploymentStatus, group api.Se
 			Phase:                     api.MemberPhaseNone,
 			PersistentVolumeClaimName: "",
 			PodName:                   "",
+			Image:                     apiObject.Status.CurrentImage,
 		}, group); err != nil {
 			return "", maskAny(err)
 		}
@@ -150,6 +155,7 @@ func createMember(log zerolog.Logger, status *api.DeploymentStatus, group api.Se
 			Phase:                     api.MemberPhaseNone,
 			PersistentVolumeClaimName: "",
 			PodName:                   "",
+			Image:                     apiObject.Status.CurrentImage,
 		}, group); err != nil {
 			return "", maskAny(err)
 		}

pkg/deployment/reconcile/action_context.go

@@ -399,12 +399,13 @@ func (ac *actionContext) GetCurrentImageInfo() (api.ImageInfo, bool) {
 // SetCurrentImage changes the CurrentImage field in the deployment
 // status to the given image.
 func (ac *actionContext) SetCurrentImage(imageInfo api.ImageInfo) error {
-	status, lastVersion := ac.context.GetStatus()
-	status.CurrentImage = &imageInfo
-	if err := ac.context.UpdateStatus(status, lastVersion); err != nil {
-		return maskAny(err)
-	}
-	return nil
+	return ac.context.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
+		if s.CurrentImage == nil || s.CurrentImage.Image != imageInfo.Image {
+			s.CurrentImage = &imageInfo
+			return true
+		}
+		return false
+	}, true)
 }
 
 // InvalidateSyncStatus resets the sync state to false and triggers an inspection

pkg/deployment/reconcile/action_set_current_image.go

@@ -0,0 +1,95 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeSetMemberCurrentImage, newSetCurrentMemberImageAction)
+}
+
+// newSetCurrentImageAction creates a new Action that implements the given
+// planned SetCurrentImage action.
+func newSetCurrentMemberImageAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &setCurrentMemberImageAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, upgradeMemberTimeout)
+
+	return a
+}
+
+// setCurrentImageAction implements an SetCurrentImage.
+type setCurrentMemberImageAction struct {
+	// actionImpl implement timeout and member id functions
+	actionImpl
+}
+
+// Start performs the start of the action.
+// Returns true if the action is completely finished, false in case
+// the start time needs to be recorded and a ready condition needs to be checked.
+func (a *setCurrentMemberImageAction) Start(ctx context.Context) (bool, error) {
+	ready, _, err := a.CheckProgress(ctx)
+	if err != nil {
+		return false, maskAny(err)
+	}
+	return ready, nil
+}
+
+// CheckProgress checks the progress of the action.
+// Returns true if the action is completely finished, false otherwise.
+func (a *setCurrentMemberImageAction) CheckProgress(ctx context.Context) (bool, bool, error) {
+	log := a.log
+
+	imageInfo, found := a.actionCtx.GetImageInfo(a.action.Image)
+	if !found {
+		log.Info().Msgf("Image not found")
+		return true, false, nil
+	}
+
+	if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
+		m, g, found := s.Members.ElementByID(a.action.MemberID)
+		if !found {
+			log.Error().Msg("No such member")
+			return false
+		}
+
+		m.Image = &imageInfo
+
+		if err := s.Members.Update(m, g); err != nil {
+			log.Error().Msg("Member update failed")
+			return false
+		}
+
+		return true
+	}); err != nil {
+		log.Error().Msg("Member failed")
+		return true, false, nil
+	}
+
+	return true, false, nil
+}

pkg/deployment/reconcile/action_upgrade_current_image.go

@@ -72,6 +72,6 @@ func (a *setCurrentImageAction) CheckProgress(ctx context.Context) (bool, bool,
 	if err := a.actionCtx.SetCurrentImage(imageInfo); err != nil {
 		return false, false, maskAny(err)
 	}
-	log.Info().Str("image", a.action.Image).Msg("Changed current image")
+	log.Info().Str("image", a.action.Image).Str("to", imageInfo.Image).Msg("Changed current main image")
 	return true, false, nil
 }

pkg/deployment/reconcile/plan_builder_rotate_upgrade.go

@@ -260,6 +260,10 @@ func podNeedsRotation(log zerolog.Logger, p *core.Pod, apiObject metav1.Object,
 		return false, ""
 	}
 
+	if m.Image != nil {
+		imageInfo = *m.Image
+	}
+
 	renderedPod, err := context.RenderPodForMember(cachedStatus, spec, status, m.ID, imageInfo)
 	if err != nil {
 		log.Err(err).Msg("Error while rendering pod")
@@ -302,14 +306,20 @@ func createUpgradeMemberPlan(log zerolog.Logger, member api.MemberStatus,
 		Str("reason", reason).
 		Str("action", string(upgradeAction)).
 		Msg("Creating upgrade plan")
-	plan := api.Plan{
-		api.NewAction(upgradeAction, group, member.ID, reason),
-		api.NewAction(api.ActionTypeWaitForMemberUp, group, member.ID),
-	}
+	var plan api.Plan
 	if status.CurrentImage == nil || status.CurrentImage.Image != imageName {
-		plan = append(api.Plan{
+		plan = append(plan,
 			api.NewAction(api.ActionTypeSetCurrentImage, group, "", reason).SetImage(imageName),
-		}, plan...)
+		)
+	}
+	if member.Image == nil || member.Image.Image != imageName {
+		plan = append(plan,
+			api.NewAction(api.ActionTypeSetMemberCurrentImage, group, member.ID, reason).SetImage(imageName),
+		)
 	}
+	plan = append(plan,
+		api.NewAction(upgradeAction, group, member.ID, reason),
+		api.NewAction(api.ActionTypeWaitForMemberUp, group, member.ID),
+	)
 	return plan
 }

pkg/deployment/resources/pod_creator.go

@@ -416,7 +416,21 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string,
 		})
 		return nil
 	}
-	status.CurrentImage = &imageInfo
+
+	if status.CurrentImage == nil {
+		status.CurrentImage = &imageInfo
+	}
+
+	m, group, found := status.Members.ElementByID(memberID)
+	if m.Image == nil {
+		m.Image = status.CurrentImage
+
+		if err := status.Members.Update(m, group); err != nil {
+			return maskAny(err)
+		}
+	}
+
+	imageInfo = *m.Image
 
 	pod, err := r.RenderPodForMember(cachedStatus, spec, status, memberID, imageInfo)
 	if err != nil {
@@ -427,7 +441,6 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string,
 	apiObject := r.context.GetAPIObject()
 	ns := r.context.GetNamespace()
 	secrets := kubecli.CoreV1().Secrets(ns)
-	m, group, found := status.Members.ElementByID(memberID)
 	if !found {
 		return maskAny(fmt.Errorf("Member '%s' not found", memberID))
 	}
@@ -464,6 +477,11 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string,
 		}
 
 		log.Debug().Str("pod-name", m.PodName).Msg("Created pod")
+		if m.Image == nil {
+			log.Debug().Str("pod-name", m.PodName).Msg("Created pod with default image")
+		} else {
+			log.Debug().Str("pod-name", m.PodName).Msg("Created pod with predefined image")
+		}
 	} else if group.IsArangosync() {
 		// Check monitoring token secret
 		if group == api.ServerGroupSyncMasters {