diff --git a/api/core/v1alpha2/vdscondition/condition.go b/api/core/v1alpha2/vdscondition/condition.go index 26506bd3f9..20937d6104 100644 --- a/api/core/v1alpha2/vdscondition/condition.go +++ b/api/core/v1alpha2/vdscondition/condition.go @@ -63,6 +63,8 @@ const ( VolumeSnapshotLost VirtualDiskSnapshotReadyReason = "Lost" // FileSystemFreezing signifies that the `VirtualDiskSnapshot` resource is in the process of freezing the filesystem of the virtual machine associated with the source virtual disk. FileSystemFreezing VirtualDiskSnapshotReadyReason = "FileSystemFreezing" + // FileSystemUnfreezing signifies that the `VirtualDiskSnapshot` resource is in the process of unfreezing the filesystem of the virtual machine associated with the source virtual disk. + FileSystemUnfreezing VirtualDiskSnapshotReadyReason = "FileSystemUnfreezing" // Snapshotting signifies that the `VirtualDiskSnapshot` resource is in the process of taking a snapshot of the virtual disk. Snapshotting VirtualDiskSnapshotReadyReason = "Snapshotting" // VirtualDiskSnapshotReady signifies that the snapshot process is complete and the `VirtualDiskSnapshot` is ready for use. diff --git a/api/core/v1alpha2/vmscondition/condition.go b/api/core/v1alpha2/vmscondition/condition.go index afcfb2c684..d28ef154e2 100644 --- a/api/core/v1alpha2/vmscondition/condition.go +++ b/api/core/v1alpha2/vmscondition/condition.go @@ -49,6 +49,8 @@ const ( VirtualDiskSnapshotLost VirtualMachineSnapshotReadyReason = "VirtualDiskSnapshotLost" // FileSystemFreezing signifies that the `VirtualMachineSnapshot` resource is in the process of freezing the filesystem of the virtual machine. FileSystemFreezing VirtualMachineSnapshotReadyReason = "FileSystemFreezing" + // FileSystemUnfreezing signifies that the `VirtualMachineSnapshot` resource is in the process of unfreezing the filesystem of the virtual machine. + FileSystemUnfreezing VirtualMachineSnapshotReadyReason = "FileSystemUnfreezing" // Snapshotting signifies that the `VirtualMachineSnapshot` resource is in the process of taking a snapshot of the virtual machine. Snapshotting VirtualMachineSnapshotReadyReason = "Snapshotting" // VirtualMachineSnapshotReady signifies that the snapshot process is complete and the `VirtualMachineSnapshot` is ready for use. diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index ba7453b2a8..a6864b5dfb 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -187,6 +187,9 @@ const ( AnnVMOPUID = AnnAPIGroupV + "/vmop-uid" // AnnVMOPSnapshotName is an annotation on vmop that represents name a snapshot created for VMOP. AnnVMOPSnapshotName = AnnAPIGroupV + "/vmop-snapshot-name" + + // AnnVMFilesystemRequest is an annotation on a virtual machine that indicates a request to freeze or unfreeze the filesystem has been sent. + AnnVMFilesystemRequest = AnnAPIGroupV + "/virtual-machine-filesystem-request" ) // AddAnnotation adds an annotation to an object diff --git a/images/virtualization-artifact/pkg/controller/service/snapshot_service.go b/images/virtualization-artifact/pkg/controller/service/snapshot_service.go index bcc45dce48..9253fed996 100644 --- a/images/virtualization-artifact/pkg/controller/service/snapshot_service.go +++ b/images/virtualization-artifact/pkg/controller/service/snapshot_service.go @@ -18,23 +18,34 @@ package service import ( "context" + "errors" "fmt" vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/common/object" - "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization/api/client/kubeclient" "github.com/deckhouse/virtualization/api/core/v1alpha2" - "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" subv1alpha2 "github.com/deckhouse/virtualization/api/subresources/v1alpha2" ) +const ( + RequestFSFreeze = "freeze" + RequestFSUnfreeze = "unfreeze" + FSFrozen = "frozen" +) + +var ( + ErrUntrustedFilesystemFrozenCondition = errors.New("the filesystem status is not synced") + ErrUnexpectedFilesystemRequest = errors.New("found unexpected filesystem request in the virtual machine instance annotations") +) + type SnapshotService struct { virtClient kubeclient.Client client Client @@ -49,37 +60,75 @@ func NewSnapshotService(virtClient kubeclient.Client, client Client, protection } } -func (s *SnapshotService) IsFrozen(vm *v1alpha2.VirtualMachine) bool { - if vm == nil { - return false +// IsFrozen checks if a freeze or unfreeze request has been performed +// and returns the "true" fsFreezeStatus if the internal virtual machine instance is "frozen", +// and "false" otherwise. +func (s *SnapshotService) IsFrozen(kvvmi *virtv1.VirtualMachineInstance) (bool, error) { + if kvvmi == nil { + return false, nil } - filesystemFrozen, _ := conditions.GetCondition(vmcondition.TypeFilesystemFrozen, vm.Status.Conditions) + if request, ok := kvvmi.Annotations[annotations.AnnVMFilesystemRequest]; ok { + return false, fmt.Errorf("failed to check %s/%s fsFreezeStatus: request: %s: %w", kvvmi.Namespace, kvvmi.Name, request, ErrUntrustedFilesystemFrozenCondition) + } - return filesystemFrozen.Status == metav1.ConditionTrue && filesystemFrozen.Reason == vmcondition.ReasonFilesystemFrozen.String() + return kvvmi.Status.FSFreezeStatus == FSFrozen, nil } -func (s *SnapshotService) CanFreeze(vm *v1alpha2.VirtualMachine) bool { - if vm == nil || vm.Status.Phase != v1alpha2.MachineRunning || s.IsFrozen(vm) { - return false +func (s *SnapshotService) CanFreeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { + if kvvmi == nil || kvvmi.Status.Phase != virtv1.Running { + return false, nil + } + + isFrozen, err := s.IsFrozen(kvvmi) + if err != nil { + return false, err + } + if isFrozen { + return false, nil } - agentReady, _ := conditions.GetCondition(vmcondition.TypeAgentReady, vm.Status.Conditions) + for _, c := range kvvmi.Status.Conditions { + if c.Type == virtv1.VirtualMachineInstanceAgentConnected { + return c.Status == corev1.ConditionTrue, nil + } + } - return agentReady.Status == metav1.ConditionTrue + return false, nil } -func (s *SnapshotService) Freeze(ctx context.Context, name, namespace string) error { - err := s.virtClient.VirtualMachines(namespace).Freeze(ctx, name, subv1alpha2.VirtualMachineFreeze{}) +// TODO: The Freeze method should be atomic because there is a chance of encountering +// a dead-end with the `ErrUntrustedFilesystemFrozenCondition` error in the SyncFSFreezeRequest method +// when the API returns an error on an freeze request. +func (s *SnapshotService) Freeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { + if request, ok := kvvmi.Annotations[annotations.AnnVMFilesystemRequest]; ok { + return fmt.Errorf("failed to freeze %s/%s virtual machine filesystem: %w: request type: %s", kvvmi.Namespace, kvvmi.Name, ErrUnexpectedFilesystemRequest, request) + } + + err := s.annotateWithFSFreezeRequest(ctx, RequestFSFreeze, kvvmi) if err != nil { - return fmt.Errorf("failed to freeze virtual machine %s/%s: %w", namespace, name, err) + return fmt.Errorf("failed to annotate internal virtual machine instance with filesystem freeze request: %w", err) + } + + err = s.virtClient.VirtualMachines(kvvmi.Namespace).Freeze(ctx, kvvmi.Name, subv1alpha2.VirtualMachineFreeze{}) + if err != nil { + return fmt.Errorf("failed to freeze %s/%s virtual machine filesystem: %w", kvvmi.Namespace, kvvmi.Name, err) } return nil } -func (s *SnapshotService) CanUnfreezeWithVirtualDiskSnapshot(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine) (bool, error) { - if vm == nil || !s.IsFrozen(vm) { +func (s *SnapshotService) CanUnfreezeWithVirtualDiskSnapshot(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { + if vm == nil { + return false, nil + } + + isFrozen, err := s.IsFrozen(kvvmi) + if err != nil { + return false, err + } + + if !isFrozen { return false, nil } @@ -91,7 +140,7 @@ func (s *SnapshotService) CanUnfreezeWithVirtualDiskSnapshot(ctx context.Context } var vdSnapshots v1alpha2.VirtualDiskSnapshotList - err := s.client.List(ctx, &vdSnapshots, &client.ListOptions{ + err = s.client.List(ctx, &vdSnapshots, &client.ListOptions{ Namespace: vm.Namespace, }) if err != nil { @@ -126,8 +175,16 @@ func (s *SnapshotService) CanUnfreezeWithVirtualDiskSnapshot(ctx context.Context return true, nil } -func (s *SnapshotService) CanUnfreezeWithVirtualMachineSnapshot(ctx context.Context, vmSnapshotName string, vm *v1alpha2.VirtualMachine) (bool, error) { - if vm == nil || !s.IsFrozen(vm) { +func (s *SnapshotService) CanUnfreezeWithVirtualMachineSnapshot(ctx context.Context, vmSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { + if vm == nil { + return false, nil + } + + isFrozen, err := s.IsFrozen(kvvmi) + if err != nil { + return false, err + } + if !isFrozen { return false, nil } @@ -139,7 +196,7 @@ func (s *SnapshotService) CanUnfreezeWithVirtualMachineSnapshot(ctx context.Cont } var vdSnapshots v1alpha2.VirtualDiskSnapshotList - err := s.client.List(ctx, &vdSnapshots, &client.ListOptions{ + err = s.client.List(ctx, &vdSnapshots, &client.ListOptions{ Namespace: vm.Namespace, }) if err != nil { @@ -174,10 +231,22 @@ func (s *SnapshotService) CanUnfreezeWithVirtualMachineSnapshot(ctx context.Cont return true, nil } -func (s *SnapshotService) Unfreeze(ctx context.Context, name, namespace string) error { - err := s.virtClient.VirtualMachines(namespace).Unfreeze(ctx, name) +// TODO: The Unfreeze method should be atomic because there is a chance of encountering +// a dead-end with the `ErrUntrustedFilesystemFrozenCondition` error in the SyncFSFreezeRequest method +// when the API returns an error on an unfreeze request. +func (s *SnapshotService) Unfreeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { + if request, ok := kvvmi.Annotations[annotations.AnnVMFilesystemRequest]; ok { + return fmt.Errorf("failed to unfreeze %s/%s virtual machine filesystem: %w: request type: %s", kvvmi.Namespace, kvvmi.Name, ErrUnexpectedFilesystemRequest, request) + } + + err := s.annotateWithFSFreezeRequest(ctx, RequestFSUnfreeze, kvvmi) if err != nil { - return fmt.Errorf("unfreeze virtual machine %s/%s: %w", namespace, name, err) + return fmt.Errorf("failed to annotate internal virtual machine instance with filesystem unfreeze request: %w", err) + } + + err = s.virtClient.VirtualMachines(kvvmi.Namespace).Unfreeze(ctx, kvvmi.Name) + if err != nil { + return fmt.Errorf("unfreeze virtual machine %s/%s: %w", kvvmi.Namespace, kvvmi.Name, err) } return nil @@ -243,3 +312,75 @@ func (s *SnapshotService) CreateVirtualDiskSnapshot(ctx context.Context, vdSnaps return vdSnapshot, nil } + +func (s *SnapshotService) GetVirtualMachineInstance(ctx context.Context, vm *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) { + if vm == nil { + return nil, nil + } + return object.FetchObject(ctx, client.ObjectKeyFromObject(vm), s.client, &virtv1.VirtualMachineInstance{}) +} + +func (s *SnapshotService) annotateWithFSFreezeRequest(ctx context.Context, requestType string, kvvmi *virtv1.VirtualMachineInstance) error { + if kvvmi == nil { + return fmt.Errorf("failed to annotate virtual machine instance; virtual machine instance cannot be nil") + } + + if kvvmi.Annotations == nil { + kvvmi.Annotations = make(map[string]string) + } + kvvmi.Annotations[annotations.AnnVMFilesystemRequest] = requestType + + err := s.client.Update(ctx, kvvmi) + if err != nil { + return err + } + + return nil +} + +func (s *SnapshotService) removeAnnFSFreezeRequest(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { + if kvvmi == nil { + return fmt.Errorf("failed to annotate virtual machine instance; virtual machine instance cannot be nil") + } + + if kvvmi.Annotations == nil { + return nil + } + + delete(kvvmi.Annotations, annotations.AnnVMFilesystemRequest) + + err := s.client.Update(ctx, kvvmi) + if err != nil { + return err + } + + return nil +} + +func (s *SnapshotService) SyncFSFreezeRequest(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { + if kvvmi == nil { + return nil + } + + request, ok := kvvmi.Annotations[annotations.AnnVMFilesystemRequest] + if !ok { + return nil + } + + switch { + case request == RequestFSFreeze && kvvmi.Status.FSFreezeStatus == FSFrozen: + err := s.removeAnnFSFreezeRequest(ctx, kvvmi) + if err != nil { + return fmt.Errorf("failed to sync kvvmi %s/%s fsFreezeStatus: request: %s: %w", kvvmi.Namespace, kvvmi.Name, request, err) + } + return nil + case request == RequestFSUnfreeze && kvvmi.Status.FSFreezeStatus != FSFrozen: + err := s.removeAnnFSFreezeRequest(ctx, kvvmi) + if err != nil { + return fmt.Errorf("failed to sync kvvmi %s/%s fsFreezeStatus: request: %s: %w", kvvmi.Namespace, kvvmi.Name, request, err) + } + return nil + default: + return fmt.Errorf("failed to sync kvvmi %s/%s fsFreezeStatus: request: %s: %w", kvvmi.Namespace, kvvmi.Name, request, ErrUntrustedFilesystemFrozenCondition) + } +} diff --git a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/deletion.go b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/deletion.go index 9731011350..b0eb8264ae 100644 --- a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/deletion.go +++ b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/deletion.go @@ -18,7 +18,10 @@ package internal import ( "context" + "errors" + "time" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -61,6 +64,11 @@ func (h DeletionHandler) Handle(ctx context.Context, vdSnapshot *v1alpha2.Virtua } } + kvvmi, err := h.snapshotter.GetVirtualMachineInstance(ctx, vm) + if err != nil { + return reconcile.Result{}, err + } + if vs != nil { err = h.snapshotter.DeleteVolumeSnapshot(ctx, vs) if err != nil { @@ -70,14 +78,20 @@ func (h DeletionHandler) Handle(ctx context.Context, vdSnapshot *v1alpha2.Virtua if vm != nil { var canUnfreeze bool - canUnfreeze, err = h.snapshotter.CanUnfreezeWithVirtualDiskSnapshot(ctx, vdSnapshot.Name, vm) + canUnfreeze, err = h.snapshotter.CanUnfreezeWithVirtualDiskSnapshot(ctx, vdSnapshot.Name, vm, kvvmi) if err != nil { + if errors.Is(err, service.ErrUntrustedFilesystemFrozenCondition) { + return reconcile.Result{}, nil + } return reconcile.Result{}, err } if canUnfreeze { - err = h.snapshotter.Unfreeze(ctx, vm.Name, vm.Namespace) + err = h.snapshotter.Unfreeze(ctx, kvvmi) if err != nil { + if k8serrors.IsConflict(err) { + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + } return reconcile.Result{}, err } } diff --git a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/interfaces.go b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/interfaces.go index 845deafc10..c792c96684 100644 --- a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/interfaces.go +++ b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/interfaces.go @@ -21,6 +21,7 @@ import ( vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" corev1 "k8s.io/api/core/v1" + virtv1 "kubevirt.io/api/core/v1" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -32,14 +33,16 @@ type VirtualDiskReadySnapshotter interface { } type LifeCycleSnapshotter interface { - Freeze(ctx context.Context, name, namespace string) error - IsFrozen(vm *v1alpha2.VirtualMachine) bool - CanFreeze(vm *v1alpha2.VirtualMachine) bool - CanUnfreezeWithVirtualDiskSnapshot(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine) (bool, error) - Unfreeze(ctx context.Context, name, namespace string) error + Freeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error + IsFrozen(kvvmi *virtv1.VirtualMachineInstance) (bool, error) + CanFreeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) + CanUnfreezeWithVirtualDiskSnapshot(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) + Unfreeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error CreateVolumeSnapshot(ctx context.Context, vs *vsv1.VolumeSnapshot) (*vsv1.VolumeSnapshot, error) GetPersistentVolumeClaim(ctx context.Context, name, namespace string) (*corev1.PersistentVolumeClaim, error) GetVirtualDisk(ctx context.Context, name, namespace string) (*v1alpha2.VirtualDisk, error) GetVirtualMachine(ctx context.Context, name, namespace string) (*v1alpha2.VirtualMachine, error) GetVolumeSnapshot(ctx context.Context, name, namespace string) (*vsv1.VolumeSnapshot, error) + SyncFSFreezeRequest(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error + GetVirtualMachineInstance(ctx context.Context, vm *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) } diff --git a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/life_cycle.go index 1d6f6ab8fc..8846dec42f 100644 --- a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/life_cycle.go @@ -19,13 +19,17 @@ package internal import ( "context" "encoding/json" + "errors" "fmt" "strings" + "time" vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/deckhouse/virtualization-controller/pkg/common/annotations" @@ -52,18 +56,7 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vdSnapshot *v1alpha2.Virtu cb := conditions.NewConditionBuilder(vdscondition.VirtualDiskSnapshotReadyType).Generation(vdSnapshot.Generation) - defer func() { - err := h.unfreezeFilesystemIfFailed(ctx, vdSnapshot) - if err != nil { - if cb.Condition().Message != "" { - cb.Message(fmt.Sprintf("%s, %s", err.Error(), cb.Condition().Message)) - } else { - cb.Message(err.Error()) - } - } - - conditions.SetCondition(cb, &vdSnapshot.Status.Conditions) - }() + defer func() { conditions.SetCondition(cb, &vdSnapshot.Status.Conditions) }() vs, err := h.snapshotter.GetVolumeSnapshot(ctx, vdSnapshot.Name, vdSnapshot.Namespace) if err != nil { @@ -71,12 +64,6 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vdSnapshot *v1alpha2.Virtu return reconcile.Result{}, err } - vd, err := h.snapshotter.GetVirtualDisk(ctx, vdSnapshot.Spec.VirtualDiskName, vdSnapshot.Namespace) - if err != nil { - setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) - return reconcile.Result{}, err - } - if vdSnapshot.DeletionTimestamp != nil { vdSnapshot.Status.Phase = v1alpha2.VirtualDiskSnapshotPhaseTerminating cb. @@ -96,7 +83,21 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vdSnapshot *v1alpha2.Virtu Status(metav1.ConditionFalse). Reason(conditions.CommonReason(readyCondition.Reason)). Message(readyCondition.Message) - return reconcile.Result{}, nil + + err = h.unfreezeFilesystemIfFailed(ctx, vdSnapshot) + switch { + case err == nil: + return reconcile.Result{}, nil + case k8serrors.IsConflict(err): + log.Debug(fmt.Sprintf("failed to unfreeze filesystem; resource update conflict error: %s", err)) + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + case errors.Is(err, service.ErrUntrustedFilesystemFrozenCondition): + log.Debug(err.Error()) + return reconcile.Result{}, nil + default: + cb.Message(fmt.Sprintf("%s, %s", err.Error(), cb.Condition().Message)) + return reconcile.Result{}, fmt.Errorf("failed to unfreeze filesystem: %w", err) + } case v1alpha2.VirtualDiskSnapshotPhaseReady: if vs == nil || vs.Status == nil || vs.Status.ReadyToUse == nil || !*vs.Status.ReadyToUse { vdSnapshot.Status.Phase = v1alpha2.VirtualDiskSnapshotPhaseFailed @@ -117,6 +118,12 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vdSnapshot *v1alpha2.Virtu return reconcile.Result{}, nil } + vd, err := h.snapshotter.GetVirtualDisk(ctx, vdSnapshot.Spec.VirtualDiskName, vdSnapshot.Namespace) + if err != nil { + setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) + return reconcile.Result{}, err + } + virtualDiskReadyCondition, _ := conditions.GetCondition(vdscondition.VirtualDiskReadyType, vdSnapshot.Status.Conditions) if vd == nil || virtualDiskReadyCondition.Status != metav1.ConditionTrue { vdSnapshot.Status.Phase = v1alpha2.VirtualDiskSnapshotPhasePending @@ -151,10 +158,51 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vdSnapshot *v1alpha2.Virtu return reconcile.Result{}, err } + kvvmi, err := h.snapshotter.GetVirtualMachineInstance(ctx, vm) + if err != nil { + setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) + return reconcile.Result{}, err + } + + err = h.snapshotter.SyncFSFreezeRequest(ctx, kvvmi) + switch { + case err == nil: + // OK. + case errors.Is(err, service.ErrUntrustedFilesystemFrozenCondition): + log.Debug(err.Error()) + cb. + Status(metav1.ConditionFalse). + Reason(vdscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{}, nil + case k8serrors.IsConflict(err): + log.Debug(fmt.Sprintf("failed to sync filesystem status; resource update conflict error: %s", err)) + cb. + Status(metav1.ConditionFalse). + Reason(vdscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + default: + setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) + return reconcile.Result{}, fmt.Errorf("failed to sync filesystem status: %w", err) + } + + isFSFrozen, err := h.snapshotter.IsFrozen(kvvmi) + if err != nil { + setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) + return reconcile.Result{}, fmt.Errorf("failed to check filesystem status: %w", err) + } + switch { case vs == nil: - if vm != nil && vm.Status.Phase != v1alpha2.MachineStopped && !h.snapshotter.IsFrozen(vm) { - if h.snapshotter.CanFreeze(vm) { + if vm != nil && vm.Status.Phase != v1alpha2.MachineStopped && !isFSFrozen { + var canFreeze bool + canFreeze, err = h.snapshotter.CanFreeze(ctx, kvvmi) + if err != nil { + setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) + return reconcile.Result{}, err + } + if canFreeze { log.Debug("Freeze the virtual machine to take a snapshot") if vdSnapshot.Status.Phase == v1alpha2.VirtualDiskSnapshotPhasePending { @@ -166,9 +214,20 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vdSnapshot *v1alpha2.Virtu return reconcile.Result{Requeue: true}, nil } - err = h.snapshotter.Freeze(ctx, vm.Name, vm.Namespace) + err = h.snapshotter.Freeze(ctx, kvvmi) if err != nil { - setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) + if k8serrors.IsConflict(err) { + log.Debug(fmt.Sprintf("failed to freeze filesystem; resource update conflict error: %s", err)) + cb. + Status(metav1.ConditionFalse). + Reason(vdscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + } + cb. + Status(metav1.ConditionFalse). + Reason(vdscondition.FileSystemFreezing). + Message(service.CapitalizeFirstLetter(err.Error() + ".")) return reconcile.Result{}, err } @@ -308,34 +367,63 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vdSnapshot *v1alpha2.Virtu default: log.Debug("The volume snapshot is ready to use") - switch { - case vm == nil, vm.Status.Phase == v1alpha2.MachineStopped: - vdSnapshot.Status.Consistent = ptr.To(true) - case h.snapshotter.IsFrozen(vm): - vdSnapshot.Status.Consistent = ptr.To(true) + if vdSnapshot.Status.Consistent == nil { + if vm == nil || vm.Status.Phase == v1alpha2.MachineStopped || isFSFrozen { + vdSnapshot.Status.Consistent = ptr.To(true) + cb. + Status(metav1.ConditionFalse). + Reason(vdscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("The consistent snapshot has been taken.")) + return reconcile.Result{RequeueAfter: 2 * time.Second}, nil + } - var canUnfreeze bool - canUnfreeze, err = h.snapshotter.CanUnfreezeWithVirtualDiskSnapshot(ctx, vdSnapshot.Name, vm) - if err != nil { + if vdSnapshot.Spec.RequiredConsistency { + err = fmt.Errorf("virtual disk snapshot is not consistent because the virtual machine %s has not been stopped or its filesystem has not been frozen", vm.Name) setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) return reconcile.Result{}, err } + } - if canUnfreeze { - log.Debug("Unfreeze the virtual machine after taking a snapshot") - - err = h.snapshotter.Unfreeze(ctx, vm.Name, vm.Namespace) - if err != nil { - setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) - return reconcile.Result{}, err - } + err = h.unfreezeFilesystem(ctx, vdSnapshot.Name, vm, kvvmi) + if err != nil { + if k8serrors.IsConflict(err) { + log.Debug(fmt.Sprintf("failed to unfreeze filesystem; resource update conflict error: %s", err)) + cb. + Status(metav1.ConditionFalse). + Reason(vdscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil } + vdSnapshot.Status.Phase = v1alpha2.VirtualDiskSnapshotPhaseInProgress + cb. + Status(metav1.ConditionFalse). + Reason(vdscondition.FileSystemUnfreezing). + Message(service.CapitalizeFirstLetter(err.Error() + ".")) + return reconcile.Result{}, err + } + + err = h.snapshotter.SyncFSFreezeRequest(ctx, kvvmi) + switch { + case err == nil: + // OK. + case errors.Is(err, service.ErrUntrustedFilesystemFrozenCondition): + log.Debug(err.Error()) + cb. + Status(metav1.ConditionFalse). + Reason(vdscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{}, nil + case k8serrors.IsConflict(err): + log.Debug(fmt.Sprintf("failed to sync filesystem status; resource update conflict error: %s", err)) + cb. + Status(metav1.ConditionFalse). + Reason(vdscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil default: - if vdSnapshot.Spec.RequiredConsistency { - err := fmt.Errorf("virtual disk snapshot is not consistent because the virtual machine %s has not been stopped or its filesystem has not been frozen", vm.Name) - setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) - return reconcile.Result{}, err - } + err = fmt.Errorf("failed to sync filesystem status: %w", err) + setPhaseConditionToFailed(cb, &vdSnapshot.Status.Phase, err) + return reconcile.Result{}, err } vdSnapshot.Status.Phase = v1alpha2.VirtualDiskSnapshotPhaseReady @@ -379,10 +467,6 @@ func setPhaseConditionToFailed(cb *conditions.ConditionBuilder, phase *v1alpha2. } func (h LifeCycleHandler) unfreezeFilesystemIfFailed(ctx context.Context, vdSnapshot *v1alpha2.VirtualDiskSnapshot) error { - if vdSnapshot.Status.Phase != v1alpha2.VirtualDiskSnapshotPhaseFailed { - return nil - } - vd, err := h.snapshotter.GetVirtualDisk(ctx, vdSnapshot.Spec.VirtualDiskName, vdSnapshot.Namespace) if err != nil { return err @@ -401,9 +485,27 @@ func (h LifeCycleHandler) unfreezeFilesystemIfFailed(ctx context.Context, vdSnap return nil } - frozenCondition, _ := conditions.GetCondition(vmcondition.TypeFilesystemFrozen, vm.Status.Conditions) - if frozenCondition.Status == metav1.ConditionTrue { - err = h.snapshotter.Unfreeze(ctx, vm.Name, vm.Namespace) + kvvmi, err := h.snapshotter.GetVirtualMachineInstance(ctx, vm) + if err != nil { + return err + } + + err = h.unfreezeFilesystem(ctx, vdSnapshot.Name, vm, kvvmi) + if err != nil { + return err + } + + return nil +} + +func (h LifeCycleHandler) unfreezeFilesystem(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) error { + canUnfreeze, err := h.snapshotter.CanUnfreezeWithVirtualDiskSnapshot(ctx, vdSnapshotName, vm, kvvmi) + if err != nil { + return err + } + + if canUnfreeze { + err = h.snapshotter.Unfreeze(ctx, kvvmi) if err != nil { return err } diff --git a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/life_cycle_test.go b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/life_cycle_test.go index 56200e6019..a9ea15a8c2 100644 --- a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/life_cycle_test.go +++ b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/life_cycle_test.go @@ -25,11 +25,11 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vdscondition" - "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" ) var _ = Describe("LifeCycle handler", func() { @@ -89,6 +89,15 @@ var _ = Describe("LifeCycle handler", func() { GetVolumeSnapshotFunc: func(_ context.Context, _, _ string) (*vsv1.VolumeSnapshot, error) { return nil, nil }, + IsFrozenFunc: func(_ *virtv1.VirtualMachineInstance) (bool, error) { + return false, nil + }, + GetVirtualMachineInstanceFunc: func(_ context.Context, _ *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) { + return nil, nil + }, + SyncFSFreezeRequestFunc: func(_ context.Context, _ *virtv1.VirtualMachineInstance) error { + return nil + }, } }) @@ -149,11 +158,16 @@ var _ = Describe("LifeCycle handler", func() { } return vs, nil } + snapshotter.CanUnfreezeWithVirtualDiskSnapshotFunc = func(_ context.Context, _ string, _ *v1alpha2.VirtualMachine, _ *virtv1.VirtualMachineInstance) (bool, error) { + return false, nil + } h := NewLifeCycleHandler(snapshotter) _, err := h.Handle(testContext(), vdSnapshot) Expect(err).To(BeNil()) + _, err = h.Handle(testContext(), vdSnapshot) + Expect(err).To(BeNil()) Expect(vdSnapshot.Status.Phase).To(Equal(v1alpha2.VirtualDiskSnapshotPhaseReady)) ready, _ := conditions.GetCondition(vdscondition.VirtualDiskSnapshotReadyType, vdSnapshot.Status.Conditions) Expect(ready.Status).To(Equal(metav1.ConditionTrue)) @@ -174,24 +188,35 @@ var _ = Describe("LifeCycle handler", func() { Phase: v1alpha2.MachineRunning, }, } + + kvvmi := &virtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{Name: "vm"}, + Status: virtv1.VirtualMachineInstanceStatus{ + Phase: virtv1.Running, + }, + } + vd.Status.AttachedToVirtualMachines = []v1alpha2.AttachedVirtualMachine{{Name: vm.Name}} snapshotter.GetVirtualMachineFunc = func(_ context.Context, _, _ string) (*v1alpha2.VirtualMachine, error) { return vm, nil } - snapshotter.IsFrozenFunc = func(_ *v1alpha2.VirtualMachine) bool { - return false + snapshotter.GetVirtualMachineInstanceFunc = func(_ context.Context, _ *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) { + return kvvmi, nil } - snapshotter.CanFreezeFunc = func(_ *v1alpha2.VirtualMachine) bool { - return true + snapshotter.IsFrozenFunc = func(_ *virtv1.VirtualMachineInstance) (bool, error) { + return false, nil } - snapshotter.FreezeFunc = func(_ context.Context, _, _ string) error { + snapshotter.CanFreezeFunc = func(_ context.Context, _ *virtv1.VirtualMachineInstance) (bool, error) { + return true, nil + } + snapshotter.FreezeFunc = func(_ context.Context, _ *virtv1.VirtualMachineInstance) error { return nil } - snapshotter.CanUnfreezeWithVirtualDiskSnapshotFunc = func(_ context.Context, _ string, _ *v1alpha2.VirtualMachine) (bool, error) { + snapshotter.CanUnfreezeWithVirtualDiskSnapshotFunc = func(_ context.Context, _ string, _ *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { return true, nil } - snapshotter.UnfreezeFunc = func(_ context.Context, _, _ string) error { + snapshotter.UnfreezeFunc = func(_ context.Context, _ *virtv1.VirtualMachineInstance) error { return nil } }) @@ -226,8 +251,8 @@ var _ = Describe("LifeCycle handler", func() { It("Cannot freeze virtual machine: deny potentially inconsistent", func() { vdSnapshot.Spec.RequiredConsistency = true - snapshotter.CanFreezeFunc = func(_ *v1alpha2.VirtualMachine) bool { - return false + snapshotter.CanFreezeFunc = func(_ context.Context, _ *virtv1.VirtualMachineInstance) (bool, error) { + return false, nil } h := NewLifeCycleHandler(snapshotter) @@ -242,8 +267,8 @@ var _ = Describe("LifeCycle handler", func() { It("Cannot freeze virtual machine: allow potentially inconsistent", func() { vdSnapshot.Spec.RequiredConsistency = false - snapshotter.CanFreezeFunc = func(_ *v1alpha2.VirtualMachine) bool { - return false + snapshotter.CanFreezeFunc = func(_ context.Context, _ *virtv1.VirtualMachineInstance) (bool, error) { + return false, nil } h := NewLifeCycleHandler(snapshotter) @@ -257,8 +282,8 @@ var _ = Describe("LifeCycle handler", func() { }) It("Unfreeze virtual machine", func() { - snapshotter.IsFrozenFunc = func(_ *v1alpha2.VirtualMachine) bool { - return true + snapshotter.IsFrozenFunc = func(_ *virtv1.VirtualMachineInstance) (bool, error) { + return true, nil } snapshotter.GetVolumeSnapshotFunc = func(_ context.Context, _, _ string) (*vsv1.VolumeSnapshot, error) { vs.Status = &vsv1.VolumeSnapshotStatus{ @@ -270,6 +295,8 @@ var _ = Describe("LifeCycle handler", func() { _, err := h.Handle(testContext(), vdSnapshot) Expect(err).To(BeNil()) + _, err = h.Handle(testContext(), vdSnapshot) + Expect(err).To(BeNil()) Expect(vdSnapshot.Status.Phase).To(Equal(v1alpha2.VirtualDiskSnapshotPhaseReady)) ready, _ := conditions.GetCondition(vdscondition.VirtualDiskSnapshotReadyType, vdSnapshot.Status.Conditions) Expect(ready.Status).To(Equal(metav1.ConditionTrue)) @@ -277,11 +304,14 @@ var _ = Describe("LifeCycle handler", func() { Expect(ready.Message).To(BeEmpty()) }) - DescribeTable("Check unfreeze if failed", func(vm *v1alpha2.VirtualMachine, expectUnfreezing bool) { - unFreezeCalled := false + DescribeTable("Check unfreeze if failed", func(vm *v1alpha2.VirtualMachine, isFrozen, canUnfreeze, expectUnfreezing bool) { + unfreezeCalled := false - snapshotter.IsFrozenFunc = func(_ *v1alpha2.VirtualMachine) bool { - return true + snapshotter.IsFrozenFunc = func(_ *virtv1.VirtualMachineInstance) (bool, error) { + return isFrozen, nil + } + snapshotter.CanUnfreezeWithVirtualDiskSnapshotFunc = func(_ context.Context, _ string, _ *v1alpha2.VirtualMachine, _ *virtv1.VirtualMachineInstance) (bool, error) { + return canUnfreeze, nil } snapshotter.GetVolumeSnapshotFunc = func(_ context.Context, _, _ string) (*vsv1.VolumeSnapshot, error) { vs.Status = &vsv1.VolumeSnapshotStatus{ @@ -289,8 +319,8 @@ var _ = Describe("LifeCycle handler", func() { } return vs, nil } - snapshotter.UnfreezeFunc = func(_ context.Context, _, _ string) error { - unFreezeCalled = true + snapshotter.UnfreezeFunc = func(_ context.Context, _ *virtv1.VirtualMachineInstance) error { + unfreezeCalled = true return nil } snapshotter.GetVirtualMachineFunc = func(_ context.Context, _, _ string) (*v1alpha2.VirtualMachine, error) { @@ -304,23 +334,11 @@ var _ = Describe("LifeCycle handler", func() { Expect(err).To(BeNil()) Expect(vdSnapshot.Status.Phase).To(Equal(v1alpha2.VirtualDiskSnapshotPhaseFailed)) - Expect(unFreezeCalled).To(Equal(expectUnfreezing)) + Expect(unfreezeCalled).To(Equal(expectUnfreezing)) }, - Entry("Has VM with frozen filesystem", - &v1alpha2.VirtualMachine{ - Status: v1alpha2.VirtualMachineStatus{ - Conditions: []metav1.Condition{ - { - Type: vmcondition.TypeFilesystemFrozen.String(), - Status: metav1.ConditionTrue, - }, - }, - }, - }, - true, - ), - Entry("Has VM with unfrozen filesystem", &v1alpha2.VirtualMachine{}, false), - Entry("Has no VM", nil, false), + Entry("Has VM with frozen filesystem", &v1alpha2.VirtualMachine{}, true, true, true), + Entry("Has VM with unfrozen filesystem", &v1alpha2.VirtualMachine{}, false, false, false), + Entry("Has no VM", nil, false, false, false), ) }) }) diff --git a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/mock.go b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/mock.go index a063e0df10..c5adf99af9 100644 --- a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/mock.go +++ b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/mock.go @@ -8,6 +8,7 @@ import ( "github.com/deckhouse/virtualization/api/core/v1alpha2" vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" corev1 "k8s.io/api/core/v1" + virtv1 "kubevirt.io/api/core/v1" "sync" ) @@ -99,16 +100,16 @@ var _ LifeCycleSnapshotter = &LifeCycleSnapshotterMock{} // // // make and configure a mocked LifeCycleSnapshotter // mockedLifeCycleSnapshotter := &LifeCycleSnapshotterMock{ -// CanFreezeFunc: func(vm *v1alpha2.VirtualMachine) bool { +// CanFreezeFunc: func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { // panic("mock out the CanFreeze method") // }, -// CanUnfreezeWithVirtualDiskSnapshotFunc: func(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine) (bool, error) { +// CanUnfreezeWithVirtualDiskSnapshotFunc: func(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { // panic("mock out the CanUnfreezeWithVirtualDiskSnapshot method") // }, // CreateVolumeSnapshotFunc: func(ctx context.Context, vs *vsv1.VolumeSnapshot) (*vsv1.VolumeSnapshot, error) { // panic("mock out the CreateVolumeSnapshot method") // }, -// FreezeFunc: func(ctx context.Context, name string, namespace string) error { +// FreezeFunc: func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { // panic("mock out the Freeze method") // }, // GetPersistentVolumeClaimFunc: func(ctx context.Context, name string, namespace string) (*corev1.PersistentVolumeClaim, error) { @@ -120,13 +121,19 @@ var _ LifeCycleSnapshotter = &LifeCycleSnapshotterMock{} // GetVirtualMachineFunc: func(ctx context.Context, name string, namespace string) (*v1alpha2.VirtualMachine, error) { // panic("mock out the GetVirtualMachine method") // }, +// GetVirtualMachineInstanceFunc: func(ctx context.Context, vm *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) { +// panic("mock out the GetVirtualMachineInstance method") +// }, // GetVolumeSnapshotFunc: func(ctx context.Context, name string, namespace string) (*vsv1.VolumeSnapshot, error) { // panic("mock out the GetVolumeSnapshot method") // }, -// IsFrozenFunc: func(vm *v1alpha2.VirtualMachine) bool { +// IsFrozenFunc: func(kvvmi *virtv1.VirtualMachineInstance) (bool, error) { // panic("mock out the IsFrozen method") // }, -// UnfreezeFunc: func(ctx context.Context, name string, namespace string) error { +// SyncFSFreezeRequestFunc: func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { +// panic("mock out the SyncFSFreezeRequest method") +// }, +// UnfreezeFunc: func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { // panic("mock out the Unfreeze method") // }, // } @@ -137,16 +144,16 @@ var _ LifeCycleSnapshotter = &LifeCycleSnapshotterMock{} // } type LifeCycleSnapshotterMock struct { // CanFreezeFunc mocks the CanFreeze method. - CanFreezeFunc func(vm *v1alpha2.VirtualMachine) bool + CanFreezeFunc func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) // CanUnfreezeWithVirtualDiskSnapshotFunc mocks the CanUnfreezeWithVirtualDiskSnapshot method. - CanUnfreezeWithVirtualDiskSnapshotFunc func(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine) (bool, error) + CanUnfreezeWithVirtualDiskSnapshotFunc func(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) // CreateVolumeSnapshotFunc mocks the CreateVolumeSnapshot method. CreateVolumeSnapshotFunc func(ctx context.Context, vs *vsv1.VolumeSnapshot) (*vsv1.VolumeSnapshot, error) // FreezeFunc mocks the Freeze method. - FreezeFunc func(ctx context.Context, name string, namespace string) error + FreezeFunc func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error // GetPersistentVolumeClaimFunc mocks the GetPersistentVolumeClaim method. GetPersistentVolumeClaimFunc func(ctx context.Context, name string, namespace string) (*corev1.PersistentVolumeClaim, error) @@ -157,21 +164,29 @@ type LifeCycleSnapshotterMock struct { // GetVirtualMachineFunc mocks the GetVirtualMachine method. GetVirtualMachineFunc func(ctx context.Context, name string, namespace string) (*v1alpha2.VirtualMachine, error) + // GetVirtualMachineInstanceFunc mocks the GetVirtualMachineInstance method. + GetVirtualMachineInstanceFunc func(ctx context.Context, vm *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) + // GetVolumeSnapshotFunc mocks the GetVolumeSnapshot method. GetVolumeSnapshotFunc func(ctx context.Context, name string, namespace string) (*vsv1.VolumeSnapshot, error) // IsFrozenFunc mocks the IsFrozen method. - IsFrozenFunc func(vm *v1alpha2.VirtualMachine) bool + IsFrozenFunc func(kvvmi *virtv1.VirtualMachineInstance) (bool, error) + + // SyncFSFreezeRequestFunc mocks the SyncFSFreezeRequest method. + SyncFSFreezeRequestFunc func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error // UnfreezeFunc mocks the Unfreeze method. - UnfreezeFunc func(ctx context.Context, name string, namespace string) error + UnfreezeFunc func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error // calls tracks calls to the methods. calls struct { // CanFreeze holds details about calls to the CanFreeze method. CanFreeze []struct { - // VM is the vm argument value. - VM *v1alpha2.VirtualMachine + // Ctx is the ctx argument value. + Ctx context.Context + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance } // CanUnfreezeWithVirtualDiskSnapshot holds details about calls to the CanUnfreezeWithVirtualDiskSnapshot method. CanUnfreezeWithVirtualDiskSnapshot []struct { @@ -181,6 +196,8 @@ type LifeCycleSnapshotterMock struct { VdSnapshotName string // VM is the vm argument value. VM *v1alpha2.VirtualMachine + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance } // CreateVolumeSnapshot holds details about calls to the CreateVolumeSnapshot method. CreateVolumeSnapshot []struct { @@ -193,10 +210,8 @@ type LifeCycleSnapshotterMock struct { Freeze []struct { // Ctx is the ctx argument value. Ctx context.Context - // Name is the name argument value. - Name string - // Namespace is the namespace argument value. - Namespace string + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance } // GetPersistentVolumeClaim holds details about calls to the GetPersistentVolumeClaim method. GetPersistentVolumeClaim []struct { @@ -225,6 +240,13 @@ type LifeCycleSnapshotterMock struct { // Namespace is the namespace argument value. Namespace string } + // GetVirtualMachineInstance holds details about calls to the GetVirtualMachineInstance method. + GetVirtualMachineInstance []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // VM is the vm argument value. + VM *v1alpha2.VirtualMachine + } // GetVolumeSnapshot holds details about calls to the GetVolumeSnapshot method. GetVolumeSnapshot []struct { // Ctx is the ctx argument value. @@ -236,17 +258,22 @@ type LifeCycleSnapshotterMock struct { } // IsFrozen holds details about calls to the IsFrozen method. IsFrozen []struct { - // VM is the vm argument value. - VM *v1alpha2.VirtualMachine + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance + } + // SyncFSFreezeRequest holds details about calls to the SyncFSFreezeRequest method. + SyncFSFreezeRequest []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance } // Unfreeze holds details about calls to the Unfreeze method. Unfreeze []struct { // Ctx is the ctx argument value. Ctx context.Context - // Name is the name argument value. - Name string - // Namespace is the namespace argument value. - Namespace string + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance } } lockCanFreeze sync.RWMutex @@ -256,25 +283,29 @@ type LifeCycleSnapshotterMock struct { lockGetPersistentVolumeClaim sync.RWMutex lockGetVirtualDisk sync.RWMutex lockGetVirtualMachine sync.RWMutex + lockGetVirtualMachineInstance sync.RWMutex lockGetVolumeSnapshot sync.RWMutex lockIsFrozen sync.RWMutex + lockSyncFSFreezeRequest sync.RWMutex lockUnfreeze sync.RWMutex } // CanFreeze calls CanFreezeFunc. -func (mock *LifeCycleSnapshotterMock) CanFreeze(vm *v1alpha2.VirtualMachine) bool { +func (mock *LifeCycleSnapshotterMock) CanFreeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { if mock.CanFreezeFunc == nil { panic("LifeCycleSnapshotterMock.CanFreezeFunc: method is nil but LifeCycleSnapshotter.CanFreeze was just called") } callInfo := struct { - VM *v1alpha2.VirtualMachine + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance }{ - VM: vm, + Ctx: ctx, + Kvvmi: kvvmi, } mock.lockCanFreeze.Lock() mock.calls.CanFreeze = append(mock.calls.CanFreeze, callInfo) mock.lockCanFreeze.Unlock() - return mock.CanFreezeFunc(vm) + return mock.CanFreezeFunc(ctx, kvvmi) } // CanFreezeCalls gets all the calls that were made to CanFreeze. @@ -282,10 +313,12 @@ func (mock *LifeCycleSnapshotterMock) CanFreeze(vm *v1alpha2.VirtualMachine) boo // // len(mockedLifeCycleSnapshotter.CanFreezeCalls()) func (mock *LifeCycleSnapshotterMock) CanFreezeCalls() []struct { - VM *v1alpha2.VirtualMachine + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } { var calls []struct { - VM *v1alpha2.VirtualMachine + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } mock.lockCanFreeze.RLock() calls = mock.calls.CanFreeze @@ -294,7 +327,7 @@ func (mock *LifeCycleSnapshotterMock) CanFreezeCalls() []struct { } // CanUnfreezeWithVirtualDiskSnapshot calls CanUnfreezeWithVirtualDiskSnapshotFunc. -func (mock *LifeCycleSnapshotterMock) CanUnfreezeWithVirtualDiskSnapshot(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine) (bool, error) { +func (mock *LifeCycleSnapshotterMock) CanUnfreezeWithVirtualDiskSnapshot(ctx context.Context, vdSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { if mock.CanUnfreezeWithVirtualDiskSnapshotFunc == nil { panic("LifeCycleSnapshotterMock.CanUnfreezeWithVirtualDiskSnapshotFunc: method is nil but LifeCycleSnapshotter.CanUnfreezeWithVirtualDiskSnapshot was just called") } @@ -302,15 +335,17 @@ func (mock *LifeCycleSnapshotterMock) CanUnfreezeWithVirtualDiskSnapshot(ctx con Ctx context.Context VdSnapshotName string VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance }{ Ctx: ctx, VdSnapshotName: vdSnapshotName, VM: vm, + Kvvmi: kvvmi, } mock.lockCanUnfreezeWithVirtualDiskSnapshot.Lock() mock.calls.CanUnfreezeWithVirtualDiskSnapshot = append(mock.calls.CanUnfreezeWithVirtualDiskSnapshot, callInfo) mock.lockCanUnfreezeWithVirtualDiskSnapshot.Unlock() - return mock.CanUnfreezeWithVirtualDiskSnapshotFunc(ctx, vdSnapshotName, vm) + return mock.CanUnfreezeWithVirtualDiskSnapshotFunc(ctx, vdSnapshotName, vm, kvvmi) } // CanUnfreezeWithVirtualDiskSnapshotCalls gets all the calls that were made to CanUnfreezeWithVirtualDiskSnapshot. @@ -321,11 +356,13 @@ func (mock *LifeCycleSnapshotterMock) CanUnfreezeWithVirtualDiskSnapshotCalls() Ctx context.Context VdSnapshotName string VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance } { var calls []struct { Ctx context.Context VdSnapshotName string VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance } mock.lockCanUnfreezeWithVirtualDiskSnapshot.RLock() calls = mock.calls.CanUnfreezeWithVirtualDiskSnapshot @@ -370,23 +407,21 @@ func (mock *LifeCycleSnapshotterMock) CreateVolumeSnapshotCalls() []struct { } // Freeze calls FreezeFunc. -func (mock *LifeCycleSnapshotterMock) Freeze(ctx context.Context, name string, namespace string) error { +func (mock *LifeCycleSnapshotterMock) Freeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { if mock.FreezeFunc == nil { panic("LifeCycleSnapshotterMock.FreezeFunc: method is nil but LifeCycleSnapshotter.Freeze was just called") } callInfo := struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance }{ - Ctx: ctx, - Name: name, - Namespace: namespace, + Ctx: ctx, + Kvvmi: kvvmi, } mock.lockFreeze.Lock() mock.calls.Freeze = append(mock.calls.Freeze, callInfo) mock.lockFreeze.Unlock() - return mock.FreezeFunc(ctx, name, namespace) + return mock.FreezeFunc(ctx, kvvmi) } // FreezeCalls gets all the calls that were made to Freeze. @@ -394,14 +429,12 @@ func (mock *LifeCycleSnapshotterMock) Freeze(ctx context.Context, name string, n // // len(mockedLifeCycleSnapshotter.FreezeCalls()) func (mock *LifeCycleSnapshotterMock) FreezeCalls() []struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } { var calls []struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } mock.lockFreeze.RLock() calls = mock.calls.Freeze @@ -529,6 +562,42 @@ func (mock *LifeCycleSnapshotterMock) GetVirtualMachineCalls() []struct { return calls } +// GetVirtualMachineInstance calls GetVirtualMachineInstanceFunc. +func (mock *LifeCycleSnapshotterMock) GetVirtualMachineInstance(ctx context.Context, vm *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) { + if mock.GetVirtualMachineInstanceFunc == nil { + panic("LifeCycleSnapshotterMock.GetVirtualMachineInstanceFunc: method is nil but LifeCycleSnapshotter.GetVirtualMachineInstance was just called") + } + callInfo := struct { + Ctx context.Context + VM *v1alpha2.VirtualMachine + }{ + Ctx: ctx, + VM: vm, + } + mock.lockGetVirtualMachineInstance.Lock() + mock.calls.GetVirtualMachineInstance = append(mock.calls.GetVirtualMachineInstance, callInfo) + mock.lockGetVirtualMachineInstance.Unlock() + return mock.GetVirtualMachineInstanceFunc(ctx, vm) +} + +// GetVirtualMachineInstanceCalls gets all the calls that were made to GetVirtualMachineInstance. +// Check the length with: +// +// len(mockedLifeCycleSnapshotter.GetVirtualMachineInstanceCalls()) +func (mock *LifeCycleSnapshotterMock) GetVirtualMachineInstanceCalls() []struct { + Ctx context.Context + VM *v1alpha2.VirtualMachine +} { + var calls []struct { + Ctx context.Context + VM *v1alpha2.VirtualMachine + } + mock.lockGetVirtualMachineInstance.RLock() + calls = mock.calls.GetVirtualMachineInstance + mock.lockGetVirtualMachineInstance.RUnlock() + return calls +} + // GetVolumeSnapshot calls GetVolumeSnapshotFunc. func (mock *LifeCycleSnapshotterMock) GetVolumeSnapshot(ctx context.Context, name string, namespace string) (*vsv1.VolumeSnapshot, error) { if mock.GetVolumeSnapshotFunc == nil { @@ -570,19 +639,19 @@ func (mock *LifeCycleSnapshotterMock) GetVolumeSnapshotCalls() []struct { } // IsFrozen calls IsFrozenFunc. -func (mock *LifeCycleSnapshotterMock) IsFrozen(vm *v1alpha2.VirtualMachine) bool { +func (mock *LifeCycleSnapshotterMock) IsFrozen(kvvmi *virtv1.VirtualMachineInstance) (bool, error) { if mock.IsFrozenFunc == nil { panic("LifeCycleSnapshotterMock.IsFrozenFunc: method is nil but LifeCycleSnapshotter.IsFrozen was just called") } callInfo := struct { - VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance }{ - VM: vm, + Kvvmi: kvvmi, } mock.lockIsFrozen.Lock() mock.calls.IsFrozen = append(mock.calls.IsFrozen, callInfo) mock.lockIsFrozen.Unlock() - return mock.IsFrozenFunc(vm) + return mock.IsFrozenFunc(kvvmi) } // IsFrozenCalls gets all the calls that were made to IsFrozen. @@ -590,10 +659,10 @@ func (mock *LifeCycleSnapshotterMock) IsFrozen(vm *v1alpha2.VirtualMachine) bool // // len(mockedLifeCycleSnapshotter.IsFrozenCalls()) func (mock *LifeCycleSnapshotterMock) IsFrozenCalls() []struct { - VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance } { var calls []struct { - VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance } mock.lockIsFrozen.RLock() calls = mock.calls.IsFrozen @@ -601,24 +670,58 @@ func (mock *LifeCycleSnapshotterMock) IsFrozenCalls() []struct { return calls } +// SyncFSFreezeRequest calls SyncFSFreezeRequestFunc. +func (mock *LifeCycleSnapshotterMock) SyncFSFreezeRequest(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { + if mock.SyncFSFreezeRequestFunc == nil { + panic("LifeCycleSnapshotterMock.SyncFSFreezeRequestFunc: method is nil but LifeCycleSnapshotter.SyncFSFreezeRequest was just called") + } + callInfo := struct { + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance + }{ + Ctx: ctx, + Kvvmi: kvvmi, + } + mock.lockSyncFSFreezeRequest.Lock() + mock.calls.SyncFSFreezeRequest = append(mock.calls.SyncFSFreezeRequest, callInfo) + mock.lockSyncFSFreezeRequest.Unlock() + return mock.SyncFSFreezeRequestFunc(ctx, kvvmi) +} + +// SyncFSFreezeRequestCalls gets all the calls that were made to SyncFSFreezeRequest. +// Check the length with: +// +// len(mockedLifeCycleSnapshotter.SyncFSFreezeRequestCalls()) +func (mock *LifeCycleSnapshotterMock) SyncFSFreezeRequestCalls() []struct { + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance +} { + var calls []struct { + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance + } + mock.lockSyncFSFreezeRequest.RLock() + calls = mock.calls.SyncFSFreezeRequest + mock.lockSyncFSFreezeRequest.RUnlock() + return calls +} + // Unfreeze calls UnfreezeFunc. -func (mock *LifeCycleSnapshotterMock) Unfreeze(ctx context.Context, name string, namespace string) error { +func (mock *LifeCycleSnapshotterMock) Unfreeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { if mock.UnfreezeFunc == nil { panic("LifeCycleSnapshotterMock.UnfreezeFunc: method is nil but LifeCycleSnapshotter.Unfreeze was just called") } callInfo := struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance }{ - Ctx: ctx, - Name: name, - Namespace: namespace, + Ctx: ctx, + Kvvmi: kvvmi, } mock.lockUnfreeze.Lock() mock.calls.Unfreeze = append(mock.calls.Unfreeze, callInfo) mock.lockUnfreeze.Unlock() - return mock.UnfreezeFunc(ctx, name, namespace) + return mock.UnfreezeFunc(ctx, kvvmi) } // UnfreezeCalls gets all the calls that were made to Unfreeze. @@ -626,14 +729,12 @@ func (mock *LifeCycleSnapshotterMock) Unfreeze(ctx context.Context, name string, // // len(mockedLifeCycleSnapshotter.UnfreezeCalls()) func (mock *LifeCycleSnapshotterMock) UnfreezeCalls() []struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } { var calls []struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } mock.lockUnfreeze.RLock() calls = mock.calls.Unfreeze diff --git a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/watcher/kvvmi_watcher.go b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/watcher/kvvmi_watcher.go new file mode 100644 index 0000000000..45c182dad8 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/watcher/kvvmi_watcher.go @@ -0,0 +1,122 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "k8s.io/apimachinery/pkg/types" + virtv1 "kubevirt.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +type KVVMIWatcher struct { + client client.Client +} + +func NewKVVMIWatcher(client client.Client) *KVVMIWatcher { + return &KVVMIWatcher{ + client: client, + } +} + +func (w KVVMIWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { + if err := ctr.Watch( + source.Kind(mgr.GetCache(), &virtv1.VirtualMachineInstance{}, + handler.TypedEnqueueRequestsFromMapFunc(w.enqueueRequests), + predicate.TypedFuncs[*virtv1.VirtualMachineInstance]{ + UpdateFunc: w.filterUpdateEvents, + }, + ), + ); err != nil { + return fmt.Errorf("error setting watch on KVVMI: %w", err) + } + return nil +} + +func (w KVVMIWatcher) enqueueRequests(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (requests []reconcile.Request) { + volumeByName := make(map[string]struct{}) + for _, v := range kvvmi.Status.VolumeStatus { + if v.PersistentVolumeClaimInfo == nil { + continue + } + + if originalName, ok := strings.CutPrefix(v.Name, "vd-"); ok { + volumeByName[originalName] = struct{}{} + } + } + + if len(volumeByName) == 0 { + return + } + + var vdSnapshots v1alpha2.VirtualDiskSnapshotList + err := w.client.List(ctx, &vdSnapshots, &client.ListOptions{ + Namespace: kvvmi.GetNamespace(), + }) + if err != nil { + slog.Default().Error(fmt.Sprintf("failed to list virtual disk snapshots: %s", err)) + return + } + + for _, vdSnapshot := range vdSnapshots.Items { + _, ok := volumeByName[vdSnapshot.Spec.VirtualDiskName] + if !ok { + continue + } + + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: vdSnapshot.Name, + Namespace: vdSnapshot.Namespace, + }, + }) + } + + return +} + +func (w KVVMIWatcher) filterUpdateEvents(e event.TypedUpdateEvent[*virtv1.VirtualMachineInstance]) bool { + oldFSFrozen := e.ObjectOld.Status.FSFreezeStatus + newFSFrozen := e.ObjectNew.Status.FSFreezeStatus + + if oldFSFrozen != newFSFrozen { + return true + } + + oldRequest, oldOk := e.ObjectOld.Annotations[annotations.AnnVMFilesystemRequest] + newRequest, newOk := e.ObjectNew.Annotations[annotations.AnnVMFilesystemRequest] + + if oldOk && newOk { + return oldRequest != newRequest + } + + return oldOk != newOk +} diff --git a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/watcher/vm_watcher.go b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/watcher/vm_watcher.go index fd717873a6..4f7eed7d9a 100644 --- a/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/watcher/vm_watcher.go +++ b/images/virtualization-artifact/pkg/controller/vdsnapshot/internal/watcher/vm_watcher.go @@ -105,13 +105,6 @@ func (w VirtualMachineWatcher) filterUpdateEvents(e event.TypedUpdateEvent[*v1al return true } - oldFSFrozen, _ := conditions.GetCondition(vmcondition.TypeFilesystemFrozen, e.ObjectOld.Status.Conditions) - newFSFrozen, _ := conditions.GetCondition(vmcondition.TypeFilesystemFrozen, e.ObjectNew.Status.Conditions) - - if oldFSFrozen.Status != newFSFrozen.Status { - return true - } - oldAgentReady, _ := conditions.GetCondition(vmcondition.TypeAgentReady, e.ObjectOld.Status.Conditions) newAgentReady, _ := conditions.GetCondition(vmcondition.TypeAgentReady, e.ObjectNew.Status.Conditions) diff --git a/images/virtualization-artifact/pkg/controller/vdsnapshot/vdsnapshot_reconciler.go b/images/virtualization-artifact/pkg/controller/vdsnapshot/vdsnapshot_reconciler.go index 1c0b3264c7..9e5a9703c5 100644 --- a/images/virtualization-artifact/pkg/controller/vdsnapshot/vdsnapshot_reconciler.go +++ b/images/virtualization-artifact/pkg/controller/vdsnapshot/vdsnapshot_reconciler.go @@ -82,6 +82,7 @@ func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr watcher.NewVirtualDiskWatcher(mgr.GetClient()), watcher.NewVolumeSnapshotWatcher(), watcher.NewVirtualMachineWatcher(mgr.GetClient()), + watcher.NewKVVMIWatcher(mgr.GetClient()), } { err := w.Watch(mgr, ctr) if err != nil { diff --git a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/interfaces.go b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/interfaces.go index 6690332a57..4744a2b801 100644 --- a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/interfaces.go +++ b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/interfaces.go @@ -20,6 +20,7 @@ import ( "context" corev1 "k8s.io/api/core/v1" + virtv1 "kubevirt.io/api/core/v1" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -37,9 +38,11 @@ type Snapshotter interface { GetPersistentVolumeClaim(ctx context.Context, name, namespace string) (*corev1.PersistentVolumeClaim, error) GetVirtualDiskSnapshot(ctx context.Context, name, namespace string) (*v1alpha2.VirtualDiskSnapshot, error) CreateVirtualDiskSnapshot(ctx context.Context, vdSnapshot *v1alpha2.VirtualDiskSnapshot) (*v1alpha2.VirtualDiskSnapshot, error) - Freeze(ctx context.Context, name, namespace string) error - Unfreeze(ctx context.Context, name, namespace string) error - IsFrozen(vm *v1alpha2.VirtualMachine) bool - CanFreeze(vm *v1alpha2.VirtualMachine) bool - CanUnfreezeWithVirtualMachineSnapshot(ctx context.Context, vmSnapshotName string, vm *v1alpha2.VirtualMachine) (bool, error) + Freeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error + Unfreeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error + IsFrozen(kvvmi *virtv1.VirtualMachineInstance) (bool, error) + CanFreeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) + CanUnfreezeWithVirtualMachineSnapshot(ctx context.Context, vmSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) + SyncFSFreezeRequest(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error + GetVirtualMachineInstance(ctx context.Context, vm *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) } diff --git a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/life_cycle.go index 42219dfdbd..6be254afc9 100644 --- a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/life_cycle.go @@ -21,11 +21,14 @@ import ( "errors" "fmt" "strings" + "time" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -73,6 +76,36 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *v1alpha2.Virtu return reconcile.Result{}, err } + kvvmi, err := h.snapshotter.GetVirtualMachineInstance(ctx, vm) + if err != nil { + h.setPhaseConditionToFailed(cb, vmSnapshot, err) + return reconcile.Result{}, err + } + + err = h.snapshotter.SyncFSFreezeRequest(ctx, kvvmi) + switch { + case err == nil: + // OK. + case errors.Is(err, service.ErrUntrustedFilesystemFrozenCondition): + log.Debug(err.Error()) + cb. + Status(metav1.ConditionFalse). + Reason(vmscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{}, nil + case k8serrors.IsConflict(err): + log.Debug(fmt.Sprintf("failed to sync filesystem status; resource update conflict error: %s", err)) + cb. + Status(metav1.ConditionFalse). + Reason(vmscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + default: + err = fmt.Errorf("failed to sync filesystem status: %w", err) + h.setPhaseConditionToFailed(cb, vmSnapshot, err) + return reconcile.Result{}, err + } + if vmSnapshot.DeletionTimestamp != nil { vmSnapshot.Status.Phase = v1alpha2.VirtualMachineSnapshotPhaseTerminating cb. @@ -80,8 +113,24 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *v1alpha2.Virtu Reason(conditions.ReasonUnknown). Message("") - _, err = h.unfreezeVirtualMachineIfCan(ctx, vmSnapshot, vm) + _, err = h.unfreezeVirtualMachineIfCan(ctx, vmSnapshot, vm, kvvmi) if err != nil { + if errors.Is(err, service.ErrUntrustedFilesystemFrozenCondition) { + log.Debug(err.Error()) + cb. + Status(metav1.ConditionFalse). + Reason(vmscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{}, nil + } + if k8serrors.IsConflict(err) { + log.Debug(fmt.Sprintf("failed to freeze filesystem; resource update conflict error: %s", err)) + cb. + Status(metav1.ConditionFalse). + Reason(vmscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + } h.setPhaseConditionToFailed(cb, vmSnapshot, err) return reconcile.Result{}, err } @@ -190,9 +239,17 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *v1alpha2.Virtu return reconcile.Result{}, err } - needToFreeze := h.needToFreeze(vm, vmSnapshot.Spec.RequiredConsistency) + needToFreeze, err := h.needToFreeze(vm, kvvmi, vmSnapshot) + if err != nil { + return reconcile.Result{}, err + } - isAwaitingConsistency := needToFreeze && !h.snapshotter.CanFreeze(vm) && vmSnapshot.Spec.RequiredConsistency + canFreeze, err := h.snapshotter.CanFreeze(ctx, kvvmi) + if err != nil { + h.setPhaseConditionToFailed(cb, vmSnapshot, err) + return reconcile.Result{}, err + } + isAwaitingConsistency := needToFreeze && !canFreeze && vmSnapshot.Spec.RequiredConsistency if isAwaitingConsistency { vmSnapshot.Status.Phase = v1alpha2.VirtualMachineSnapshotPhasePending msg := fmt.Sprintf( @@ -243,8 +300,16 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *v1alpha2.Virtu // 2. Ensure the virtual machine is consistent for snapshotting. if needToFreeze { - hasFrozen, err = h.freezeVirtualMachine(ctx, vm, vmSnapshot) + hasFrozen, err = h.freezeVirtualMachine(ctx, kvvmi, vmSnapshot) if err != nil { + if k8serrors.IsConflict(err) { + log.Debug(fmt.Sprintf("failed to freeze filesystem; resource update conflict error: %s", err)) + cb. + Status(metav1.ConditionFalse). + Reason(vmscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + } h.setPhaseConditionToFailed(cb, vmSnapshot, err) return reconcile.Result{}, err } @@ -333,9 +398,21 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *v1alpha2.Virtu } // 7. Unfreeze VirtualMachine if can. - unfrozen, err := h.unfreezeVirtualMachineIfCan(ctx, vmSnapshot, vm) + unfrozen, err := h.unfreezeVirtualMachineIfCan(ctx, vmSnapshot, vm, kvvmi) if err != nil { - h.setPhaseConditionToFailed(cb, vmSnapshot, err) + if k8serrors.IsConflict(err) { + log.Debug(fmt.Sprintf("failed to unfreeze filesystem; resource update conflict error: %s", err)) + cb. + Status(metav1.ConditionFalse). + Reason(vmscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + } + vmSnapshot.Status.Phase = v1alpha2.VirtualMachineSnapshotPhaseInProgress + cb. + Status(metav1.ConditionFalse). + Reason(vmscondition.FileSystemUnfreezing). + Message(service.CapitalizeFirstLetter(err.Error() + ".")) return reconcile.Result{}, err } @@ -346,7 +423,32 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *v1alpha2.Virtu return reconcile.Result{}, err } - // 9. Move to Ready phase. + // 9. Synchronize FSFreezeRequest with KVVMI status. + err = h.snapshotter.SyncFSFreezeRequest(ctx, kvvmi) + switch { + case err == nil: + // OK. + case errors.Is(err, service.ErrUntrustedFilesystemFrozenCondition): + log.Debug(err.Error()) + cb. + Status(metav1.ConditionFalse). + Reason(vmscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{}, nil + case k8serrors.IsConflict(err): + log.Debug(fmt.Sprintf("failed to sync filesystem status; resource update conflict error: %s", err)) + cb. + Status(metav1.ConditionFalse). + Reason(vmscondition.Snapshotting). + Message(service.CapitalizeFirstLetter("Waiting for the filesystem of the virtual machine to be synced.")) + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + default: + err = fmt.Errorf("failed to sync filesystem status: %w", err) + h.setPhaseConditionToFailed(cb, vmSnapshot, err) + return reconcile.Result{}, err + } + + // 10. Move to Ready phase. log.Debug("The virtual disk snapshots are taken: the virtual machine snapshot is Ready now", "unfrozen", unfrozen) vmSnapshot.Status.Phase = v1alpha2.VirtualMachineSnapshotPhaseReady @@ -490,48 +592,64 @@ func (h LifeCycleHandler) areVirtualDiskSnapshotsConsistent(vdSnapshots []*v1alp return true } -func (h LifeCycleHandler) needToFreeze(vm *v1alpha2.VirtualMachine, requiredConsistency bool) bool { - if !requiredConsistency { - return false +func (h LifeCycleHandler) needToFreeze(vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance, vmsnapshot *v1alpha2.VirtualMachineSnapshot) (bool, error) { + if vmsnapshot.Status.Consistent != nil && *vmsnapshot.Status.Consistent { + return false, nil + } + + if !vmsnapshot.Spec.RequiredConsistency { + return false, nil } if vm.Status.Phase == v1alpha2.MachineStopped { - return false + return false, nil } - if h.snapshotter.IsFrozen(vm) { - return false + isFrozen, err := h.snapshotter.IsFrozen(kvvmi) + if err != nil { + return false, err + } + if isFrozen { + return false, nil } - return true + return true, nil } -func (h LifeCycleHandler) freezeVirtualMachine(ctx context.Context, vm *v1alpha2.VirtualMachine, vmSnapshot *v1alpha2.VirtualMachineSnapshot) (bool, error) { - if vm.Status.Phase != v1alpha2.MachineRunning { - return false, errors.New("cannot freeze not Running virtual machine") +func (h LifeCycleHandler) freezeVirtualMachine(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance, vmSnapshot *v1alpha2.VirtualMachineSnapshot) (bool, error) { + if kvvmi.Status.Phase != virtv1.Running { + return false, fmt.Errorf("cannot freeze not Running %s/%s virtual machine", kvvmi.Namespace, kvvmi.Name) } - err := h.snapshotter.Freeze(ctx, vm.Name, vm.Namespace) + err := h.snapshotter.Freeze(ctx, kvvmi) if err != nil { - return false, fmt.Errorf("freeze the virtual machine %q: %w", vm.Name, err) + return false, fmt.Errorf("freeze the virtual machine %s/%s: %w", kvvmi.Namespace, kvvmi.Name, err) } h.recorder.Event( vmSnapshot, corev1.EventTypeNormal, v1alpha2.ReasonVMSnapshottingFrozen, - fmt.Sprintf("The file system of the virtual machine %q is frozen.", vm.Name), + fmt.Sprintf("The file system of the virtual machine %q is frozen.", kvvmi.Name), ) return true, nil } -func (h LifeCycleHandler) unfreezeVirtualMachineIfCan(ctx context.Context, vmSnapshot *v1alpha2.VirtualMachineSnapshot, vm *v1alpha2.VirtualMachine) (bool, error) { - if vm == nil || vm.Status.Phase != v1alpha2.MachineRunning || !h.snapshotter.IsFrozen(vm) { +func (h LifeCycleHandler) unfreezeVirtualMachineIfCan(ctx context.Context, vmSnapshot *v1alpha2.VirtualMachineSnapshot, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { + if vm == nil || vm.Status.Phase != v1alpha2.MachineRunning { + return false, nil + } + + isFrozen, err := h.snapshotter.IsFrozen(kvvmi) + if err != nil { + return false, err + } + if !isFrozen { return false, nil } - canUnfreeze, err := h.snapshotter.CanUnfreezeWithVirtualMachineSnapshot(ctx, vmSnapshot.Name, vm) + canUnfreeze, err := h.snapshotter.CanUnfreezeWithVirtualMachineSnapshot(ctx, vmSnapshot.Name, vm, kvvmi) if err != nil { return false, err } @@ -540,7 +658,7 @@ func (h LifeCycleHandler) unfreezeVirtualMachineIfCan(ctx context.Context, vmSna return false, nil } - err = h.snapshotter.Unfreeze(ctx, vm.Name, vm.Namespace) + err = h.snapshotter.Unfreeze(ctx, kvvmi) if err != nil { return false, fmt.Errorf("unfreeze the virtual machine %q: %w", vm.Name, err) } diff --git a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/life_cycle_test.go b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/life_cycle_test.go index 66561c384a..78752c91b4 100644 --- a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/life_cycle_test.go +++ b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/life_cycle_test.go @@ -25,6 +25,7 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/deckhouse/virtualization-controller/pkg/common/testutil" @@ -42,6 +43,7 @@ var _ = Describe("LifeCycle handler", func() { var storer *StorerMock var vd *v1alpha2.VirtualDisk var vm *v1alpha2.VirtualMachine + var kvvmi *virtv1.VirtualMachineInstance var secret *corev1.Secret var vdSnapshot *v1alpha2.VirtualDiskSnapshot var vmSnapshot *v1alpha2.VirtualMachineSnapshot @@ -88,6 +90,13 @@ var _ = Describe("LifeCycle handler", func() { }, } + kvvmi = &virtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{Name: "vm"}, + Status: virtv1.VirtualMachineInstanceStatus{ + Phase: virtv1.Running, + }, + } + secret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: vm.Name}, } @@ -124,16 +133,16 @@ var _ = Describe("LifeCycle handler", func() { GetVirtualMachineFunc: func(_ context.Context, _, _ string) (*v1alpha2.VirtualMachine, error) { return vm, nil }, - IsFrozenFunc: func(_ *v1alpha2.VirtualMachine) bool { - return true + IsFrozenFunc: func(*virtv1.VirtualMachineInstance) (bool, error) { + return true, nil }, - CanUnfreezeWithVirtualMachineSnapshotFunc: func(_ context.Context, _ string, _ *v1alpha2.VirtualMachine) (bool, error) { + CanUnfreezeWithVirtualMachineSnapshotFunc: func(_ context.Context, _ string, _ *v1alpha2.VirtualMachine, _ *virtv1.VirtualMachineInstance) (bool, error) { return true, nil }, - CanFreezeFunc: func(_ *v1alpha2.VirtualMachine) bool { - return false + CanFreezeFunc: func(_ context.Context, _ *virtv1.VirtualMachineInstance) (bool, error) { + return false, nil }, - UnfreezeFunc: func(ctx context.Context, _, _ string) error { + UnfreezeFunc: func(ctx context.Context, _ *virtv1.VirtualMachineInstance) error { return nil }, GetSecretFunc: func(_ context.Context, _, _ string) (*corev1.Secret, error) { @@ -142,6 +151,12 @@ var _ = Describe("LifeCycle handler", func() { GetVirtualDiskSnapshotFunc: func(_ context.Context, _, _ string) (*v1alpha2.VirtualDiskSnapshot, error) { return vdSnapshot, nil }, + GetVirtualMachineInstanceFunc: func(_ context.Context, _ *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) { + return kvvmi, nil + }, + SyncFSFreezeRequestFunc: func(_ context.Context, _ *virtv1.VirtualMachineInstance) error { + return nil + }, } var err error @@ -248,11 +263,11 @@ var _ = Describe("LifeCycle handler", func() { }) It("The virtual machine is potentially inconsistent", func() { - snapshotter.IsFrozenFunc = func(_ *v1alpha2.VirtualMachine) bool { - return false + snapshotter.IsFrozenFunc = func(_ *virtv1.VirtualMachineInstance) (bool, error) { + return false, nil } - snapshotter.CanFreezeFunc = func(_ *v1alpha2.VirtualMachine) bool { - return false + snapshotter.CanFreezeFunc = func(_ context.Context, _ *virtv1.VirtualMachineInstance) (bool, error) { + return false, nil } h := NewLifeCycleHandler(recorder, snapshotter, storer, fakeClient) @@ -267,13 +282,13 @@ var _ = Describe("LifeCycle handler", func() { }) It("The virtual machine has frozen", func() { - snapshotter.IsFrozenFunc = func(_ *v1alpha2.VirtualMachine) bool { - return false + snapshotter.IsFrozenFunc = func(_ *virtv1.VirtualMachineInstance) (bool, error) { + return false, nil } - snapshotter.CanFreezeFunc = func(_ *v1alpha2.VirtualMachine) bool { - return true + snapshotter.CanFreezeFunc = func(_ context.Context, _ *virtv1.VirtualMachineInstance) (bool, error) { + return true, nil } - snapshotter.FreezeFunc = func(_ context.Context, _, _ string) error { + snapshotter.FreezeFunc = func(_ context.Context, _ *virtv1.VirtualMachineInstance) error { return nil } diff --git a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/mock.go b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/mock.go index 134a491adb..67e5c82199 100644 --- a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/mock.go +++ b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/mock.go @@ -7,6 +7,7 @@ import ( "context" "github.com/deckhouse/virtualization/api/core/v1alpha2" corev1 "k8s.io/api/core/v1" + virtv1 "kubevirt.io/api/core/v1" "sync" ) @@ -98,16 +99,16 @@ var _ Snapshotter = &SnapshotterMock{} // // // make and configure a mocked Snapshotter // mockedSnapshotter := &SnapshotterMock{ -// CanFreezeFunc: func(vm *v1alpha2.VirtualMachine) bool { +// CanFreezeFunc: func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { // panic("mock out the CanFreeze method") // }, -// CanUnfreezeWithVirtualMachineSnapshotFunc: func(ctx context.Context, vmSnapshotName string, vm *v1alpha2.VirtualMachine) (bool, error) { +// CanUnfreezeWithVirtualMachineSnapshotFunc: func(ctx context.Context, vmSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { // panic("mock out the CanUnfreezeWithVirtualMachineSnapshot method") // }, // CreateVirtualDiskSnapshotFunc: func(ctx context.Context, vdSnapshot *v1alpha2.VirtualDiskSnapshot) (*v1alpha2.VirtualDiskSnapshot, error) { // panic("mock out the CreateVirtualDiskSnapshot method") // }, -// FreezeFunc: func(ctx context.Context, name string, namespace string) error { +// FreezeFunc: func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { // panic("mock out the Freeze method") // }, // GetPersistentVolumeClaimFunc: func(ctx context.Context, name string, namespace string) (*corev1.PersistentVolumeClaim, error) { @@ -125,10 +126,16 @@ var _ Snapshotter = &SnapshotterMock{} // GetVirtualMachineFunc: func(ctx context.Context, name string, namespace string) (*v1alpha2.VirtualMachine, error) { // panic("mock out the GetVirtualMachine method") // }, -// IsFrozenFunc: func(vm *v1alpha2.VirtualMachine) bool { +// GetVirtualMachineInstanceFunc: func(ctx context.Context, vm *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) { +// panic("mock out the GetVirtualMachineInstance method") +// }, +// IsFrozenFunc: func(kvvmi *virtv1.VirtualMachineInstance) (bool, error) { // panic("mock out the IsFrozen method") // }, -// UnfreezeFunc: func(ctx context.Context, name string, namespace string) error { +// SyncFSFreezeRequestFunc: func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { +// panic("mock out the SyncFSFreezeRequest method") +// }, +// UnfreezeFunc: func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { // panic("mock out the Unfreeze method") // }, // } @@ -139,16 +146,16 @@ var _ Snapshotter = &SnapshotterMock{} // } type SnapshotterMock struct { // CanFreezeFunc mocks the CanFreeze method. - CanFreezeFunc func(vm *v1alpha2.VirtualMachine) bool + CanFreezeFunc func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) // CanUnfreezeWithVirtualMachineSnapshotFunc mocks the CanUnfreezeWithVirtualMachineSnapshot method. - CanUnfreezeWithVirtualMachineSnapshotFunc func(ctx context.Context, vmSnapshotName string, vm *v1alpha2.VirtualMachine) (bool, error) + CanUnfreezeWithVirtualMachineSnapshotFunc func(ctx context.Context, vmSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) // CreateVirtualDiskSnapshotFunc mocks the CreateVirtualDiskSnapshot method. CreateVirtualDiskSnapshotFunc func(ctx context.Context, vdSnapshot *v1alpha2.VirtualDiskSnapshot) (*v1alpha2.VirtualDiskSnapshot, error) // FreezeFunc mocks the Freeze method. - FreezeFunc func(ctx context.Context, name string, namespace string) error + FreezeFunc func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error // GetPersistentVolumeClaimFunc mocks the GetPersistentVolumeClaim method. GetPersistentVolumeClaimFunc func(ctx context.Context, name string, namespace string) (*corev1.PersistentVolumeClaim, error) @@ -165,18 +172,26 @@ type SnapshotterMock struct { // GetVirtualMachineFunc mocks the GetVirtualMachine method. GetVirtualMachineFunc func(ctx context.Context, name string, namespace string) (*v1alpha2.VirtualMachine, error) + // GetVirtualMachineInstanceFunc mocks the GetVirtualMachineInstance method. + GetVirtualMachineInstanceFunc func(ctx context.Context, vm *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) + // IsFrozenFunc mocks the IsFrozen method. - IsFrozenFunc func(vm *v1alpha2.VirtualMachine) bool + IsFrozenFunc func(kvvmi *virtv1.VirtualMachineInstance) (bool, error) + + // SyncFSFreezeRequestFunc mocks the SyncFSFreezeRequest method. + SyncFSFreezeRequestFunc func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error // UnfreezeFunc mocks the Unfreeze method. - UnfreezeFunc func(ctx context.Context, name string, namespace string) error + UnfreezeFunc func(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error // calls tracks calls to the methods. calls struct { // CanFreeze holds details about calls to the CanFreeze method. CanFreeze []struct { - // VM is the vm argument value. - VM *v1alpha2.VirtualMachine + // Ctx is the ctx argument value. + Ctx context.Context + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance } // CanUnfreezeWithVirtualMachineSnapshot holds details about calls to the CanUnfreezeWithVirtualMachineSnapshot method. CanUnfreezeWithVirtualMachineSnapshot []struct { @@ -186,6 +201,8 @@ type SnapshotterMock struct { VmSnapshotName string // VM is the vm argument value. VM *v1alpha2.VirtualMachine + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance } // CreateVirtualDiskSnapshot holds details about calls to the CreateVirtualDiskSnapshot method. CreateVirtualDiskSnapshot []struct { @@ -198,10 +215,8 @@ type SnapshotterMock struct { Freeze []struct { // Ctx is the ctx argument value. Ctx context.Context - // Name is the name argument value. - Name string - // Namespace is the namespace argument value. - Namespace string + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance } // GetPersistentVolumeClaim holds details about calls to the GetPersistentVolumeClaim method. GetPersistentVolumeClaim []struct { @@ -248,19 +263,31 @@ type SnapshotterMock struct { // Namespace is the namespace argument value. Namespace string } - // IsFrozen holds details about calls to the IsFrozen method. - IsFrozen []struct { + // GetVirtualMachineInstance holds details about calls to the GetVirtualMachineInstance method. + GetVirtualMachineInstance []struct { + // Ctx is the ctx argument value. + Ctx context.Context // VM is the vm argument value. VM *v1alpha2.VirtualMachine } + // IsFrozen holds details about calls to the IsFrozen method. + IsFrozen []struct { + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance + } + // SyncFSFreezeRequest holds details about calls to the SyncFSFreezeRequest method. + SyncFSFreezeRequest []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance + } // Unfreeze holds details about calls to the Unfreeze method. Unfreeze []struct { // Ctx is the ctx argument value. Ctx context.Context - // Name is the name argument value. - Name string - // Namespace is the namespace argument value. - Namespace string + // Kvvmi is the kvvmi argument value. + Kvvmi *virtv1.VirtualMachineInstance } } lockCanFreeze sync.RWMutex @@ -272,24 +299,28 @@ type SnapshotterMock struct { lockGetVirtualDisk sync.RWMutex lockGetVirtualDiskSnapshot sync.RWMutex lockGetVirtualMachine sync.RWMutex + lockGetVirtualMachineInstance sync.RWMutex lockIsFrozen sync.RWMutex + lockSyncFSFreezeRequest sync.RWMutex lockUnfreeze sync.RWMutex } // CanFreeze calls CanFreezeFunc. -func (mock *SnapshotterMock) CanFreeze(vm *v1alpha2.VirtualMachine) bool { +func (mock *SnapshotterMock) CanFreeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { if mock.CanFreezeFunc == nil { panic("SnapshotterMock.CanFreezeFunc: method is nil but Snapshotter.CanFreeze was just called") } callInfo := struct { - VM *v1alpha2.VirtualMachine + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance }{ - VM: vm, + Ctx: ctx, + Kvvmi: kvvmi, } mock.lockCanFreeze.Lock() mock.calls.CanFreeze = append(mock.calls.CanFreeze, callInfo) mock.lockCanFreeze.Unlock() - return mock.CanFreezeFunc(vm) + return mock.CanFreezeFunc(ctx, kvvmi) } // CanFreezeCalls gets all the calls that were made to CanFreeze. @@ -297,10 +328,12 @@ func (mock *SnapshotterMock) CanFreeze(vm *v1alpha2.VirtualMachine) bool { // // len(mockedSnapshotter.CanFreezeCalls()) func (mock *SnapshotterMock) CanFreezeCalls() []struct { - VM *v1alpha2.VirtualMachine + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } { var calls []struct { - VM *v1alpha2.VirtualMachine + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } mock.lockCanFreeze.RLock() calls = mock.calls.CanFreeze @@ -309,7 +342,7 @@ func (mock *SnapshotterMock) CanFreezeCalls() []struct { } // CanUnfreezeWithVirtualMachineSnapshot calls CanUnfreezeWithVirtualMachineSnapshotFunc. -func (mock *SnapshotterMock) CanUnfreezeWithVirtualMachineSnapshot(ctx context.Context, vmSnapshotName string, vm *v1alpha2.VirtualMachine) (bool, error) { +func (mock *SnapshotterMock) CanUnfreezeWithVirtualMachineSnapshot(ctx context.Context, vmSnapshotName string, vm *v1alpha2.VirtualMachine, kvvmi *virtv1.VirtualMachineInstance) (bool, error) { if mock.CanUnfreezeWithVirtualMachineSnapshotFunc == nil { panic("SnapshotterMock.CanUnfreezeWithVirtualMachineSnapshotFunc: method is nil but Snapshotter.CanUnfreezeWithVirtualMachineSnapshot was just called") } @@ -317,15 +350,17 @@ func (mock *SnapshotterMock) CanUnfreezeWithVirtualMachineSnapshot(ctx context.C Ctx context.Context VmSnapshotName string VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance }{ Ctx: ctx, VmSnapshotName: vmSnapshotName, VM: vm, + Kvvmi: kvvmi, } mock.lockCanUnfreezeWithVirtualMachineSnapshot.Lock() mock.calls.CanUnfreezeWithVirtualMachineSnapshot = append(mock.calls.CanUnfreezeWithVirtualMachineSnapshot, callInfo) mock.lockCanUnfreezeWithVirtualMachineSnapshot.Unlock() - return mock.CanUnfreezeWithVirtualMachineSnapshotFunc(ctx, vmSnapshotName, vm) + return mock.CanUnfreezeWithVirtualMachineSnapshotFunc(ctx, vmSnapshotName, vm, kvvmi) } // CanUnfreezeWithVirtualMachineSnapshotCalls gets all the calls that were made to CanUnfreezeWithVirtualMachineSnapshot. @@ -336,11 +371,13 @@ func (mock *SnapshotterMock) CanUnfreezeWithVirtualMachineSnapshotCalls() []stru Ctx context.Context VmSnapshotName string VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance } { var calls []struct { Ctx context.Context VmSnapshotName string VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance } mock.lockCanUnfreezeWithVirtualMachineSnapshot.RLock() calls = mock.calls.CanUnfreezeWithVirtualMachineSnapshot @@ -385,23 +422,21 @@ func (mock *SnapshotterMock) CreateVirtualDiskSnapshotCalls() []struct { } // Freeze calls FreezeFunc. -func (mock *SnapshotterMock) Freeze(ctx context.Context, name string, namespace string) error { +func (mock *SnapshotterMock) Freeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { if mock.FreezeFunc == nil { panic("SnapshotterMock.FreezeFunc: method is nil but Snapshotter.Freeze was just called") } callInfo := struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance }{ - Ctx: ctx, - Name: name, - Namespace: namespace, + Ctx: ctx, + Kvvmi: kvvmi, } mock.lockFreeze.Lock() mock.calls.Freeze = append(mock.calls.Freeze, callInfo) mock.lockFreeze.Unlock() - return mock.FreezeFunc(ctx, name, namespace) + return mock.FreezeFunc(ctx, kvvmi) } // FreezeCalls gets all the calls that were made to Freeze. @@ -409,14 +444,12 @@ func (mock *SnapshotterMock) Freeze(ctx context.Context, name string, namespace // // len(mockedSnapshotter.FreezeCalls()) func (mock *SnapshotterMock) FreezeCalls() []struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } { var calls []struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } mock.lockFreeze.RLock() calls = mock.calls.Freeze @@ -624,20 +657,56 @@ func (mock *SnapshotterMock) GetVirtualMachineCalls() []struct { return calls } +// GetVirtualMachineInstance calls GetVirtualMachineInstanceFunc. +func (mock *SnapshotterMock) GetVirtualMachineInstance(ctx context.Context, vm *v1alpha2.VirtualMachine) (*virtv1.VirtualMachineInstance, error) { + if mock.GetVirtualMachineInstanceFunc == nil { + panic("SnapshotterMock.GetVirtualMachineInstanceFunc: method is nil but Snapshotter.GetVirtualMachineInstance was just called") + } + callInfo := struct { + Ctx context.Context + VM *v1alpha2.VirtualMachine + }{ + Ctx: ctx, + VM: vm, + } + mock.lockGetVirtualMachineInstance.Lock() + mock.calls.GetVirtualMachineInstance = append(mock.calls.GetVirtualMachineInstance, callInfo) + mock.lockGetVirtualMachineInstance.Unlock() + return mock.GetVirtualMachineInstanceFunc(ctx, vm) +} + +// GetVirtualMachineInstanceCalls gets all the calls that were made to GetVirtualMachineInstance. +// Check the length with: +// +// len(mockedSnapshotter.GetVirtualMachineInstanceCalls()) +func (mock *SnapshotterMock) GetVirtualMachineInstanceCalls() []struct { + Ctx context.Context + VM *v1alpha2.VirtualMachine +} { + var calls []struct { + Ctx context.Context + VM *v1alpha2.VirtualMachine + } + mock.lockGetVirtualMachineInstance.RLock() + calls = mock.calls.GetVirtualMachineInstance + mock.lockGetVirtualMachineInstance.RUnlock() + return calls +} + // IsFrozen calls IsFrozenFunc. -func (mock *SnapshotterMock) IsFrozen(vm *v1alpha2.VirtualMachine) bool { +func (mock *SnapshotterMock) IsFrozen(kvvmi *virtv1.VirtualMachineInstance) (bool, error) { if mock.IsFrozenFunc == nil { panic("SnapshotterMock.IsFrozenFunc: method is nil but Snapshotter.IsFrozen was just called") } callInfo := struct { - VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance }{ - VM: vm, + Kvvmi: kvvmi, } mock.lockIsFrozen.Lock() mock.calls.IsFrozen = append(mock.calls.IsFrozen, callInfo) mock.lockIsFrozen.Unlock() - return mock.IsFrozenFunc(vm) + return mock.IsFrozenFunc(kvvmi) } // IsFrozenCalls gets all the calls that were made to IsFrozen. @@ -645,10 +714,10 @@ func (mock *SnapshotterMock) IsFrozen(vm *v1alpha2.VirtualMachine) bool { // // len(mockedSnapshotter.IsFrozenCalls()) func (mock *SnapshotterMock) IsFrozenCalls() []struct { - VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance } { var calls []struct { - VM *v1alpha2.VirtualMachine + Kvvmi *virtv1.VirtualMachineInstance } mock.lockIsFrozen.RLock() calls = mock.calls.IsFrozen @@ -656,24 +725,58 @@ func (mock *SnapshotterMock) IsFrozenCalls() []struct { return calls } +// SyncFSFreezeRequest calls SyncFSFreezeRequestFunc. +func (mock *SnapshotterMock) SyncFSFreezeRequest(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { + if mock.SyncFSFreezeRequestFunc == nil { + panic("SnapshotterMock.SyncFSFreezeRequestFunc: method is nil but Snapshotter.SyncFSFreezeRequest was just called") + } + callInfo := struct { + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance + }{ + Ctx: ctx, + Kvvmi: kvvmi, + } + mock.lockSyncFSFreezeRequest.Lock() + mock.calls.SyncFSFreezeRequest = append(mock.calls.SyncFSFreezeRequest, callInfo) + mock.lockSyncFSFreezeRequest.Unlock() + return mock.SyncFSFreezeRequestFunc(ctx, kvvmi) +} + +// SyncFSFreezeRequestCalls gets all the calls that were made to SyncFSFreezeRequest. +// Check the length with: +// +// len(mockedSnapshotter.SyncFSFreezeRequestCalls()) +func (mock *SnapshotterMock) SyncFSFreezeRequestCalls() []struct { + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance +} { + var calls []struct { + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance + } + mock.lockSyncFSFreezeRequest.RLock() + calls = mock.calls.SyncFSFreezeRequest + mock.lockSyncFSFreezeRequest.RUnlock() + return calls +} + // Unfreeze calls UnfreezeFunc. -func (mock *SnapshotterMock) Unfreeze(ctx context.Context, name string, namespace string) error { +func (mock *SnapshotterMock) Unfreeze(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) error { if mock.UnfreezeFunc == nil { panic("SnapshotterMock.UnfreezeFunc: method is nil but Snapshotter.Unfreeze was just called") } callInfo := struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance }{ - Ctx: ctx, - Name: name, - Namespace: namespace, + Ctx: ctx, + Kvvmi: kvvmi, } mock.lockUnfreeze.Lock() mock.calls.Unfreeze = append(mock.calls.Unfreeze, callInfo) mock.lockUnfreeze.Unlock() - return mock.UnfreezeFunc(ctx, name, namespace) + return mock.UnfreezeFunc(ctx, kvvmi) } // UnfreezeCalls gets all the calls that were made to Unfreeze. @@ -681,14 +784,12 @@ func (mock *SnapshotterMock) Unfreeze(ctx context.Context, name string, namespac // // len(mockedSnapshotter.UnfreezeCalls()) func (mock *SnapshotterMock) UnfreezeCalls() []struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } { var calls []struct { - Ctx context.Context - Name string - Namespace string + Ctx context.Context + Kvvmi *virtv1.VirtualMachineInstance } mock.lockUnfreeze.RLock() calls = mock.calls.Unfreeze diff --git a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/watcher/kvvmi_watcher.go b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/watcher/kvvmi_watcher.go new file mode 100644 index 0000000000..b63c0063a2 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/watcher/kvvmi_watcher.go @@ -0,0 +1,106 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "context" + "fmt" + "log/slog" + + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + virtv1 "kubevirt.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/controller/indexer" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +type KVVMIWatcher struct { + client client.Client +} + +func NewKVVMIWatcher(client client.Client) *KVVMIWatcher { + return &KVVMIWatcher{ + client: client, + } +} + +func (w KVVMIWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { + if err := ctr.Watch( + source.Kind(mgr.GetCache(), &virtv1.VirtualMachineInstance{}, + handler.TypedEnqueueRequestsFromMapFunc(w.enqueueRequests), + predicate.TypedFuncs[*virtv1.VirtualMachineInstance]{ + UpdateFunc: w.filterUpdateEvents, + }, + ), + ); err != nil { + return fmt.Errorf("error setting watch on KVVMI: %w", err) + } + return nil +} + +func (w KVVMIWatcher) enqueueRequests(ctx context.Context, kvvmi *virtv1.VirtualMachineInstance) (requests []reconcile.Request) { + var vmSnapshots v1alpha2.VirtualMachineSnapshotList + err := w.client.List(ctx, &vmSnapshots, &client.ListOptions{ + Namespace: kvvmi.GetNamespace(), + FieldSelector: fields.OneTermEqualSelector(indexer.IndexFieldVMSnapshotByVM, kvvmi.GetName()), + }) + if err != nil { + slog.Default().Error(fmt.Sprintf("failed to list virtual machine snapshots: %s", err)) + return + } + + for _, vmSnapshot := range vmSnapshots.Items { + if vmSnapshot.Spec.VirtualMachineName == kvvmi.GetName() { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: vmSnapshot.Name, + Namespace: vmSnapshot.Namespace, + }, + }) + } + } + + return +} + +func (w KVVMIWatcher) filterUpdateEvents(e event.TypedUpdateEvent[*virtv1.VirtualMachineInstance]) bool { + oldFSFrozen := e.ObjectOld.Status.FSFreezeStatus + newFSFrozen := e.ObjectNew.Status.FSFreezeStatus + + if oldFSFrozen != newFSFrozen { + return true + } + + oldRequest, oldOk := e.ObjectOld.Annotations[annotations.AnnVMFilesystemRequest] + newRequest, newOk := e.ObjectNew.Annotations[annotations.AnnVMFilesystemRequest] + + if oldOk && newOk { + return oldRequest != newRequest + } + + return oldOk != newOk +} diff --git a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/watcher/vm_watcher.go b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/watcher/vm_watcher.go index 03f03c1aec..67e13804af 100644 --- a/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/watcher/vm_watcher.go +++ b/images/virtualization-artifact/pkg/controller/vmsnapshot/internal/watcher/vm_watcher.go @@ -96,13 +96,6 @@ func (w VirtualMachineWatcher) filterUpdateEvents(e event.TypedUpdateEvent[*v1al return true } - oldFSFrozen, _ := conditions.GetCondition(vmcondition.TypeFilesystemFrozen, e.ObjectOld.Status.Conditions) - newFSFrozen, _ := conditions.GetCondition(vmcondition.TypeFilesystemFrozen, e.ObjectNew.Status.Conditions) - - if oldFSFrozen.Reason != newFSFrozen.Reason { - return true - } - oldSnapshotting, _ := conditions.GetCondition(vmcondition.TypeSnapshotting, e.ObjectOld.Status.Conditions) newSnapshotting, _ := conditions.GetCondition(vmcondition.TypeSnapshotting, e.ObjectNew.Status.Conditions) diff --git a/images/virtualization-artifact/pkg/controller/vmsnapshot/vmsnapshot_reconciler.go b/images/virtualization-artifact/pkg/controller/vmsnapshot/vmsnapshot_reconciler.go index 750a70a2b9..c9a1513b18 100644 --- a/images/virtualization-artifact/pkg/controller/vmsnapshot/vmsnapshot_reconciler.go +++ b/images/virtualization-artifact/pkg/controller/vmsnapshot/vmsnapshot_reconciler.go @@ -82,6 +82,7 @@ func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr watcher.NewVirtualDiskSnapshotWatcher(mgr.GetClient()), watcher.NewVirtualMachineWatcher(mgr.GetClient()), watcher.NewVirtualDiskWatcher(mgr.GetClient()), + watcher.NewKVVMIWatcher(mgr.GetClient()), } { err := w.Watch(mgr, ctr) if err != nil { diff --git a/test/e2e/legacy/vd_snapshots.go b/test/e2e/legacy/vd_snapshots.go index 4095812880..4a622bf97e 100644 --- a/test/e2e/legacy/vd_snapshots.go +++ b/test/e2e/legacy/vd_snapshots.go @@ -30,6 +30,7 @@ import ( "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" + "github.com/deckhouse/virtualization/test/e2e/internal/config" kc "github.com/deckhouse/virtualization/test/e2e/internal/kubectl" "github.com/deckhouse/virtualization/test/e2e/internal/util" ) @@ -360,19 +361,21 @@ var _ = Describe("VirtualDiskSnapshots", Ordered, func() { Context("When test is completed", func() { It("deletes test case resources", func() { - DeleteTestCaseResources(ns, ResourcesToDelete{ - KustomizationDir: conf.TestData.VdSnapshots, - AdditionalResources: []AdditionalResource{ - { - Resource: kc.ResourceVDSnapshot, - Labels: hasNoConsumerLabel, + if config.IsCleanUpNeeded() { + DeleteTestCaseResources(ns, ResourcesToDelete{ + KustomizationDir: conf.TestData.VdSnapshots, + AdditionalResources: []AdditionalResource{ + { + Resource: kc.ResourceVDSnapshot, + Labels: hasNoConsumerLabel, + }, + { + Resource: kc.ResourceVDSnapshot, + Labels: attachedVirtualDiskLabel, + }, }, - { - Resource: kc.ResourceVDSnapshot, - Labels: attachedVirtualDiskLabel, - }, - }, - }) + }) + } }) }) })