From 88c6181ff0d96c70e9a35046b5a8506560147368 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Mon, 18 Mar 2019 10:57:09 +0100 Subject: [PATCH 1/6] Added test for volume resize. Added code to do so. --- pkg/deployment/resources/pvc_inspector.go | 19 +++++ tests/persistent_volumes_test.go | 92 ++++++++++++++++++++++- 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/pkg/deployment/resources/pvc_inspector.go b/pkg/deployment/resources/pvc_inspector.go index 2c24edfbb..38ec8634f 100644 --- a/pkg/deployment/resources/pvc_inspector.go +++ b/pkg/deployment/resources/pvc_inspector.go @@ -29,6 +29,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/metrics" "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + apiv1 "k8s.io/api/core/v1" ) var ( @@ -57,6 +58,7 @@ func (r *Resources) InspectPVCs(ctx context.Context) (util.Interval, error) { // Update member status from all pods found status, _ := r.context.GetStatus() + spec := r.context.GetSpec() for _, p := range pvcs { // PVC belongs to this deployment, update metric inspectedPVCsCounters.WithLabelValues(deploymentName).Inc() @@ -79,6 +81,23 @@ func (r *Resources) InspectPVCs(ctx context.Context) (util.Interval, error) { continue } + // Resize inspector + groupSpec := spec.GetServerGroupSpec(group) + if volumeSize, ok := p.Spec.Resources.Requests[apiv1.ResourceStorage]; ok { + requestedSize := groupSpec.Resources.Requests[apiv1.ResourceStorage] + if volumeSize.Cmp(requestedSize) < 0 { + // Size of the volume is smaller than the requested size + // Update the pvc with the request size + p.Spec.Resources.Requests[apiv1.ResourceStorage] = requestedSize + + log.Debug().Str("pvc-capacity", volumeSize.String()).Str("requested", requestedSize.String()).Msg("PVC capacity differes - updating") + kube := r.context.GetKubeCli() + if _, err := kube.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()).Update(&p); err != nil { + log.Error().Err(err).Msg("Failed to update pvc") + } + } + } + if k8sutil.IsPersistentVolumeClaimMarkedForDeletion(&p) { // Process finalizers if x, err := r.runPVCFinalizers(ctx, &p, group, memberStatus); err != nil { diff --git a/tests/persistent_volumes_test.go b/tests/persistent_volumes_test.go index 4edf4520e..4d9062553 100644 --- a/tests/persistent_volumes_test.go +++ b/tests/persistent_volumes_test.go @@ -25,17 +25,24 @@ import ( "fmt" "strings" "testing" + "time" + + "github.com/arangodb/arangosync/pkg/retry" "github.com/dchest/uniuri" "github.com/stretchr/testify/assert" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" kubeArangoClient "github.com/arangodb/kube-arangodb/pkg/client" + corev1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" //"github.com/arangodb/kube-arangodb/pkg/util" ) // TODO - add description -func TestPersistence(t *testing.T) { +func TestPVCExists(t *testing.T) { longOrSkip(t) k8sNameSpace := getNamespace(t) @@ -78,3 +85,86 @@ func TestPersistence(t *testing.T) { // Cleanup removeDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace) } + +func TestPVCResize(t *testing.T) { + longOrSkip(t) + + k8sNameSpace := getNamespace(t) + k8sClient := mustNewKubeClient(t) + + mode := api.DeploymentModeCluster + engine := api.StorageEngineRocksDB + + size10GB, _ := resource.ParseQuantity("10Gi") + size08GB, _ := resource.ParseQuantity("8Gi") + + deploymentClient := kubeArangoClient.MustNewInCluster() + deploymentTemplate := newDeployment(strings.Replace(fmt.Sprintf("trsz-%s-%s-%s", mode[:2], engine[:2], uniuri.NewLen(4)), ".", "", -1)) + deploymentTemplate.Spec.Mode = api.NewMode(mode) + deploymentTemplate.Spec.StorageEngine = api.NewStorageEngine(engine) + deploymentTemplate.Spec.TLS = api.TLSSpec{} + deploymentTemplate.Spec.DBServers.Resources.Requests = corev1.ResourceList{corev1.ResourceStorage: size08GB} + deploymentTemplate.Spec.SetDefaults(deploymentTemplate.GetName()) // this must be last + assert.NoError(t, deploymentTemplate.Spec.Validate()) + + // Create deployment + _, err := deploymentClient.DatabaseV1alpha().ArangoDeployments(k8sNameSpace).Create(deploymentTemplate) + defer removeDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace) + assert.NoError(t, err, "failed to create deplyment: %s", err) + + depl, err := waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady()) + assert.NoError(t, err, fmt.Sprintf("Deployment not running in time: %s", err)) + + // Get list of all pvcs for dbservers + for _, m := range depl.Status.Members.DBServers { + pvc, err := k8sClient.CoreV1().PersistentVolumeClaims(k8sNameSpace).Get(m.PersistentVolumeClaimName, metav1.GetOptions{}) + assert.NoError(t, err, "failed to get pvc: %s", err) + volumeSize, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + assert.True(t, ok, "pvc does not have storage resource") + assert.True(t, volumeSize.Cmp(size08GB) == 0, "wrong volume size: expected: %s, found: %s", size08GB.String(), volumeSize.String()) + } + + // Update the deployment + // Try to change image version + depl, err = updateDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, + func(depl *api.DeploymentSpec) { + depl.DBServers.Resources.Requests[corev1.ResourceStorage] = size10GB + }) + if err != nil { + t.Fatalf("Failed to update the deployment") + } else { + t.Log("Updated deployment") + } + + if err := retry.Retry(func() error { + // Get list of all pvcs for dbservers and check for new size + for _, m := range depl.Status.Members.DBServers { + pvc, err := k8sClient.CoreV1().PersistentVolumeClaims(k8sNameSpace).Get(m.PersistentVolumeClaimName, metav1.GetOptions{}) + if err != nil { + return err + } + volumeSize, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + if !ok { + return fmt.Errorf("pvc does not have storage resource") + } + if volumeSize.Cmp(size10GB) != 0 { + return fmt.Errorf("wrong pvc size: expected: %s, found: %s", size10GB.String(), volumeSize.String()) + } + volume, err := k8sClient.CoreV1().PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{}) + if err != nil { + return err + } + volumeSize, ok = volume.Spec.Capacity[corev1.ResourceStorage] + if !ok { + return fmt.Errorf("pv does not have storage resource") + } + if volumeSize.Cmp(size10GB) != 0 { + return fmt.Errorf("wrong volume size: expected: %s, found: %s", size10GB.String(), volumeSize.String()) + } + } + return nil + }, 1*time.Minute); err != nil { + t.Fatalf("PVCs not resized: %s", err.Error()) + } + +} From 82376fd82258f09bf5ab8dd3e52f4d2a2bd421d0 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Mon, 18 Mar 2019 11:02:14 +0100 Subject: [PATCH 2/6] Don't crash is Storage is not set. --- pkg/deployment/resources/pvc_inspector.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/deployment/resources/pvc_inspector.go b/pkg/deployment/resources/pvc_inspector.go index 38ec8634f..448af944b 100644 --- a/pkg/deployment/resources/pvc_inspector.go +++ b/pkg/deployment/resources/pvc_inspector.go @@ -83,17 +83,18 @@ func (r *Resources) InspectPVCs(ctx context.Context) (util.Interval, error) { // Resize inspector groupSpec := spec.GetServerGroupSpec(group) - if volumeSize, ok := p.Spec.Resources.Requests[apiv1.ResourceStorage]; ok { - requestedSize := groupSpec.Resources.Requests[apiv1.ResourceStorage] - if volumeSize.Cmp(requestedSize) < 0 { - // Size of the volume is smaller than the requested size - // Update the pvc with the request size - p.Spec.Resources.Requests[apiv1.ResourceStorage] = requestedSize + if requestedSize, ok := groupSpec.Resources.Requests[apiv1.ResourceStorage]; ok { + if volumeSize, ok := p.Spec.Resources.Requests[apiv1.ResourceStorage]; ok { + if volumeSize.Cmp(requestedSize) < 0 { + // Size of the volume is smaller than the requested size + // Update the pvc with the request size + p.Spec.Resources.Requests[apiv1.ResourceStorage] = requestedSize - log.Debug().Str("pvc-capacity", volumeSize.String()).Str("requested", requestedSize.String()).Msg("PVC capacity differes - updating") - kube := r.context.GetKubeCli() - if _, err := kube.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()).Update(&p); err != nil { - log.Error().Err(err).Msg("Failed to update pvc") + log.Debug().Str("pvc-capacity", volumeSize.String()).Str("requested", requestedSize.String()).Msg("PVC capacity differes - updating") + kube := r.context.GetKubeCli() + if _, err := kube.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()).Update(&p); err != nil { + log.Error().Err(err).Msg("Failed to update pvc") + } } } } From 80ba00b9d477ddfe548ab2972eebdfd0b968a13d Mon Sep 17 00:00:00 2001 From: lamai93 Date: Mon, 18 Mar 2019 12:25:07 +0100 Subject: [PATCH 3/6] Create rotate member plan when pvc have a file system resize pending. --- pkg/deployment/reconcile/plan_builder_storage.go | 15 ++++++++++----- pkg/util/k8sutil/pvc.go | 12 +++++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/pkg/deployment/reconcile/plan_builder_storage.go b/pkg/deployment/reconcile/plan_builder_storage.go index 87090785c..132e0b0c7 100644 --- a/pkg/deployment/reconcile/plan_builder_storage.go +++ b/pkg/deployment/reconcile/plan_builder_storage.go @@ -57,10 +57,6 @@ func createRotateServerStoragePlan(log zerolog.Logger, apiObject k8sutil.APIObje } groupSpec := spec.GetServerGroupSpec(group) storageClassName := groupSpec.GetStorageClassName() - if storageClassName == "" { - // Using default storage class name - continue - } // Load PVC pvc, err := getPVC(m.PersistentVolumeClaimName) if err != nil { @@ -71,10 +67,17 @@ func createRotateServerStoragePlan(log zerolog.Logger, apiObject k8sutil.APIObje continue } replacementNeeded := false - if util.StringOrDefault(pvc.Spec.StorageClassName) != storageClassName { + if util.StringOrDefault(pvc.Spec.StorageClassName) != storageClassName && storageClassName != "" { // Storageclass has changed + log.Debug().Str("pod-name", m.PodName). + Str("pvc-storage-class", util.StringOrDefault(pvc.Spec.StorageClassName)). + Str("group-storage-class", storageClassName).Msg("Storage class has changed - pod needs replacement") replacementNeeded = true } + rotationNeeded := false + if k8sutil.IsPersistentVolumeClaimFileSystemResizePending(pvc) { + rotationNeeded = true + } if replacementNeeded { if group != api.ServerGroupAgents && group != api.ServerGroupDBServers { // Only agents & dbservers are allowed to change their storage class. @@ -107,6 +110,8 @@ func createRotateServerStoragePlan(log zerolog.Logger, apiObject k8sutil.APIObje ) } } + } else if rotationNeeded { + plan = createRotateMemberPlan(log, m, group, "Filesystem resize pending") } } return nil diff --git a/pkg/util/k8sutil/pvc.go b/pkg/util/k8sutil/pvc.go index b175e83d8..b9ae8c1ea 100644 --- a/pkg/util/k8sutil/pvc.go +++ b/pkg/util/k8sutil/pvc.go @@ -37,11 +37,21 @@ type PersistentVolumeClaimInterface interface { Get(name string, options metav1.GetOptions) (*v1.PersistentVolumeClaim, error) } -// IsPersistentVolumeClaimMarkedForDeletion returns true if the pod has been marked for deletion. +// IsPersistentVolumeClaimMarkedForDeletion returns true if the pvc has been marked for deletion. func IsPersistentVolumeClaimMarkedForDeletion(pvc *v1.PersistentVolumeClaim) bool { return pvc.DeletionTimestamp != nil } +// IsPersistentVolumeClaimFileSystemResizePending returns true if the pvc has FileSystemResizePending set to true +func IsPersistentVolumeClaimFileSystemResizePending(pvc *v1.PersistentVolumeClaim) bool { + for _, c := range pvc.Status.Conditions { + if c.Type == v1.PersistentVolumeClaimFileSystemResizePending && c.Status == v1.ConditionTrue { + return true + } + } + return false +} + // CreatePersistentVolumeClaimName returns the name of the persistent volume claim for a member with // a given id in a deployment with a given name. func CreatePersistentVolumeClaimName(deploymentName, role, id string) string { From 1c1a6f132f4b347f38c406a7d044decd3268cc1f Mon Sep 17 00:00:00 2001 From: lamai93 Date: Mon, 18 Mar 2019 12:46:45 +0100 Subject: [PATCH 4/6] Added event when pvc is resized. Added log if the user tries to shrink a volume. --- pkg/deployment/resources/pvc_inspector.go | 9 +++++++-- pkg/util/k8sutil/events.go | 9 +++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pkg/deployment/resources/pvc_inspector.go b/pkg/deployment/resources/pvc_inspector.go index 448af944b..de205b953 100644 --- a/pkg/deployment/resources/pvc_inspector.go +++ b/pkg/deployment/resources/pvc_inspector.go @@ -85,16 +85,21 @@ func (r *Resources) InspectPVCs(ctx context.Context) (util.Interval, error) { groupSpec := spec.GetServerGroupSpec(group) if requestedSize, ok := groupSpec.Resources.Requests[apiv1.ResourceStorage]; ok { if volumeSize, ok := p.Spec.Resources.Requests[apiv1.ResourceStorage]; ok { - if volumeSize.Cmp(requestedSize) < 0 { + cmp := volumeSize.Cmp(requestedSize) + if cmp < 0 { // Size of the volume is smaller than the requested size // Update the pvc with the request size p.Spec.Resources.Requests[apiv1.ResourceStorage] = requestedSize - log.Debug().Str("pvc-capacity", volumeSize.String()).Str("requested", requestedSize.String()).Msg("PVC capacity differes - updating") + log.Debug().Str("pvc-capacity", volumeSize.String()).Str("requested", requestedSize.String()).Msg("PVC capacity differs - updating") kube := r.context.GetKubeCli() if _, err := kube.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()).Update(&p); err != nil { log.Error().Err(err).Msg("Failed to update pvc") } + r.context.CreateEvent(k8sutil.NewPVCResizedEvent(r.context.GetAPIObject(), p.Name)) + } else if cmp > 0 { + log.Error().Str("server-group", group.AsRole()).Str("pvc-storage-size", volumeSize.String()).Str("requested-size", requestedSize.String()). + Msg("Volume size should not shrink") } } } diff --git a/pkg/util/k8sutil/events.go b/pkg/util/k8sutil/events.go index ca4352d13..ff9b19547 100644 --- a/pkg/util/k8sutil/events.go +++ b/pkg/util/k8sutil/events.go @@ -193,6 +193,15 @@ func NewDowntimeNotAllowedEvent(apiObject APIObject, operation string) *Event { return event } +// NewPVCResizedEvent creates an event indicating that a PVC has been resized +func NewPVCResizedEvent(apiObject APIObject, pvcname string) *Event { + event := newDeploymentEvent(apiObject) + event.Type = v1.EventTypeNormal + event.Reason = "PVC Resized" + event.Message = fmt.Sprintf("The persistent volume claim %s has been resized", pvcname) + return event +} + // NewUpgradeNotAllowedEvent creates an event indicating that an upgrade (or downgrade) is not allowed. func NewUpgradeNotAllowedEvent(apiObject APIObject, fromVersion, toVersion driver.Version, From 571c0cb579f7a4b1010a3e915f7608cd9747ef87 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Mon, 18 Mar 2019 13:35:45 +0100 Subject: [PATCH 5/6] Tests wait for PVC to be resized. --- tests/persistent_volumes_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/persistent_volumes_test.go b/tests/persistent_volumes_test.go index 4d9062553..2804564e2 100644 --- a/tests/persistent_volumes_test.go +++ b/tests/persistent_volumes_test.go @@ -34,8 +34,8 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" kubeArangoClient "github.com/arangodb/kube-arangodb/pkg/client" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" //"github.com/arangodb/kube-arangodb/pkg/util" @@ -161,9 +161,12 @@ func TestPVCResize(t *testing.T) { if volumeSize.Cmp(size10GB) != 0 { return fmt.Errorf("wrong volume size: expected: %s, found: %s", size10GB.String(), volumeSize.String()) } + if k8sutil.IsPersistentVolumeClaimFileSystemResizePending(pvc) { + return fmt.Errorf("persistent volume claim file system resize pending") + } } return nil - }, 1*time.Minute); err != nil { + }, 5*time.Minute); err != nil { t.Fatalf("PVCs not resized: %s", err.Error()) } From e8025978016d1f1e7513384558db2dccc3c7dcc9 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Mon, 18 Mar 2019 15:57:28 +0100 Subject: [PATCH 6/6] Added CannotShrink Event. --- pkg/deployment/resources/pvc_inspector.go | 1 + pkg/util/k8sutil/events.go | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/pkg/deployment/resources/pvc_inspector.go b/pkg/deployment/resources/pvc_inspector.go index de205b953..495569602 100644 --- a/pkg/deployment/resources/pvc_inspector.go +++ b/pkg/deployment/resources/pvc_inspector.go @@ -100,6 +100,7 @@ func (r *Resources) InspectPVCs(ctx context.Context) (util.Interval, error) { } else if cmp > 0 { log.Error().Str("server-group", group.AsRole()).Str("pvc-storage-size", volumeSize.String()).Str("requested-size", requestedSize.String()). Msg("Volume size should not shrink") + r.context.CreateEvent(k8sutil.NewCannotShrinkVolumeEvent(r.context.GetAPIObject(), p.Name)) } } } diff --git a/pkg/util/k8sutil/events.go b/pkg/util/k8sutil/events.go index ff9b19547..c61af0b14 100644 --- a/pkg/util/k8sutil/events.go +++ b/pkg/util/k8sutil/events.go @@ -202,6 +202,15 @@ func NewPVCResizedEvent(apiObject APIObject, pvcname string) *Event { return event } +// NewCannotShrinkVolumeEvent creates an event indicating that the user tried to shrink a PVC +func NewCannotShrinkVolumeEvent(apiObject APIObject, pvcname string) *Event { + event := newDeploymentEvent(apiObject) + event.Type = v1.EventTypeNormal + event.Reason = "PVC Shrinked" + event.Message = fmt.Sprintf("The persistent volume claim %s can not be shrinked", pvcname) + return event +} + // NewUpgradeNotAllowedEvent creates an event indicating that an upgrade (or downgrade) is not allowed. func NewUpgradeNotAllowedEvent(apiObject APIObject, fromVersion, toVersion driver.Version,