From 2f70902bccd02e94f0e28e9c17e3bdf069c6f515 Mon Sep 17 00:00:00 2001 From: jmckulk Date: Fri, 5 Sep 2025 12:24:37 -0400 Subject: [PATCH 1/2] Add support for Auto Grow Tuning - Add new fields for Autogrow that allow configuring usage trigger and maxGrow size. - Update pgdata, pgwal, and repo volume fields to use new types - Update tests to account for new fields - Update Sidecar bash logic to use new autogrow fields --- ...ator.crunchydata.com_postgresclusters.yaml | 200 ++++++++++++++++++ .../postgrescluster/autogrow_test.go | 81 +++---- .../postgrescluster/cluster_test.go | 16 +- .../postgrescluster/helpers_test.go | 10 +- .../postgrescluster/instance_test.go | 52 +++-- .../postgrescluster/patroni_test.go | 22 +- .../postgrescluster/pgadmin_test.go | 14 +- .../postgrescluster/pgbackrest_test.go | 71 ++++--- .../postgrescluster/volumes_test.go | 68 +++--- internal/pgbackrest/config.go | 67 +++++- internal/pgbackrest/config_test.go | 61 +++++- internal/pgbackrest/pgbackrest_test.go | 12 +- internal/pgbackrest/reconcile.go | 7 +- internal/pgbackrest/reconcile_test.go | 66 ++++-- internal/postgres/config.go | 43 ++-- internal/postgres/config_test.go | 2 +- internal/postgres/reconcile.go | 6 +- internal/postgres/reconcile_test.go | 27 ++- internal/util/volumes.go | 41 ++++ internal/util/volumes_test.go | 86 ++++++++ .../v1/postgrescluster_types.go | 4 +- .../v1beta1/pgbackrest_types.go | 2 +- .../v1beta1/postgrescluster_types.go | 4 +- .../v1beta1/shared_types.go | 63 ++++++ .../v1beta1/shared_types_test.go | 46 ++++ .../v1beta1/zz_generated.deepcopy.go | 20 ++ 26 files changed, 885 insertions(+), 206 deletions(-) diff --git a/config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml b/config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml index facd00f95e..b7915891c6 100644 --- a/config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml +++ b/config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml @@ -3117,6 +3117,31 @@ spec: type: string type: array x-kubernetes-list-type: atomic + autoGrow: + description: |- + AutoGrowSpec provides options to tune volume auto-growing behavior. + Auto grow requires that a limit be set on the PVC. + properties: + maxGrow: + anyOf: + - type: integer + - type: string + description: |- + MaxGrow is the maximum size to which the volume can be automatically + expanded. If not set, the volume will grow by 50% of the original size each + time the Trigger threshold is exceeded. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + trigger: + default: 75 + description: |- + Trigger is the percentage of used space at which to trigger a volume + expansion. + format: int32 + maximum: 90 + minimum: 50 + type: integer + type: object dataSource: description: |- dataSource field can be used to specify either: @@ -6479,6 +6504,31 @@ spec: type: string type: array x-kubernetes-list-type: atomic + autoGrow: + description: |- + AutoGrowSpec provides options to tune volume auto-growing behavior. + Auto grow requires that a limit be set on the PVC. + properties: + maxGrow: + anyOf: + - type: integer + - type: string + description: |- + MaxGrow is the maximum size to which the volume can be automatically + expanded. If not set, the volume will grow by 50% of the original size each + time the Trigger threshold is exceeded. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + trigger: + default: 75 + description: |- + Trigger is the percentage of used space at which to trigger a volume + expansion. + format: int32 + maximum: 90 + minimum: 50 + type: integer + type: object dataSource: description: |- dataSource field can be used to specify either: @@ -10470,6 +10520,31 @@ spec: type: string type: array x-kubernetes-list-type: atomic + autoGrow: + description: |- + AutoGrowSpec provides options to tune volume auto-growing behavior. + Auto grow requires that a limit be set on the PVC. + properties: + maxGrow: + anyOf: + - type: integer + - type: string + description: |- + MaxGrow is the maximum size to which the volume can be automatically + expanded. If not set, the volume will grow by 50% of the original size each + time the Trigger threshold is exceeded. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + trigger: + default: 75 + description: |- + Trigger is the percentage of used space at which to trigger a volume + expansion. + format: int32 + maximum: 90 + minimum: 50 + type: integer + type: object dataSource: description: |- dataSource field can be used to specify either: @@ -11552,6 +11627,31 @@ spec: type: string type: array x-kubernetes-list-type: atomic + autoGrow: + description: |- + AutoGrowSpec provides options to tune volume auto-growing behavior. + Auto grow requires that a limit be set on the PVC. + properties: + maxGrow: + anyOf: + - type: integer + - type: string + description: |- + MaxGrow is the maximum size to which the volume can be automatically + expanded. If not set, the volume will grow by 50% of the original size each + time the Trigger threshold is exceeded. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + trigger: + default: 75 + description: |- + Trigger is the percentage of used space at which to trigger a volume + expansion. + format: int32 + maximum: 90 + minimum: 50 + type: integer + type: object dataSource: description: |- dataSource field can be used to specify either: @@ -22050,6 +22150,31 @@ spec: type: string type: array x-kubernetes-list-type: atomic + autoGrow: + description: |- + AutoGrowSpec provides options to tune volume auto-growing behavior. + Auto grow requires that a limit be set on the PVC. + properties: + maxGrow: + anyOf: + - type: integer + - type: string + description: |- + MaxGrow is the maximum size to which the volume can be automatically + expanded. If not set, the volume will grow by 50% of the original size each + time the Trigger threshold is exceeded. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + trigger: + default: 75 + description: |- + Trigger is the percentage of used space at which to trigger a volume + expansion. + format: int32 + maximum: 90 + minimum: 50 + type: integer + type: object dataSource: description: |- dataSource field can be used to specify either: @@ -25412,6 +25537,31 @@ spec: type: string type: array x-kubernetes-list-type: atomic + autoGrow: + description: |- + AutoGrowSpec provides options to tune volume auto-growing behavior. + Auto grow requires that a limit be set on the PVC. + properties: + maxGrow: + anyOf: + - type: integer + - type: string + description: |- + MaxGrow is the maximum size to which the volume can be automatically + expanded. If not set, the volume will grow by 50% of the original size each + time the Trigger threshold is exceeded. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + trigger: + default: 75 + description: |- + Trigger is the percentage of used space at which to trigger a volume + expansion. + format: int32 + maximum: 90 + minimum: 50 + type: integer + type: object dataSource: description: |- dataSource field can be used to specify either: @@ -29403,6 +29553,31 @@ spec: type: string type: array x-kubernetes-list-type: atomic + autoGrow: + description: |- + AutoGrowSpec provides options to tune volume auto-growing behavior. + Auto grow requires that a limit be set on the PVC. + properties: + maxGrow: + anyOf: + - type: integer + - type: string + description: |- + MaxGrow is the maximum size to which the volume can be automatically + expanded. If not set, the volume will grow by 50% of the original size each + time the Trigger threshold is exceeded. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + trigger: + default: 75 + description: |- + Trigger is the percentage of used space at which to trigger a volume + expansion. + format: int32 + maximum: 90 + minimum: 50 + type: integer + type: object dataSource: description: |- dataSource field can be used to specify either: @@ -30485,6 +30660,31 @@ spec: type: string type: array x-kubernetes-list-type: atomic + autoGrow: + description: |- + AutoGrowSpec provides options to tune volume auto-growing behavior. + Auto grow requires that a limit be set on the PVC. + properties: + maxGrow: + anyOf: + - type: integer + - type: string + description: |- + MaxGrow is the maximum size to which the volume can be automatically + expanded. If not set, the volume will grow by 50% of the original size each + time the Trigger threshold is exceeded. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + trigger: + default: 75 + description: |- + Trigger is the percentage of used space at which to trigger a volume + expansion. + format: int32 + maximum: 90 + minimum: 50 + type: integer + type: object dataSource: description: |- dataSource field can be used to specify either: diff --git a/internal/controller/postgrescluster/autogrow_test.go b/internal/controller/postgrescluster/autogrow_test.go index 3e75df6326..764d8aadde 100644 --- a/internal/controller/postgrescluster/autogrow_test.go +++ b/internal/controller/postgrescluster/autogrow_test.go @@ -45,22 +45,24 @@ func TestStoreDesiredRequest(t *testing.T) { InstanceSets: []v1beta1.PostgresInstanceSetSpec{{ Name: "red", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Limits: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }}}, - WALVolumeClaimSpec: &v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Limits: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }}}, + DataVolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }}}}, + WALVolumeClaimSpec: &v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }}}}, }, { Name: "blue", Replicas: initialize.Int32(1), - WALVolumeClaimSpec: &v1beta1.VolumeClaimSpec{}, + WALVolumeClaimSpec: &v1beta1.VolumeClaimSpecWithAutoGrow{}, }, { Name: "green", Replicas: initialize.Int32(1), @@ -70,12 +72,13 @@ func TestStoreDesiredRequest(t *testing.T) { Repos: []v1beta1.PGBackRestRepo{{ Name: "repo1", Volume: &v1beta1.RepoPVC{ - VolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Limits: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }}}, + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }}}}, }, }, { Name: "repo2", @@ -238,36 +241,40 @@ func TestLimitIsSet(t *testing.T) { InstanceSets: []v1beta1.PostgresInstanceSetSpec{{ Name: "red", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Limits: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }}}, - WALVolumeClaimSpec: &v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Limits: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceStorage: resource.MustParse("2Gi"), - }}}, + DataVolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }}}}, + WALVolumeClaimSpec: &v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("2Gi"), + }}}}, }, { Name: "blue", Replicas: initialize.Int32(1), }, { Name: "green", Replicas: initialize.Int32(1), - WALVolumeClaimSpec: &v1beta1.VolumeClaimSpec{}, + WALVolumeClaimSpec: &v1beta1.VolumeClaimSpecWithAutoGrow{}, }}, Backups: v1beta1.Backups{ PGBackRest: v1beta1.PGBackRestArchive{ Repos: []v1beta1.PGBackRestRepo{{ Name: "repo1", Volume: &v1beta1.RepoPVC{ - VolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Limits: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceStorage: resource.MustParse("1Gi"), + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, }, }, }}}, diff --git a/internal/controller/postgrescluster/cluster_test.go b/internal/controller/postgrescluster/cluster_test.go index 5fa92d32cf..b819291ae4 100644 --- a/internal/controller/postgrescluster/cluster_test.go +++ b/internal/controller/postgrescluster/cluster_test.go @@ -142,11 +142,11 @@ func TestCustomLabels(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "daisy-instance1", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }, { Name: "daisy-instance2", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} cluster.Spec.Metadata = &v1beta1.Metadata{ Labels: map[string]string{"my.cluster.label": "daisy"}, @@ -190,14 +190,14 @@ func TestCustomLabels(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "max-instance", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), Metadata: &v1beta1.Metadata{ Labels: map[string]string{"my.instance.label": "max"}, }, }, { Name: "lucy-instance", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), Metadata: &v1beta1.Metadata{ Labels: map[string]string{"my.instance.label": "lucy"}, }, @@ -380,11 +380,11 @@ func TestCustomAnnotations(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "daisy-instance1", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }, { Name: "daisy-instance2", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} cluster.Spec.Metadata = &v1beta1.Metadata{ Annotations: map[string]string{"my.cluster.annotation": "daisy"}, @@ -429,14 +429,14 @@ func TestCustomAnnotations(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "max-instance", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), Metadata: &v1beta1.Metadata{ Annotations: map[string]string{"my.instance.annotation": "max"}, }, }, { Name: "lucy-instance", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), Metadata: &v1beta1.Metadata{ Annotations: map[string]string{"my.instance.annotation": "lucy"}, }, diff --git a/internal/controller/postgrescluster/helpers_test.go b/internal/controller/postgrescluster/helpers_test.go index 4542f651a9..9f7d177627 100644 --- a/internal/controller/postgrescluster/helpers_test.go +++ b/internal/controller/postgrescluster/helpers_test.go @@ -102,6 +102,12 @@ func testVolumeClaimSpec() v1beta1.VolumeClaimSpec { } } +func testVolumeClaimSpecWithAutoGrow() v1beta1.VolumeClaimSpecWithAutoGrow { + return v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: testVolumeClaimSpec(), + } +} + func testCluster() *v1beta1.PostgresCluster { // Defines a base cluster spec that can be used by tests to generate a // cluster with an expected number of instances @@ -118,7 +124,7 @@ func testCluster() *v1beta1.PostgresCluster { InstanceSets: []v1beta1.PostgresInstanceSetSpec{{ Name: "instance1", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }}, Backups: v1beta1.Backups{ PGBackRest: v1beta1.PGBackRestArchive{ @@ -126,7 +132,7 @@ func testCluster() *v1beta1.PostgresCluster { Repos: []v1beta1.PGBackRestRepo{{ Name: "repo1", Volume: &v1beta1.RepoPVC{ - VolumeClaimSpec: testVolumeClaimSpec(), + VolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }, }}, }, diff --git a/internal/controller/postgrescluster/instance_test.go b/internal/controller/postgrescluster/instance_test.go index 0b24e38d72..1d17e4f9f3 100644 --- a/internal/controller/postgrescluster/instance_test.go +++ b/internal/controller/postgrescluster/instance_test.go @@ -558,6 +558,8 @@ func TestAddPGBackRestToInstancePodSpec(t *testing.T) { # Return size in Mebibytes. manageAutogrowAnnotation() { local volume=$1 + local trigger=$2 + local maxGrow=$3 size=$(df --block-size=M "/pgbackrest/${volume}") read -r _ size _ <<< "${size#*$'\n'}" @@ -566,9 +568,19 @@ func TestAddPGBackRestToInstancePodSpec(t *testing.T) { sizeInt="${size//M/}" # Use the sed punctuation class, because the shell will not accept the percent sign in an expansion. useInt=${use//[[:punct:]]/} - triggerExpansion="$((useInt > 75))" + triggerExpansion="$((useInt > trigger))" if [[ ${triggerExpansion} -eq 1 ]]; then newSize="$(((sizeInt / 2)+sizeInt))" + # Only compare with maxGrow if it is set (not empty) + if [[ -n "${maxGrow}" ]]; then + # check to see how much we would normally grow + sizeDiff=$((newSize - sizeInt)) + + # Compare the size difference to the maxGrow; if it is greater, cap it to maxGrow + if [[ ${sizeDiff} -gt ${maxGrow} ]]; then + newSize=$((sizeInt + maxGrow)) + fi + fi newSizeMi="${newSize}Mi" d='[{"op": "add", "path": "/metadata/annotations/suggested-'"${volume}"'-pvc-size", "value": "'"${newSizeMi}"'"}]' curl --cacert "${CACERT}" --header "Authorization: Bearer ${TOKEN}" -XPATCH "${APISERVER}/api/v1/namespaces/${NAMESPACE}/pods/${HOSTNAME}?fieldManager=kubectl-annotate" -H "Content-Type: application/json-patch+json" --data "${d}" @@ -595,22 +607,22 @@ func TestAddPGBackRestToInstancePodSpec(t *testing.T) { # manage autogrow annotation for the repo1 volume, if it exists if [[ -d /pgbackrest/repo1 ]]; then - manageAutogrowAnnotation "repo1" + manageAutogrowAnnotation "repo1" "75" "" fi # manage autogrow annotation for the repo2 volume, if it exists if [[ -d /pgbackrest/repo2 ]]; then - manageAutogrowAnnotation "repo2" + manageAutogrowAnnotation "repo2" "75" "" fi # manage autogrow annotation for the repo3 volume, if it exists if [[ -d /pgbackrest/repo3 ]]; then - manageAutogrowAnnotation "repo3" + manageAutogrowAnnotation "repo3" "75" "" fi # manage autogrow annotation for the repo4 volume, if it exists if [[ -d /pgbackrest/repo4 ]]; then - manageAutogrowAnnotation "repo4" + manageAutogrowAnnotation "repo4" "75" "" fi done @@ -724,6 +736,8 @@ func TestAddPGBackRestToInstancePodSpec(t *testing.T) { # Return size in Mebibytes. manageAutogrowAnnotation() { local volume=$1 + local trigger=$2 + local maxGrow=$3 size=$(df --block-size=M "/pgbackrest/${volume}") read -r _ size _ <<< "${size#*$'\n'}" @@ -732,9 +746,19 @@ func TestAddPGBackRestToInstancePodSpec(t *testing.T) { sizeInt="${size//M/}" # Use the sed punctuation class, because the shell will not accept the percent sign in an expansion. useInt=${use//[[:punct:]]/} - triggerExpansion="$((useInt > 75))" + triggerExpansion="$((useInt > trigger))" if [[ ${triggerExpansion} -eq 1 ]]; then newSize="$(((sizeInt / 2)+sizeInt))" + # Only compare with maxGrow if it is set (not empty) + if [[ -n "${maxGrow}" ]]; then + # check to see how much we would normally grow + sizeDiff=$((newSize - sizeInt)) + + # Compare the size difference to the maxGrow; if it is greater, cap it to maxGrow + if [[ ${sizeDiff} -gt ${maxGrow} ]]; then + newSize=$((sizeInt + maxGrow)) + fi + fi newSizeMi="${newSize}Mi" d='[{"op": "add", "path": "/metadata/annotations/suggested-'"${volume}"'-pvc-size", "value": "'"${newSizeMi}"'"}]' curl --cacert "${CACERT}" --header "Authorization: Bearer ${TOKEN}" -XPATCH "${APISERVER}/api/v1/namespaces/${NAMESPACE}/pods/${HOSTNAME}?fieldManager=kubectl-annotate" -H "Content-Type: application/json-patch+json" --data "${d}" @@ -761,22 +785,22 @@ func TestAddPGBackRestToInstancePodSpec(t *testing.T) { # manage autogrow annotation for the repo1 volume, if it exists if [[ -d /pgbackrest/repo1 ]]; then - manageAutogrowAnnotation "repo1" + manageAutogrowAnnotation "repo1" "75" "" fi # manage autogrow annotation for the repo2 volume, if it exists if [[ -d /pgbackrest/repo2 ]]; then - manageAutogrowAnnotation "repo2" + manageAutogrowAnnotation "repo2" "75" "" fi # manage autogrow annotation for the repo3 volume, if it exists if [[ -d /pgbackrest/repo3 ]]; then - manageAutogrowAnnotation "repo3" + manageAutogrowAnnotation "repo3" "75" "" fi # manage autogrow annotation for the repo4 volume, if it exists if [[ -d /pgbackrest/repo4 ]]; then - manageAutogrowAnnotation "repo4" + manageAutogrowAnnotation "repo4" "75" "" fi done @@ -1514,7 +1538,7 @@ func TestGenerateInstanceStatefulSetIntent(t *testing.T) { InstanceSets: []v1beta1.PostgresInstanceSetSpec{{ Name: "instance1", Replicas: initialize.Int32(1), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{ MaxSkew: int32(1), TopologyKey: "kubernetes.io/hostname", @@ -1703,7 +1727,7 @@ func TestFindAvailableInstanceNames(t *testing.T) { expectedInstanceNames: []string{"instance1-def"}, }, { set: v1beta1.PostgresInstanceSetSpec{Name: "instance1", - WALVolumeClaimSpec: &v1beta1.VolumeClaimSpec{}}, + WALVolumeClaimSpec: &v1beta1.VolumeClaimSpecWithAutoGrow{}}, fakeObservedInstances: newObservedInstances( &v1beta1.PostgresCluster{Spec: v1beta1.PostgresClusterSpec{ InstanceSets: []v1beta1.PostgresInstanceSetSpec{{Name: "instance1"}}, @@ -1730,7 +1754,7 @@ func TestFindAvailableInstanceNames(t *testing.T) { expectedInstanceNames: []string{}, }, { set: v1beta1.PostgresInstanceSetSpec{Name: "instance1", - WALVolumeClaimSpec: &v1beta1.VolumeClaimSpec{}}, + WALVolumeClaimSpec: &v1beta1.VolumeClaimSpecWithAutoGrow{}}, fakeObservedInstances: newObservedInstances( &v1beta1.PostgresCluster{Spec: v1beta1.PostgresClusterSpec{ InstanceSets: []v1beta1.PostgresInstanceSetSpec{{Name: "instance1"}}, @@ -1754,7 +1778,7 @@ func TestFindAvailableInstanceNames(t *testing.T) { expectedInstanceNames: []string{"instance1-def"}, }, { set: v1beta1.PostgresInstanceSetSpec{Name: "instance1", - WALVolumeClaimSpec: &v1beta1.VolumeClaimSpec{}}, + WALVolumeClaimSpec: &v1beta1.VolumeClaimSpecWithAutoGrow{}}, fakeObservedInstances: newObservedInstances( &v1beta1.PostgresCluster{Spec: v1beta1.PostgresClusterSpec{ InstanceSets: []v1beta1.PostgresInstanceSetSpec{{Name: "instance1"}}, diff --git a/internal/controller/postgrescluster/patroni_test.go b/internal/controller/postgrescluster/patroni_test.go index 728b75aee3..b7fe885305 100644 --- a/internal/controller/postgrescluster/patroni_test.go +++ b/internal/controller/postgrescluster/patroni_test.go @@ -661,7 +661,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} if test.enabled { cluster.Spec.Patroni = &v1beta1.PatroniSpec{ @@ -775,7 +775,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} timelineCall, timelineCallNoLeader = false, false called, failover, callError, callFails = false, false, true, false @@ -798,7 +798,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} timelineCall, timelineCallNoLeader = false, true called, failover, callError, callFails = false, false, false, false @@ -821,7 +821,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} timelineCall, timelineCallNoLeader = true, false called, failover, callError, callFails = false, false, false, false @@ -844,7 +844,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} cluster.Status.Patroni.SwitchoverTimeline = initialize.Int64(11) timelineCall, timelineCallNoLeader = true, false @@ -868,7 +868,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} cluster.Status.Patroni.SwitchoverTimeline = initialize.Int64(11) timelineCall, timelineCallNoLeader = true, false @@ -892,7 +892,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} cluster.Status.Patroni.SwitchoverTimeline = initialize.Int64(4) timelineCall, timelineCallNoLeader = true, false @@ -917,7 +917,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} cluster.Status.Patroni.SwitchoverTimeline = initialize.Int64(4) timelineCall, timelineCallNoLeader = true, false @@ -941,7 +941,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} cluster.Status.Patroni.SwitchoverTimeline = initialize.Int64(4) timelineCall, timelineCallNoLeader = true, false @@ -966,7 +966,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} cluster.Status.Patroni.SwitchoverTimeline = initialize.Int64(4) timelineCall, timelineCallNoLeader = true, false @@ -992,7 +992,7 @@ func TestReconcilePatroniSwitchover(t *testing.T) { cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{ Name: "target", Replicas: initialize.Int32(2), - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }} cluster.Status.Patroni.SwitchoverTimeline = initialize.Int64(4) timelineCall, timelineCallNoLeader = true, false diff --git a/internal/controller/postgrescluster/pgadmin_test.go b/internal/controller/postgrescluster/pgadmin_test.go index 1d0a305b2a..fc585d8952 100644 --- a/internal/controller/postgrescluster/pgadmin_test.go +++ b/internal/controller/postgrescluster/pgadmin_test.go @@ -846,18 +846,20 @@ func pgAdminTestCluster(ns corev1.Namespace) *v1beta1.PostgresCluster { Spec: v1beta1.PostgresClusterSpec{ PostgresVersion: 13, InstanceSets: []v1beta1.PostgresInstanceSetSpec{{ - DataVolumeClaimSpec: testVolumeClaimSpec(), + DataVolumeClaimSpec: testVolumeClaimSpecWithAutoGrow(), }}, Backups: v1beta1.Backups{ PGBackRest: v1beta1.PGBackRestArchive{ Repos: []v1beta1.PGBackRestRepo{{ Name: "repo1", Volume: &v1beta1.RepoPVC{ - VolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, }, }, }, diff --git a/internal/controller/postgrescluster/pgbackrest_test.go b/internal/controller/postgrescluster/pgbackrest_test.go index 0eac94ba7e..d87223a2ee 100644 --- a/internal/controller/postgrescluster/pgbackrest_test.go +++ b/internal/controller/postgrescluster/pgbackrest_test.go @@ -66,13 +66,13 @@ func fakePostgresCluster(clusterName, namespace, clusterUID string, Image: "example.com/crunchy-postgres-ha:test", InstanceSets: []v1beta1.PostgresInstanceSetSpec{{ Name: "instance1", - DataVolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, + DataVolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }}}, }, }}, Backups: v1beta1.Backups{ @@ -117,11 +117,13 @@ func fakePostgresCluster(clusterName, namespace, clusterUID string, postgresCluster.Spec.Backups.PGBackRest.Repos[0] = v1beta1.PGBackRestRepo{ Name: "repo1", Volume: &v1beta1.RepoPVC{ - VolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, - Resources: corev1.VolumeResourceRequirements{ - Requests: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceStorage: resource.MustParse("1Gi"), + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, }, }, }, @@ -2429,13 +2431,13 @@ func TestCopyConfigurationResources(t *testing.T) { Image: "example.com/crunchy-postgres-ha:test", InstanceSets: []v1beta1.PostgresInstanceSetSpec{{ Name: "instance1", - DataVolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, + DataVolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }}}, }, }}, Backups: v1beta1.Backups{ @@ -2481,13 +2483,13 @@ func TestCopyConfigurationResources(t *testing.T) { }, InstanceSets: []v1beta1.PostgresInstanceSetSpec{{ Name: "instance1", - DataVolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, + DataVolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }}}, }, }}, Backups: v1beta1.Backups{ @@ -2717,7 +2719,7 @@ volumes: spec := r.generateBackupJobSpecIntent(ctx, &cluster, v1beta1.PGBackRestRepo{ Volume: &v1beta1.RepoPVC{ - VolumeClaimSpec: v1beta1.VolumeClaimSpec{}, + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{}, }, }, "", @@ -4428,13 +4430,16 @@ func TestGetRepoHostVolumeRequests(t *testing.T) { // A limit is expected otherwise an empty string ("") is returned testRepoPVC := func() *v1beta1.RepoPVC { return &v1beta1.RepoPVC{ - VolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Limits: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceStorage: resource.MustParse("1Gi"), + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, }, - }}} + }, + }} } cluster := &v1beta1.PostgresCluster{ diff --git a/internal/controller/postgrescluster/volumes_test.go b/internal/controller/postgrescluster/volumes_test.go index 85087d079b..c579e3f578 100644 --- a/internal/controller/postgrescluster/volumes_test.go +++ b/internal/controller/postgrescluster/volumes_test.go @@ -391,14 +391,14 @@ func TestReconcileConfigureExistingPVCs(t *testing.T) { }, InstanceSets: []v1beta1.PostgresInstanceSetSpec{{ Name: "instance1", - DataVolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteMany}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, + DataVolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }}}, }, }}, Backups: v1beta1.Backups{ @@ -407,14 +407,16 @@ func TestReconcileConfigureExistingPVCs(t *testing.T) { Repos: []v1beta1.PGBackRestRepo{{ Name: "repo1", Volume: &v1beta1.RepoPVC{ - VolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteMany}, - Resources: corev1.VolumeResourceRequirements{ - Requests: map[corev1.ResourceName]resource. - Quantity{ - corev1.ResourceStorage: resource. - MustParse("1Gi"), + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource. + Quantity{ + corev1.ResourceStorage: resource. + MustParse("1Gi"), + }, }, }, }, @@ -674,14 +676,14 @@ func TestReconcileMoveDirectories(t *testing.T) { }, }, PriorityClassName: initialize.String("some-priority-class"), - DataVolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteMany}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, + DataVolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }}}, }, }}, Backups: v1beta1.Backups{ @@ -698,14 +700,16 @@ func TestReconcileMoveDirectories(t *testing.T) { Repos: []v1beta1.PGBackRestRepo{{ Name: "repo1", Volume: &v1beta1.RepoPVC{ - VolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteMany}, - Resources: corev1.VolumeResourceRequirements{ - Requests: map[corev1.ResourceName]resource. - Quantity{ - corev1.ResourceStorage: resource. - MustParse("1Gi"), + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource. + Quantity{ + corev1.ResourceStorage: resource. + MustParse("1Gi"), + }, }, }, }, diff --git a/internal/pgbackrest/config.go b/internal/pgbackrest/config.go index 40036b2abc..090f119d1c 100644 --- a/internal/pgbackrest/config.go +++ b/internal/pgbackrest/config.go @@ -21,6 +21,7 @@ import ( "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/internal/postgres" "github.com/crunchydata/postgres-operator/internal/shell" + "github.com/crunchydata/postgres-operator/internal/util" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -605,7 +606,36 @@ func getExternalRepoConfigs(repo v1beta1.PGBackRestRepo) map[string]string { // reloadCommand returns an entrypoint that convinces the pgBackRest TLS server // to reload its options and certificate files when they change. The process // will appear as name in `ps` and `top`. -func reloadCommand(name string) []string { +func reloadCommand(name string, repos []v1beta1.PGBackRestRepo) []string { + var repo1MaxGrow string + var repo2MaxGrow string + var repo3MaxGrow string + var repo4MaxGrow string + repo1Trigger := util.AutoGrowTriggerDefault + repo2Trigger := util.AutoGrowTriggerDefault + repo3Trigger := util.AutoGrowTriggerDefault + repo4Trigger := util.AutoGrowTriggerDefault + + // Loop through any repos and see if we have autogrow configured + // If we do, set the variables corresponding to each of the 4 repos + for _, repo := range repos { + // Only check repos that have a volume with autogrow configured + if repo.Volume == nil || repo.Volume.VolumeClaimSpec.AutoGrow == nil { + continue + } + spec := repo.Volume.VolumeClaimSpec + switch repo.Name { + case "repo1": + repo1Trigger, repo1MaxGrow = util.GetAutoGrowFromSpec(&spec) + case "repo2": + repo2Trigger, repo2MaxGrow = util.GetAutoGrowFromSpec(&spec) + case "repo3": + repo3Trigger, repo3MaxGrow = util.GetAutoGrowFromSpec(&spec) + case "repo4": + repo4Trigger, repo4MaxGrow = util.GetAutoGrowFromSpec(&spec) + } + } + // Use a Bash loop to periodically check the mtime of the mounted server // volume and configuration file. When either changes, signal pgBackRest // and print the observed timestamp. @@ -632,7 +662,7 @@ func reloadCommand(name string) []string { // for selective parsing of the provided lines. The percent value is stripped of // the '%' and then used to determine if a expansion should be triggered by // setting the calculated volume size using the 'size' variable. - const script = ` + script := fmt.Sprintf(` # Parameters for curl when managing autogrow annotation. APISERVER="https://kubernetes.default.svc" SERVICEACCOUNT="/var/run/secrets/kubernetes.io/serviceaccount" @@ -644,6 +674,8 @@ CACERT=${SERVICEACCOUNT}/ca.crt # Return size in Mebibytes. manageAutogrowAnnotation() { local volume=$1 + local trigger=$2 + local maxGrow=$3 size=$(df --block-size=M "/pgbackrest/${volume}") read -r _ size _ <<< "${size#*$'\n'}" @@ -652,9 +684,19 @@ manageAutogrowAnnotation() { sizeInt="${size//M/}" # Use the sed punctuation class, because the shell will not accept the percent sign in an expansion. useInt=${use//[[:punct:]]/} - triggerExpansion="$((useInt > 75))" + triggerExpansion="$((useInt > trigger))" if [[ ${triggerExpansion} -eq 1 ]]; then newSize="$(((sizeInt / 2)+sizeInt))" + # Only compare with maxGrow if it is set (not empty) + if [[ -n "${maxGrow}" ]]; then + # check to see how much we would normally grow + sizeDiff=$((newSize - sizeInt)) + + # Compare the size difference to the maxGrow; if it is greater, cap it to maxGrow + if [[ ${sizeDiff} -gt ${maxGrow} ]]; then + newSize=$((sizeInt + maxGrow)) + fi + fi newSizeMi="${newSize}Mi" d='[{"op": "add", "path": "/metadata/annotations/suggested-'"${volume}"'-pvc-size", "value": "'"${newSizeMi}"'"}]' curl --cacert "${CACERT}" --header "Authorization: Bearer ${TOKEN}" -XPATCH "${APISERVER}/api/v1/namespaces/${NAMESPACE}/pods/${HOSTNAME}?fieldManager=kubectl-annotate" -H "Content-Type: application/json-patch+json" --data "${d}" @@ -668,7 +710,7 @@ until read -r -t 5 -u "${fd}"; do pkill -HUP --exact --parent=0 pgbackrest then exec {fd}>&- && exec {fd}<> <(:||:) - stat --dereference --format='Loaded configuration dated %y' "${filename}" + stat --dereference --format='Loaded configuration dated %%y' "${filename}" elif { [[ "${directory}" -nt "/proc/self/fd/${fd}" ]] || [[ "${authority}" -nt "/proc/self/fd/${fd}" ]] @@ -676,31 +718,36 @@ until read -r -t 5 -u "${fd}"; do pkill -HUP --exact --parent=0 pgbackrest then exec {fd}>&- && exec {fd}<> <(:||:) - stat --format='Loaded certificates dated %y' "${directory}" + stat --format='Loaded certificates dated %%y' "${directory}" fi # manage autogrow annotation for the repo1 volume, if it exists if [[ -d /pgbackrest/repo1 ]]; then - manageAutogrowAnnotation "repo1" + manageAutogrowAnnotation "repo1" "%s" "%s" fi # manage autogrow annotation for the repo2 volume, if it exists if [[ -d /pgbackrest/repo2 ]]; then - manageAutogrowAnnotation "repo2" + manageAutogrowAnnotation "repo2" "%s" "%s" fi # manage autogrow annotation for the repo3 volume, if it exists if [[ -d /pgbackrest/repo3 ]]; then - manageAutogrowAnnotation "repo3" + manageAutogrowAnnotation "repo3" "%s" "%s" fi # manage autogrow annotation for the repo4 volume, if it exists if [[ -d /pgbackrest/repo4 ]]; then - manageAutogrowAnnotation "repo4" + manageAutogrowAnnotation "repo4" "%s" "%s" fi done -` +`, + repo1Trigger, repo1MaxGrow, + repo2Trigger, repo2MaxGrow, + repo3Trigger, repo3MaxGrow, + repo4Trigger, repo4MaxGrow, + ) // Elide the above script from `ps` and `top` by wrapping it in a function // and calling that. diff --git a/internal/pgbackrest/config_test.go b/internal/pgbackrest/config_test.go index c1b4e0b155..4617b3a80a 100644 --- a/internal/pgbackrest/config_test.go +++ b/internal/pgbackrest/config_test.go @@ -609,7 +609,58 @@ func TestMakePGBackrestLogDir(t *testing.T) { func TestReloadCommand(t *testing.T) { shellcheck := require.ShellCheck(t) - command := reloadCommand("some-name") + repo1Size := resource.MustParse("1Gi") + repo2Size := resource.MustParse("2Gi") + repo3Size := resource.MustParse("3Gi") + repo4Size := resource.MustParse("4Gi") + + // Create a command with all four repos having auto-grow enabled. + // Check that the generated script has the correct values for each repo + // when calling manageAutogrowAnnotation. + // + // Note that the actual values for trigger and max-grow are not important + // here, just that they are correctly passed through to the script. + command := reloadCommand("some-name", []v1beta1.PGBackRestRepo{{ + Name: "repo1", + Volume: &v1beta1.RepoPVC{ + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + AutoGrow: &v1beta1.AutoGrowSpec{ + Trigger: initialize.Int32(10), + MaxGrow: &repo1Size, + }, + }, + }, + }, { + Name: "repo2", + Volume: &v1beta1.RepoPVC{ + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + AutoGrow: &v1beta1.AutoGrowSpec{ + Trigger: initialize.Int32(20), + MaxGrow: &repo2Size, + }, + }, + }, + }, { + Name: "repo3", + Volume: &v1beta1.RepoPVC{ + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + AutoGrow: &v1beta1.AutoGrowSpec{ + Trigger: initialize.Int32(30), + MaxGrow: &repo3Size, + }, + }, + }, + }, { + Name: "repo4", + Volume: &v1beta1.RepoPVC{ + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + AutoGrow: &v1beta1.AutoGrowSpec{ + Trigger: initialize.Int32(40), + MaxGrow: &repo4Size, + }, + }, + }, + }}) // Expect a bash command with an inline script. assert.DeepEqual(t, command[:3], []string{"bash", "-ceu", "--"}) @@ -624,10 +675,16 @@ func TestReloadCommand(t *testing.T) { cmd := exec.CommandContext(t.Context(), shellcheck, "--enable=all", file) output, err := cmd.CombinedOutput() assert.NilError(t, err, "%q\n%s", cmd.Args, output) + + assert.Assert(t, cmp.Contains(command[3], "manageAutogrowAnnotation \"repo1\" \"10\" \"1024\"")) + assert.Assert(t, cmp.Contains(command[3], "manageAutogrowAnnotation \"repo2\" \"20\" \"2048\"")) + assert.Assert(t, cmp.Contains(command[3], "manageAutogrowAnnotation \"repo3\" \"30\" \"3072\"")) + assert.Assert(t, cmp.Contains(command[3], "manageAutogrowAnnotation \"repo4\" \"40\" \"4096\"")) + } func TestReloadCommandPrettyYAML(t *testing.T) { - assert.Assert(t, cmp.MarshalContains(reloadCommand("any"), "\n- |"), + assert.Assert(t, cmp.MarshalContains(reloadCommand("any", nil), "\n- |"), "expected literal block scalar") } diff --git a/internal/pgbackrest/pgbackrest_test.go b/internal/pgbackrest/pgbackrest_test.go index f3f870f89b..5aba171a07 100644 --- a/internal/pgbackrest/pgbackrest_test.go +++ b/internal/pgbackrest/pgbackrest_test.go @@ -59,11 +59,13 @@ fi Repos: []v1beta1.PGBackRestRepo{{ Name: "repo1", Volume: &v1beta1.RepoPVC{ - VolumeClaimSpec: v1beta1.VolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, - Resources: corev1.VolumeResourceRequirements{ - Requests: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceStorage: resource.MustParse("1Gi"), + VolumeClaimSpec: v1beta1.VolumeClaimSpecWithAutoGrow{ + VolumeClaimSpec: v1beta1.VolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, }, }, }, diff --git a/internal/pgbackrest/reconcile.go b/internal/pgbackrest/reconcile.go index 426e1312f6..e35cb00455 100644 --- a/internal/pgbackrest/reconcile.go +++ b/internal/pgbackrest/reconcile.go @@ -355,10 +355,11 @@ func addServerContainerAndVolume( container.VolumeMounts = append(container.VolumeMounts, mount) } } - reloader := corev1.Container{ - Name: naming.ContainerPGBackRestConfig, - Command: reloadCommand(naming.ContainerPGBackRestConfig), + Name: naming.ContainerPGBackRestConfig, + Command: reloadCommand( + naming.ContainerPGBackRestConfig, + cluster.Spec.Backups.PGBackRest.Repos), Image: container.Image, ImagePullPolicy: container.ImagePullPolicy, SecurityContext: initialize.RestrictedSecurityContext(), diff --git a/internal/pgbackrest/reconcile_test.go b/internal/pgbackrest/reconcile_test.go index a543879ed6..3cd62ba8bf 100644 --- a/internal/pgbackrest/reconcile_test.go +++ b/internal/pgbackrest/reconcile_test.go @@ -745,6 +745,8 @@ func TestAddServerToInstancePod(t *testing.T) { # Return size in Mebibytes. manageAutogrowAnnotation() { local volume=$1 + local trigger=$2 + local maxGrow=$3 size=$(df --block-size=M "/pgbackrest/${volume}") read -r _ size _ <<< "${size#*$'\n'}" @@ -753,9 +755,19 @@ func TestAddServerToInstancePod(t *testing.T) { sizeInt="${size//M/}" # Use the sed punctuation class, because the shell will not accept the percent sign in an expansion. useInt=${use//[[:punct:]]/} - triggerExpansion="$((useInt > 75))" + triggerExpansion="$((useInt > trigger))" if [[ ${triggerExpansion} -eq 1 ]]; then newSize="$(((sizeInt / 2)+sizeInt))" + # Only compare with maxGrow if it is set (not empty) + if [[ -n "${maxGrow}" ]]; then + # check to see how much we would normally grow + sizeDiff=$((newSize - sizeInt)) + + # Compare the size difference to the maxGrow; if it is greater, cap it to maxGrow + if [[ ${sizeDiff} -gt ${maxGrow} ]]; then + newSize=$((sizeInt + maxGrow)) + fi + fi newSizeMi="${newSize}Mi" d='[{"op": "add", "path": "/metadata/annotations/suggested-'"${volume}"'-pvc-size", "value": "'"${newSizeMi}"'"}]' curl --cacert "${CACERT}" --header "Authorization: Bearer ${TOKEN}" -XPATCH "${APISERVER}/api/v1/namespaces/${NAMESPACE}/pods/${HOSTNAME}?fieldManager=kubectl-annotate" -H "Content-Type: application/json-patch+json" --data "${d}" @@ -782,22 +794,22 @@ func TestAddServerToInstancePod(t *testing.T) { # manage autogrow annotation for the repo1 volume, if it exists if [[ -d /pgbackrest/repo1 ]]; then - manageAutogrowAnnotation "repo1" + manageAutogrowAnnotation "repo1" "75" "" fi # manage autogrow annotation for the repo2 volume, if it exists if [[ -d /pgbackrest/repo2 ]]; then - manageAutogrowAnnotation "repo2" + manageAutogrowAnnotation "repo2" "75" "" fi # manage autogrow annotation for the repo3 volume, if it exists if [[ -d /pgbackrest/repo3 ]]; then - manageAutogrowAnnotation "repo3" + manageAutogrowAnnotation "repo3" "75" "" fi # manage autogrow annotation for the repo4 volume, if it exists if [[ -d /pgbackrest/repo4 ]]; then - manageAutogrowAnnotation "repo4" + manageAutogrowAnnotation "repo4" "75" "" fi done @@ -923,6 +935,8 @@ func TestAddServerToInstancePod(t *testing.T) { # Return size in Mebibytes. manageAutogrowAnnotation() { local volume=$1 + local trigger=$2 + local maxGrow=$3 size=$(df --block-size=M "/pgbackrest/${volume}") read -r _ size _ <<< "${size#*$'\n'}" @@ -931,9 +945,19 @@ func TestAddServerToInstancePod(t *testing.T) { sizeInt="${size//M/}" # Use the sed punctuation class, because the shell will not accept the percent sign in an expansion. useInt=${use//[[:punct:]]/} - triggerExpansion="$((useInt > 75))" + triggerExpansion="$((useInt > trigger))" if [[ ${triggerExpansion} -eq 1 ]]; then newSize="$(((sizeInt / 2)+sizeInt))" + # Only compare with maxGrow if it is set (not empty) + if [[ -n "${maxGrow}" ]]; then + # check to see how much we would normally grow + sizeDiff=$((newSize - sizeInt)) + + # Compare the size difference to the maxGrow; if it is greater, cap it to maxGrow + if [[ ${sizeDiff} -gt ${maxGrow} ]]; then + newSize=$((sizeInt + maxGrow)) + fi + fi newSizeMi="${newSize}Mi" d='[{"op": "add", "path": "/metadata/annotations/suggested-'"${volume}"'-pvc-size", "value": "'"${newSizeMi}"'"}]' curl --cacert "${CACERT}" --header "Authorization: Bearer ${TOKEN}" -XPATCH "${APISERVER}/api/v1/namespaces/${NAMESPACE}/pods/${HOSTNAME}?fieldManager=kubectl-annotate" -H "Content-Type: application/json-patch+json" --data "${d}" @@ -960,22 +984,22 @@ func TestAddServerToInstancePod(t *testing.T) { # manage autogrow annotation for the repo1 volume, if it exists if [[ -d /pgbackrest/repo1 ]]; then - manageAutogrowAnnotation "repo1" + manageAutogrowAnnotation "repo1" "75" "" fi # manage autogrow annotation for the repo2 volume, if it exists if [[ -d /pgbackrest/repo2 ]]; then - manageAutogrowAnnotation "repo2" + manageAutogrowAnnotation "repo2" "75" "" fi # manage autogrow annotation for the repo3 volume, if it exists if [[ -d /pgbackrest/repo3 ]]; then - manageAutogrowAnnotation "repo3" + manageAutogrowAnnotation "repo3" "75" "" fi # manage autogrow annotation for the repo4 volume, if it exists if [[ -d /pgbackrest/repo4 ]]; then - manageAutogrowAnnotation "repo4" + manageAutogrowAnnotation "repo4" "75" "" fi done @@ -1090,6 +1114,8 @@ func TestAddServerToRepoPod(t *testing.T) { # Return size in Mebibytes. manageAutogrowAnnotation() { local volume=$1 + local trigger=$2 + local maxGrow=$3 size=$(df --block-size=M "/pgbackrest/${volume}") read -r _ size _ <<< "${size#*$'\n'}" @@ -1098,9 +1124,19 @@ func TestAddServerToRepoPod(t *testing.T) { sizeInt="${size//M/}" # Use the sed punctuation class, because the shell will not accept the percent sign in an expansion. useInt=${use//[[:punct:]]/} - triggerExpansion="$((useInt > 75))" + triggerExpansion="$((useInt > trigger))" if [[ ${triggerExpansion} -eq 1 ]]; then newSize="$(((sizeInt / 2)+sizeInt))" + # Only compare with maxGrow if it is set (not empty) + if [[ -n "${maxGrow}" ]]; then + # check to see how much we would normally grow + sizeDiff=$((newSize - sizeInt)) + + # Compare the size difference to the maxGrow; if it is greater, cap it to maxGrow + if [[ ${sizeDiff} -gt ${maxGrow} ]]; then + newSize=$((sizeInt + maxGrow)) + fi + fi newSizeMi="${newSize}Mi" d='[{"op": "add", "path": "/metadata/annotations/suggested-'"${volume}"'-pvc-size", "value": "'"${newSizeMi}"'"}]' curl --cacert "${CACERT}" --header "Authorization: Bearer ${TOKEN}" -XPATCH "${APISERVER}/api/v1/namespaces/${NAMESPACE}/pods/${HOSTNAME}?fieldManager=kubectl-annotate" -H "Content-Type: application/json-patch+json" --data "${d}" @@ -1127,22 +1163,22 @@ func TestAddServerToRepoPod(t *testing.T) { # manage autogrow annotation for the repo1 volume, if it exists if [[ -d /pgbackrest/repo1 ]]; then - manageAutogrowAnnotation "repo1" + manageAutogrowAnnotation "repo1" "75" "" fi # manage autogrow annotation for the repo2 volume, if it exists if [[ -d /pgbackrest/repo2 ]]; then - manageAutogrowAnnotation "repo2" + manageAutogrowAnnotation "repo2" "75" "" fi # manage autogrow annotation for the repo3 volume, if it exists if [[ -d /pgbackrest/repo3 ]]; then - manageAutogrowAnnotation "repo3" + manageAutogrowAnnotation "repo3" "75" "" fi # manage autogrow annotation for the repo4 volume, if it exists if [[ -d /pgbackrest/repo4 ]]; then - manageAutogrowAnnotation "repo4" + manageAutogrowAnnotation "repo4" "75" "" fi done diff --git a/internal/postgres/config.go b/internal/postgres/config.go index 5513f88cef..1e9f52a7e7 100644 --- a/internal/postgres/config.go +++ b/internal/postgres/config.go @@ -18,6 +18,7 @@ import ( "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/internal/shell" + "github.com/crunchydata/postgres-operator/internal/util" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -266,7 +267,15 @@ func Environment(cluster *v1beta1.PostgresCluster) []corev1.EnvVar { // reloadCommand returns an entrypoint that convinces PostgreSQL to reload // certificate files when they change. The process will appear as name in `ps` // and `top`. -func reloadCommand(name string) []string { +func reloadCommand( + name string, + pgdataAutoGrowVolumeSpec *v1beta1.VolumeClaimSpecWithAutoGrow, + pgwalAutoGrowVolumeSpec *v1beta1.VolumeClaimSpecWithAutoGrow, +) []string { + + pgdataTrigger, pgdataMaxGrow := util.GetAutoGrowFromSpec(pgdataAutoGrowVolumeSpec) + pgwalTrigger, pgwalMaxGrow := util.GetAutoGrowFromSpec(pgwalAutoGrowVolumeSpec) + // Use a Bash loop to periodically check the mtime of the mounted // certificate volume. When it changes, copy the replication certificate, // signal PostgreSQL, and print the observed timestamp. @@ -301,15 +310,27 @@ CACERT=${SERVICEACCOUNT}/ca.crt # Return size in Mebibytes. manageAutogrowAnnotation() { local volume=$1 + local trigger=$2 + local maxGrow=$3 - size=$(df --human-readable --block-size=M /"${volume}" | awk 'FNR == 2 {print $2}') - use=$(df --human-readable /"${volume}" | awk 'FNR == 2 {print $5}') + size=$(df --block-size=M /"${volume}" | awk 'FNR == 2 {print $2}') + use=$(df /"${volume}" | awk 'FNR == 2 {print $5}') sizeInt="${size//M/}" # Use the sed punctuation class, because the shell will not accept the percent sign in an expansion. useInt=$(echo $use | sed 's/[[:punct:]]//g') - triggerExpansion="$((useInt > 75))" - if [ $triggerExpansion -eq 1 ]; then + triggerExpansion="$((useInt > trigger))" + if [[ $triggerExpansion -eq 1 ]]; then newSize="$(((sizeInt / 2)+sizeInt))" + # Only compare with maxGrow if it is set (not empty) + if [[ -n "$maxGrow" ]]; then + # check to see how much we would normally grow + sizeDiff=$((newSize - sizeInt)) + + # Compare the size difference to the maxGrow; if it is greater, cap it to maxGrow + if [[ $sizeDiff -gt $maxGrow ]]; then + newSize=$((sizeInt + maxGrow)) + fi + fi newSizeMi="${newSize}Mi" d='[{"op": "add", "path": "/metadata/annotations/suggested-'"${volume}"'-pvc-size", "value": "'"$newSizeMi"'"}]' curl --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -XPATCH "${APISERVER}/api/v1/namespaces/${NAMESPACE}/pods/${HOSTNAME}?fieldManager=kubectl-annotate" -H "Content-Type: application/json-patch+json" --data "$d" @@ -329,11 +350,10 @@ while read -r -t 5 -u "${fd}" ||:; do fi # manage autogrow annotation for the pgData volume - manageAutogrowAnnotation "pgdata" - + manageAutogrowAnnotation "pgdata" "%s" "%s" # manage autogrow annotation for the pgWAL volume, if it exists if [[ -d /pgwal ]]; then - manageAutogrowAnnotation "pgwal" + manageAutogrowAnnotation "pgwal" "%s" "%s" fi done `, @@ -342,13 +362,10 @@ done naming.ReplicationCertPath, naming.ReplicationPrivateKeyPath, naming.ReplicationCACertPath, + pgdataTrigger, pgdataMaxGrow, + pgwalTrigger, pgwalMaxGrow, ) - // this is used to close out the while loop started above after adding the required - // auto grow annotation scripts - // finalDone := `done - // ` - // Elide the above script from `ps` and `top` by wrapping it in a function // and calling that. wrapper := `monitor() {` + script + `}; export -f monitor; exec -a "$0" bash -ceu monitor` diff --git a/internal/postgres/config_test.go b/internal/postgres/config_test.go index 32490df26b..cd4962be79 100644 --- a/internal/postgres/config_test.go +++ b/internal/postgres/config_test.go @@ -136,7 +136,7 @@ func TestWALDirectory(t *testing.T) { assert.Equal(t, WALDirectory(cluster, instance), "/pgdata/pg13_wal") // with WAL volume - instance.WALVolumeClaimSpec = new(v1beta1.VolumeClaimSpec) + instance.WALVolumeClaimSpec = new(v1beta1.VolumeClaimSpecWithAutoGrow) assert.Equal(t, WALDirectory(cluster, instance), "/pgwal/pg13_wal") } diff --git a/internal/postgres/reconcile.go b/internal/postgres/reconcile.go index cf8b02e4b8..0e01159e37 100644 --- a/internal/postgres/reconcile.go +++ b/internal/postgres/reconcile.go @@ -174,7 +174,11 @@ func InstancePod(ctx context.Context, reloader := corev1.Container{ Name: naming.ContainerClientCertCopy, - Command: reloadCommand(naming.ContainerClientCertCopy), + Command: reloadCommand( + naming.ContainerClientCertCopy, + &inInstanceSpec.DataVolumeClaimSpec, + inInstanceSpec.WALVolumeClaimSpec, + ), Image: container.Image, ImagePullPolicy: container.ImagePullPolicy, diff --git a/internal/postgres/reconcile_test.go b/internal/postgres/reconcile_test.go index 5950ddafc9..bcc8090428 100644 --- a/internal/postgres/reconcile_test.go +++ b/internal/postgres/reconcile_test.go @@ -181,15 +181,27 @@ containers: # Return size in Mebibytes. manageAutogrowAnnotation() { local volume=$1 + local trigger=$2 + local maxGrow=$3 - size=$(df --human-readable --block-size=M /"${volume}" | awk 'FNR == 2 {print $2}') - use=$(df --human-readable /"${volume}" | awk 'FNR == 2 {print $5}') + size=$(df --block-size=M /"${volume}" | awk 'FNR == 2 {print $2}') + use=$(df /"${volume}" | awk 'FNR == 2 {print $5}') sizeInt="${size//M/}" # Use the sed punctuation class, because the shell will not accept the percent sign in an expansion. useInt=$(echo $use | sed 's/[[:punct:]]//g') - triggerExpansion="$((useInt > 75))" - if [ $triggerExpansion -eq 1 ]; then + triggerExpansion="$((useInt > trigger))" + if [[ $triggerExpansion -eq 1 ]]; then newSize="$(((sizeInt / 2)+sizeInt))" + # Only compare with maxGrow if it is set (not empty) + if [[ -n "$maxGrow" ]]; then + # check to see how much we would normally grow + sizeDiff=$((newSize - sizeInt)) + + # Compare the size difference to the maxGrow; if it is greater, cap it to maxGrow + if [[ $sizeDiff -gt $maxGrow ]]; then + newSize=$((sizeInt + maxGrow)) + fi + fi newSizeMi="${newSize}Mi" d='[{"op": "add", "path": "/metadata/annotations/suggested-'"${volume}"'-pvc-size", "value": "'"$newSizeMi"'"}]' curl --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -XPATCH "${APISERVER}/api/v1/namespaces/${NAMESPACE}/pods/${HOSTNAME}?fieldManager=kubectl-annotate" -H "Content-Type: application/json-patch+json" --data "$d" @@ -209,11 +221,10 @@ containers: fi # manage autogrow annotation for the pgData volume - manageAutogrowAnnotation "pgdata" - + manageAutogrowAnnotation "pgdata" "75" "" # manage autogrow annotation for the pgWAL volume, if it exists if [[ -d /pgwal ]]; then - manageAutogrowAnnotation "pgwal" + manageAutogrowAnnotation "pgwal" "75" "" fi done }; export -f monitor; exec -a "$0" bash -ceu monitor @@ -616,7 +627,7 @@ volumes: walVolume.Name = "walvol" instance := new(v1beta1.PostgresInstanceSetSpec) - instance.WALVolumeClaimSpec = new(v1beta1.VolumeClaimSpec) + instance.WALVolumeClaimSpec = new(v1beta1.VolumeClaimSpecWithAutoGrow) pod := new(corev1.PodTemplateSpec) InstancePod(ctx, cluster, instance, diff --git a/internal/util/volumes.go b/internal/util/volumes.go index 9870920a97..4151eef76b 100644 --- a/internal/util/volumes.go +++ b/internal/util/volumes.go @@ -10,9 +10,15 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + "github.com/crunchydata/postgres-operator/internal/initialize" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) +const ( + // Default auto-grow trigger percentage + AutoGrowTriggerDefault = "75" +) + // AdditionalVolumeMount creates a [corev1.VolumeMount] at `/volumes/{name}` of volume `volumes-{name}`. func AdditionalVolumeMount(name string, readOnly bool) corev1.VolumeMount { return corev1.VolumeMount{ @@ -84,3 +90,38 @@ func addVolumesAndMounts(pod *corev1.PodSpec, volumes []v1beta1.AdditionalVolume return missingContainers } + +// GetAutoGrowFromSpec extracts AutoGrow settings from the provided AutoGrowSpec. +// It returns two strings: (trigger, maxGrow). +// +// - trigger: the auto-grow threshold as a decimal percentage string (e.g., "75"). +// Defaults to "75" when no trigger is configured. +// - maxGrow: the maximum growth size as a whole number of mebibytes (MiB). +// When no MaxGrow is configured, this is an empty string. +// +// If AutoGrow is nil, the function uses the default trigger ("75") and leaves +// maxGrow empty. When set, Trigger is formatted as a base-10 string and MaxGrow is +// converted from a resource quantity (bytes) to mebibytes by dividing bytes by 1024*1024. +func GetAutoGrowFromSpec(spec *v1beta1.VolumeClaimSpecWithAutoGrow) (string, string) { + // MaxGrow is optional; an empty string means "no limit" on growth and the volume + // will grow by 50% each time it is triggered. + maxGrow := "" + + // We always want to set default trigger; We will override it if + // the user has set a different value. + trigger := AutoGrowTriggerDefault + + // If AutoGrow is configured on the VolumeClaimSpecWithAutoGrow, extract + // the Trigger and MaxGrow values + if spec != nil && spec.AutoGrow != nil { + if t := spec.AutoGrow.Trigger; t != nil { + trigger = fmt.Sprintf("%d", initialize.FromPointer(t)) + } + if mg := spec.AutoGrow.MaxGrow; mg != nil { + // Value() returns bytes, convert to mebibytes + mebibytes := mg.Value() / (1024 * 1024) + maxGrow = fmt.Sprintf("%d", mebibytes) + } + } + return trigger, maxGrow +} diff --git a/internal/util/volumes_test.go b/internal/util/volumes_test.go index 81e4f17a3f..ee5ebaff9e 100644 --- a/internal/util/volumes_test.go +++ b/internal/util/volumes_test.go @@ -11,7 +11,9 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "gotest.tools/v3/assert" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "github.com/crunchydata/postgres-operator/internal/initialize" "github.com/crunchydata/postgres-operator/internal/testing/cmp" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -278,3 +280,87 @@ func TestAddVolumeAndMountsToPod(t *testing.T) { AddCloudLogVolumeToPod(out, "volume-name") alwaysExpect(t, out) } + +// TestGetAutoGrowFromRepo verifies that GetAutoGrowFromSpec returns the correct +// trigger and max grow values for various AutoGrowSpec inputs. It uses a +// table-driven approach to cover: +// - nil AutoGrowSpec: default trigger "75" with no max grow +// - Trigger only: provided trigger with no max grow +// - MaxGrow only: default trigger "75" with MaxGrow converted to MiB (e.g., 2Gi -> "2048") +// - Both set: provided trigger and MaxGrow converted to MiB (e.g., 512Mi -> "512") +// +// The test asserts that the returned strings match the expected trigger and max values. +func TestGetAutoGrowFromRepo(t *testing.T) { + tc := []struct { + name string + autoGrow *v1beta1.VolumeClaimSpecWithAutoGrow + expectedTrigger string + expectedMaxGrow string + }{{ + name: "autogrow-not-set", + autoGrow: nil, + expectedTrigger: "75", + expectedMaxGrow: "", + }, { + name: "autogrow-set-trigger-only", + autoGrow: &v1beta1.VolumeClaimSpecWithAutoGrow{ + AutoGrow: &v1beta1.AutoGrowSpec{ + Trigger: initialize.Int32(10), + }, + }, + expectedTrigger: "10", + expectedMaxGrow: "", + }, { + name: "autogrow-set-maxgrow-only", + autoGrow: &v1beta1.VolumeClaimSpecWithAutoGrow{ + AutoGrow: &v1beta1.AutoGrowSpec{ + MaxGrow: initialize.Pointer(resource.MustParse("2Gi")), + }, + }, + expectedTrigger: "75", + expectedMaxGrow: "2048", + }, { + name: "autogrow-set-both", + autoGrow: &v1beta1.VolumeClaimSpecWithAutoGrow{ + AutoGrow: &v1beta1.AutoGrowSpec{ + Trigger: initialize.Int32(90), + MaxGrow: initialize.Pointer(resource.MustParse("512Mi")), + }, + }, + expectedTrigger: "90", + expectedMaxGrow: "512", + }, { + name: "autogrow-set-maxgrow-only-small", + autoGrow: &v1beta1.VolumeClaimSpecWithAutoGrow{ + AutoGrow: &v1beta1.AutoGrowSpec{ + MaxGrow: initialize.Pointer(resource.MustParse("512Ki")), + }, + }, + expectedTrigger: "75", + expectedMaxGrow: "0", + }, { + name: "autogrow-set-maxgrow-only-exact-mib", + autoGrow: &v1beta1.VolumeClaimSpecWithAutoGrow{ + AutoGrow: &v1beta1.AutoGrowSpec{ + MaxGrow: initialize.Pointer(resource.MustParse("1Mi")), + }, + }, + expectedTrigger: "75", + expectedMaxGrow: "1", + }, { + name: "autogrow-set-maxgrow-only-large", + autoGrow: &v1beta1.VolumeClaimSpecWithAutoGrow{ + AutoGrow: &v1beta1.AutoGrowSpec{ + MaxGrow: initialize.Pointer(resource.MustParse("5Ti")), + }, + }, + expectedTrigger: "75", + expectedMaxGrow: "5242880", + }} + + for _, test := range tc { + trigger, max := GetAutoGrowFromSpec(test.autoGrow) + assert.Equal(t, trigger, test.expectedTrigger) + assert.Equal(t, max, test.expectedMaxGrow) + } +} diff --git a/pkg/apis/postgres-operator.crunchydata.com/v1/postgrescluster_types.go b/pkg/apis/postgres-operator.crunchydata.com/v1/postgrescluster_types.go index f1873bdc7e..fc409ee0ce 100644 --- a/pkg/apis/postgres-operator.crunchydata.com/v1/postgrescluster_types.go +++ b/pkg/apis/postgres-operator.crunchydata.com/v1/postgrescluster_types.go @@ -478,7 +478,7 @@ type PostgresInstanceSetSpec struct { // More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes // --- // +required - DataVolumeClaimSpec v1beta1.VolumeClaimSpec `json:"dataVolumeClaimSpec"` + DataVolumeClaimSpec v1beta1.VolumeClaimSpecWithAutoGrow `json:"dataVolumeClaimSpec"` // Priority class name for the PostgreSQL pod. Changing this value causes // PostgreSQL to restart. @@ -520,7 +520,7 @@ type PostgresInstanceSetSpec struct { // More info: https://www.postgresql.org/docs/current/wal.html // --- // +optional - WALVolumeClaimSpec *v1beta1.VolumeClaimSpec `json:"walVolumeClaimSpec,omitempty"` + WALVolumeClaimSpec *v1beta1.VolumeClaimSpecWithAutoGrow `json:"walVolumeClaimSpec,omitempty"` // The list of tablespaces volumes to mount for this postgrescluster // This field requires enabling TablespaceVolumes feature gate diff --git a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/pgbackrest_types.go b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/pgbackrest_types.go index d100e9999f..d9777bdcd5 100644 --- a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/pgbackrest_types.go +++ b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/pgbackrest_types.go @@ -363,7 +363,7 @@ type RepoPVC struct { // Defines a PersistentVolumeClaim spec used to create and/or bind a volume // --- // +required - VolumeClaimSpec VolumeClaimSpec `json:"volumeClaimSpec"` + VolumeClaimSpec VolumeClaimSpecWithAutoGrow `json:"volumeClaimSpec"` } // RepoAzure represents a pgBackRest repository that is created using Azure storage diff --git a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go index 397b6d4f76..4374fa5e4e 100644 --- a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go +++ b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go @@ -475,7 +475,7 @@ type PostgresInstanceSetSpec struct { // More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes // --- // +required - DataVolumeClaimSpec VolumeClaimSpec `json:"dataVolumeClaimSpec"` + DataVolumeClaimSpec VolumeClaimSpecWithAutoGrow `json:"dataVolumeClaimSpec"` // Priority class name for the PostgreSQL pod. Changing this value causes // PostgreSQL to restart. @@ -517,7 +517,7 @@ type PostgresInstanceSetSpec struct { // More info: https://www.postgresql.org/docs/current/wal.html // --- // +optional - WALVolumeClaimSpec *VolumeClaimSpec `json:"walVolumeClaimSpec,omitempty"` + WALVolumeClaimSpec *VolumeClaimSpecWithAutoGrow `json:"walVolumeClaimSpec,omitempty"` // The list of tablespaces volumes to mount for this postgrescluster // This field requires enabling TablespaceVolumes feature gate diff --git a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/shared_types.go b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/shared_types.go index 4325dae255..4f276a8d07 100644 --- a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/shared_types.go +++ b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/shared_types.go @@ -8,9 +8,12 @@ import ( "encoding/json" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kube-openapi/pkg/validation/strfmt" + + "github.com/crunchydata/postgres-operator/internal/initialize" ) // --- @@ -149,6 +152,66 @@ func (spec *VolumeClaimSpec) AsPersistentVolumeClaimSpec() corev1.PersistentVolu return out } +// VolumeClaimSpecWithAutoGrow extends VolumeClaimSpec with options for +// automatic volume growth. +// +structType=atomic +type VolumeClaimSpecWithAutoGrow struct { + VolumeClaimSpec `json:",inline"` + + // +optional + AutoGrow *AutoGrowSpec `json:"autoGrow,omitempty"` +} + +// AutoGrowSpec provides options to tune volume auto-growing behavior. +// Auto grow requires that a limit be set on the PVC. +type AutoGrowSpec struct { + // Trigger is the percentage of used space at which to trigger a volume + // expansion. + // +optional + // +kubebuilder:default=75 + // +kubebuilder:validation:Minimum=50 + // +kubebuilder:validation:Maximum=90 + Trigger *int32 `json:"trigger,omitempty"` + + // MaxGrow is the maximum size to which the volume can be automatically + // expanded. If not set, the volume will grow by 50% of the original size each + // time the Trigger threshold is exceeded. + // +optional + MaxGrow *resource.Quantity `json:"maxGrow,omitempty"` +} + +func (spec *AutoGrowSpec) Default() { + if spec.Trigger == nil { + spec.Trigger = initialize.Int32(75) + } +} + +func (spec *VolumeClaimSpecWithAutoGrow) DeepCopyInto(out *VolumeClaimSpecWithAutoGrow) { + spec.VolumeClaimSpec.DeepCopyInto(&out.VolumeClaimSpec) + if spec.AutoGrow != nil { + out.AutoGrow = new(AutoGrowSpec) + spec.AutoGrow.DeepCopyInto(out.AutoGrow) + } else { + out.AutoGrow = nil + } +} + +// DeepCopyInto copies the receiver into out. Both must be non-nil. +func (spec *AutoGrowSpec) DeepCopyInto(out *AutoGrowSpec) { + *out = *spec + if spec.MaxGrow != nil { + q := spec.MaxGrow.DeepCopy() + out.MaxGrow = &q + } else { + out.MaxGrow = nil + } +} + +// AsPersistentVolumeClaimSpec returns a copy of the embedded VolumeClaimSpec as a [corev1.PersistentVolumeClaimSpec]. +func (spec *VolumeClaimSpecWithAutoGrow) AsPersistentVolumeClaimSpec() corev1.PersistentVolumeClaimSpec { + return spec.VolumeClaimSpec.AsPersistentVolumeClaimSpec() +} + // --- // SchemalessObject is a map compatible with JSON object. // diff --git a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/shared_types_test.go b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/shared_types_test.go index 89f0c3f42a..bbecd39c9a 100644 --- a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/shared_types_test.go +++ b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/shared_types_test.go @@ -14,6 +14,8 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kube-openapi/pkg/validation/strfmt" "sigs.k8s.io/yaml" + + "github.com/crunchydata/postgres-operator/internal/initialize" ) func TestAdditionalVolumeAsVolume(t *testing.T) { @@ -276,3 +278,47 @@ func TestVolumeClaimSpecYAML(t *testing.T) { }, }) } + +func TestVolumeClaimSpecWithAutoGrowYAML(t *testing.T) { + t.Parallel() + + var zero VolumeClaimSpecWithAutoGrow + out, err := yaml.Marshal(zero) + assert.NilError(t, err) + assert.DeepEqual(t, string(out), "resources: {}\n") + + var parsed VolumeClaimSpecWithAutoGrow + assert.NilError(t, yaml.Unmarshal([]byte(`{ + accessModes: [ReadWriteMany], + resources: { requests: { storage: 1Gi } }, + storageClassName: zork, + autoGrow: { trigger: 50, maxGrow: 100Mi }, + }`), &parsed)) + + zork := "zork" + maxGrow := resource.MustParse("100Mi") + assert.DeepEqual(t, parsed, VolumeClaimSpecWithAutoGrow{ + AutoGrow: &AutoGrowSpec{ + Trigger: initialize.Int32(50), + MaxGrow: &maxGrow, + }, + VolumeClaimSpec: VolumeClaimSpec{ + StorageClassName: &zork, + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }) +} + +func TestAutoGrowDefault(t *testing.T) { + var autoGrow AutoGrowSpec + autoGrow.Default() + + assert.Equal(t, *autoGrow.Trigger, int32(75)) +} diff --git a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/zz_generated.deepcopy.go b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/zz_generated.deepcopy.go index e75af532a2..8843869827 100644 --- a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/zz_generated.deepcopy.go @@ -53,6 +53,16 @@ func (in *AdditionalVolume) DeepCopy() *AdditionalVolume { return out } +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoGrowSpec. +func (in *AutoGrowSpec) DeepCopy() *AutoGrowSpec { + if in == nil { + return nil + } + out := new(AutoGrowSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupJobs) DeepCopyInto(out *BackupJobs) { *out = *in @@ -2969,6 +2979,16 @@ func (in *VolumeClaimSpec) DeepCopy() *VolumeClaimSpec { return out } +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeClaimSpecWithAutoGrow. +func (in *VolumeClaimSpecWithAutoGrow) DeepCopy() *VolumeClaimSpecWithAutoGrow { + if in == nil { + return nil + } + out := new(VolumeClaimSpecWithAutoGrow) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeSnapshots) DeepCopyInto(out *VolumeSnapshots) { *out = *in From 19d904222df10796c3c4bf7d3b19688a019db63d Mon Sep 17 00:00:00 2001 From: jmckulk Date: Fri, 5 Sep 2025 13:39:38 -0400 Subject: [PATCH 2/2] Ignore dupword --- .golangci.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.golangci.yaml b/.golangci.yaml index a1de0813b4..76d7d2fa1f 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -63,6 +63,11 @@ linters: - zerologlint settings: + dupword: + ignore: + # We might see duplicate instances of 'fi' if we end two bash 'if' statements + - fi + depguard: rules: everything: