From 031a6d420a44d827e56d0c7f13ec7d2cbdef26f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 1 Oct 2024 14:13:26 +0300 Subject: [PATCH 1/6] K8SPG-260: Fix restart after upgrade --- .../controller/postgrescluster/instance.go | 1 + .../postgrescluster/instance_test.go | 3 + .../controller/postgrescluster/pgadmin.go | 1 + .../controller/postgrescluster/pgbackrest.go | 6 +- .../postgrescluster/pgbackrest_test.go | 13 +- .../controller/postgrescluster/pgmonitor.go | 2 +- .../postgrescluster/pgmonitor_test.go | 4 + internal/controller/postgrescluster/util.go | 5 +- .../controller/postgrescluster/util_test.go | 10 +- .../controller/postgrescluster/volumes.go | 6 +- internal/controller/standalone_pgadmin/pod.go | 4 +- internal/initialize/security.go | 12 +- internal/initialize/security_test.go | 6 +- internal/pgadmin/reconcile.go | 4 +- internal/pgadmin/reconcile_test.go | 5 + internal/pgbackrest/config.go | 29 ++- internal/pgbackrest/config_test.go | 4 +- internal/pgbackrest/reconcile.go | 10 +- internal/pgbackrest/reconcile_test.go | 14 +- internal/pgbouncer/config.go | 16 +- internal/pgbouncer/config_test.go | 2 +- internal/pgbouncer/reconcile.go | 6 +- internal/pgbouncer/reconcile_test.go | 5 + internal/postgres/config.go | 174 +++++++++++++++--- internal/postgres/config_test.go | 4 +- internal/postgres/reconcile.go | 16 +- internal/postgres/reconcile_test.go | 4 +- .../v2/perconapgcluster_types.go | 7 +- .../v1beta1/postgrescluster_types.go | 11 ++ 29 files changed, 307 insertions(+), 77 deletions(-) diff --git a/internal/controller/postgrescluster/instance.go b/internal/controller/postgrescluster/instance.go index 151b43cbc9..ae5eda0f44 100644 --- a/internal/controller/postgrescluster/instance.go +++ b/internal/controller/postgrescluster/instance.go @@ -1215,6 +1215,7 @@ func (r *Reconciler) reconcileInstance( // containers if err == nil { addNSSWrapper( + cluster, config.PostgresContainerImage(cluster), cluster.Spec.ImagePullPolicy, &instance.Spec.Template) diff --git a/internal/controller/postgrescluster/instance_test.go b/internal/controller/postgrescluster/instance_test.go index 3247410a3d..eb63b7bafb 100644 --- a/internal/controller/postgrescluster/instance_test.go +++ b/internal/controller/postgrescluster/instance_test.go @@ -541,6 +541,9 @@ func TestAddPGBackRestToInstancePodSpec(t *testing.T) { cluster := v1beta1.PostgresCluster{} cluster.Name = "hippo" cluster.Default() + cluster.SetLabels(map[string]string{ + naming.LabelVersion: "2.5.0", + }) certificates := corev1.Secret{} certificates.Name = "some-secret" diff --git a/internal/controller/postgrescluster/pgadmin.go b/internal/controller/postgrescluster/pgadmin.go index 1d76dd90b8..db40d670fc 100644 --- a/internal/controller/postgrescluster/pgadmin.go +++ b/internal/controller/postgrescluster/pgadmin.go @@ -361,6 +361,7 @@ func (r *Reconciler) reconcilePGAdminStatefulSet( // add nss_wrapper init container and add nss_wrapper env vars to the pgAdmin // container addNSSWrapper( + cluster, config.PGAdminContainerImage(cluster), cluster.Spec.ImagePullPolicy, &sts.Spec.Template) diff --git a/internal/controller/postgrescluster/pgbackrest.go b/internal/controller/postgrescluster/pgbackrest.go index daf99e4da6..3ac264a5a3 100644 --- a/internal/controller/postgrescluster/pgbackrest.go +++ b/internal/controller/postgrescluster/pgbackrest.go @@ -640,6 +640,7 @@ func (r *Reconciler) generateRepoHostIntent(ctx context.Context, postgresCluster // add nss_wrapper init container and add nss_wrapper env vars to the pgbackrest // container addNSSWrapper( + postgresCluster, config.PGBackRestContainerImage(postgresCluster), postgresCluster.Spec.ImagePullPolicy, &repo.Spec.Template) @@ -736,7 +737,7 @@ func generateBackupJobSpecIntent(postgresCluster *v1beta1.PostgresCluster, Image: config.PGBackRestContainerImage(postgresCluster), ImagePullPolicy: postgresCluster.Spec.ImagePullPolicy, Name: naming.PGBackRestRepoContainerName, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(postgresCluster.CompareVersion("2.5.0") >= 0), } if postgresCluster.Spec.Backups.PGBackRest.Jobs != nil { @@ -1193,6 +1194,7 @@ func (r *Reconciler) reconcileRestoreJob(ctx context.Context, // add nss_wrapper init container and add nss_wrapper env vars to the pgbackrest restore // container addNSSWrapper( + cluster, config.PGBackRestContainerImage(cluster), cluster.Spec.ImagePullPolicy, &restoreJob.Spec.Template) @@ -1245,7 +1247,7 @@ func (r *Reconciler) generateRestoreJobIntent(cluster *v1beta1.PostgresCluster, Name: naming.PGBackRestRestoreContainerName, VolumeMounts: volumeMounts, Env: []corev1.EnvVar{{Name: "PGHOST", Value: "/tmp"}}, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(cluster.CompareVersion("2.5.0") >= 0), Resources: dataSource.Resources, }}, RestartPolicy: corev1.RestartPolicyNever, diff --git a/internal/controller/postgrescluster/pgbackrest_test.go b/internal/controller/postgrescluster/pgbackrest_test.go index 077800964c..3089ae3f41 100644 --- a/internal/controller/postgrescluster/pgbackrest_test.go +++ b/internal/controller/postgrescluster/pgbackrest_test.go @@ -2383,7 +2383,16 @@ func TestCopyConfigurationResources(t *testing.T) { func TestGenerateBackupJobIntent(t *testing.T) { t.Run("empty", func(t *testing.T) { spec := generateBackupJobSpecIntent( - &v1beta1.PostgresCluster{}, v1beta1.PGBackRestRepo{}, + &v1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "", + Namespace: "", + Labels: map[string]string{ + naming.LabelVersion: "2.5.0", + }, + }, + }, + v1beta1.PGBackRestRepo{}, "", nil, nil, ) @@ -2814,7 +2823,7 @@ func TestGenerateRestoreJobIntent(t *testing.T) { }) t.Run("SecurityContext", func(t *testing.T) { assert.DeepEqual(t, job.Spec.Template.Spec.Containers[0].SecurityContext, - initialize.RestrictedSecurityContext()) + initialize.RestrictedSecurityContext(true)) }) t.Run("Resources", func(t *testing.T) { assert.DeepEqual(t, job.Spec.Template.Spec.Containers[0].Resources, diff --git a/internal/controller/postgrescluster/pgmonitor.go b/internal/controller/postgrescluster/pgmonitor.go index 94aba727d4..280ce05c50 100644 --- a/internal/controller/postgrescluster/pgmonitor.go +++ b/internal/controller/postgrescluster/pgmonitor.go @@ -270,7 +270,7 @@ func addPGMonitorExporterToInstancePodSpec( withBuiltInCollectors := !strings.EqualFold(cluster.Annotations[naming.PostgresExporterCollectorsAnnotation], "None") - securityContext := initialize.RestrictedSecurityContext() + securityContext := initialize.RestrictedSecurityContext(cluster.CompareVersion("2.5.0") >= 0) exporterContainer := corev1.Container{ Name: naming.ContainerPGMonitorExporter, Image: config.PGExporterContainerImage(cluster), diff --git a/internal/controller/postgrescluster/pgmonitor_test.go b/internal/controller/postgrescluster/pgmonitor_test.go index 7d8b6602b8..7dc7645f9a 100644 --- a/internal/controller/postgrescluster/pgmonitor_test.go +++ b/internal/controller/postgrescluster/pgmonitor_test.go @@ -100,6 +100,10 @@ func TestAddPGMonitorExporterToInstancePodSpec(t *testing.T) { cluster.Spec.Port = initialize.Int32(5432) cluster.Spec.ImagePullPolicy = corev1.PullAlways + cluster.SetLabels(map[string]string{ + naming.LabelVersion: "2.5.0", + }) + resources := corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("100m"), diff --git a/internal/controller/postgrescluster/util.go b/internal/controller/postgrescluster/util.go index 7cb5445b0e..33799a35a0 100644 --- a/internal/controller/postgrescluster/util.go +++ b/internal/controller/postgrescluster/util.go @@ -28,6 +28,7 @@ import ( "github.com/percona/percona-postgresql-operator/internal/initialize" "github.com/percona/percona-postgresql-operator/internal/naming" + "github.com/percona/percona-postgresql-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) const ( @@ -219,7 +220,7 @@ func addTMPEmptyDir(template *corev1.PodTemplateSpec, sizeLimit resource.Quantit // containers in the Pod template. Additionally, an init container is added to the Pod template // as needed to setup the nss_wrapper. Please note that the nss_wrapper is required for // compatibility with OpenShift: https://access.redhat.com/articles/4859371. -func addNSSWrapper(image string, imagePullPolicy corev1.PullPolicy, template *corev1.PodTemplateSpec) { +func addNSSWrapper(cluster *v1beta1.PostgresCluster, image string, imagePullPolicy corev1.PullPolicy, template *corev1.PodTemplateSpec) { nssWrapperCmd := postgresNSSWrapperPrefix + nssWrapperScript for i, c := range template.Spec.Containers { @@ -250,7 +251,7 @@ func addNSSWrapper(image string, imagePullPolicy corev1.PullPolicy, template *co Image: image, ImagePullPolicy: imagePullPolicy, Name: naming.ContainerNSSWrapperInit, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(cluster.CompareVersion("2.5.0") >= 0), } // Here we set the NSS wrapper container resources to the 'database', 'pgadmin' diff --git a/internal/controller/postgrescluster/util_test.go b/internal/controller/postgrescluster/util_test.go index e6c2076f55..65d8321f2e 100644 --- a/internal/controller/postgrescluster/util_test.go +++ b/internal/controller/postgrescluster/util_test.go @@ -24,9 +24,11 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/percona/percona-postgresql-operator/internal/naming" "github.com/percona/percona-postgresql-operator/internal/testing/cmp" + "github.com/percona/percona-postgresql-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) func TestSafeHash32(t *testing.T) { @@ -119,6 +121,12 @@ func TestAddDevSHM(t *testing.T) { } func TestAddNSSWrapper(t *testing.T) { + cluster := &v1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nss-wrapper-test", + Namespace: "nss-wrapper-test", + }, + } image := "test-image" imagePullPolicy := corev1.PullAlways @@ -226,7 +234,7 @@ func TestAddNSSWrapper(t *testing.T) { template := tc.podTemplate beforeAddNSS := template.DeepCopy().Spec.Containers - addNSSWrapper(image, imagePullPolicy, template) + addNSSWrapper(cluster, image, imagePullPolicy, template) t.Run("container-updated", func(t *testing.T) { // Each container that requires the nss_wrapper envs should be updated diff --git a/internal/controller/postgrescluster/volumes.go b/internal/controller/postgrescluster/volumes.go index 467a9f9bb7..ced685e730 100644 --- a/internal/controller/postgrescluster/volumes.go +++ b/internal/controller/postgrescluster/volumes.go @@ -477,7 +477,7 @@ func (r *Reconciler) reconcileMovePGDataDir(ctx context.Context, Image: config.PostgresContainerImage(cluster), ImagePullPolicy: cluster.Spec.ImagePullPolicy, Name: naming.ContainerJobMovePGDataDir, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(cluster.CompareVersion("2.5.0") >= 0), VolumeMounts: []corev1.VolumeMount{postgres.DataVolumeMount()}, } if len(cluster.Spec.InstanceSets) > 0 { @@ -596,7 +596,7 @@ func (r *Reconciler) reconcileMoveWALDir(ctx context.Context, Image: config.PostgresContainerImage(cluster), ImagePullPolicy: cluster.Spec.ImagePullPolicy, Name: naming.ContainerJobMovePGWALDir, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(cluster.CompareVersion("2.5.0") >= 0), VolumeMounts: []corev1.VolumeMount{postgres.WALVolumeMount()}, } if len(cluster.Spec.InstanceSets) > 0 { @@ -720,7 +720,7 @@ func (r *Reconciler) reconcileMoveRepoDir(ctx context.Context, Image: config.PGBackRestContainerImage(cluster), ImagePullPolicy: cluster.Spec.ImagePullPolicy, Name: naming.ContainerJobMovePGBackRestRepoDir, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(cluster.CompareVersion("2.5.0") >= 0), VolumeMounts: []corev1.VolumeMount{pgbackrest.RepoVolumeMount()}, } if cluster.Spec.Backups.PGBackRest.RepoHost != nil { diff --git a/internal/controller/standalone_pgadmin/pod.go b/internal/controller/standalone_pgadmin/pod.go index 024270fc6e..8278264b5d 100644 --- a/internal/controller/standalone_pgadmin/pod.go +++ b/internal/controller/standalone_pgadmin/pod.go @@ -113,7 +113,7 @@ func pod( Image: config.StandalonePGAdminContainerImage(inPGAdmin), ImagePullPolicy: inPGAdmin.Spec.ImagePullPolicy, Resources: inPGAdmin.Spec.Resources, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(true), Ports: []corev1.ContainerPort{{ Name: naming.PortPGAdmin, ContainerPort: int32(pgAdminPort), @@ -180,7 +180,7 @@ func pod( Image: container.Image, ImagePullPolicy: container.ImagePullPolicy, Resources: container.Resources, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(true), VolumeMounts: []corev1.VolumeMount{ // Volume to write a custom `config_system.py` file to. { diff --git a/internal/initialize/security.go b/internal/initialize/security.go index 49291db478..7eb865d308 100644 --- a/internal/initialize/security.go +++ b/internal/initialize/security.go @@ -31,8 +31,8 @@ func PodSecurityContext() *corev1.PodSecurityContext { // RestrictedSecurityContext returns a v1.SecurityContext with safe defaults. // See https://docs.k8s.io/concepts/security/pod-security-standards/ -func RestrictedSecurityContext() *corev1.SecurityContext { - return &corev1.SecurityContext{ +func RestrictedSecurityContext(enableSeccompProfile bool) *corev1.SecurityContext { + secContext := &corev1.SecurityContext{ // Prevent any container processes from gaining privileges. AllowPrivilegeEscalation: Bool(false), @@ -51,9 +51,13 @@ func RestrictedSecurityContext() *corev1.SecurityContext { // Fail to start the container if its image runs as UID 0 (root). RunAsNonRoot: Bool(true), + } - SeccompProfile: &corev1.SeccompProfile{ + if enableSeccompProfile { + secContext.SeccompProfile = &corev1.SeccompProfile{ Type: corev1.SeccompProfileTypeRuntimeDefault, - }, + } } + + return secContext } diff --git a/internal/initialize/security_test.go b/internal/initialize/security_test.go index 66271379d9..7a3a42b997 100644 --- a/internal/initialize/security_test.go +++ b/internal/initialize/security_test.go @@ -53,7 +53,7 @@ func TestPodSecurityContext(t *testing.T) { // > lower-trust users. t.Run("Restricted", func(t *testing.T) { if assert.Check(t, psc.RunAsNonRoot == nil) { - assert.Assert(t, initialize.RestrictedSecurityContext().RunAsNonRoot != nil, + assert.Assert(t, initialize.RestrictedSecurityContext(true).RunAsNonRoot != nil, `RunAsNonRoot should be delegated to the container-level v1.SecurityContext`) } @@ -61,14 +61,14 @@ func TestPodSecurityContext(t *testing.T) { `Containers must not set runAsUser to 0`) if assert.Check(t, psc.SeccompProfile == nil) { - assert.Assert(t, initialize.RestrictedSecurityContext().SeccompProfile != nil, + assert.Assert(t, initialize.RestrictedSecurityContext(true).SeccompProfile != nil, `SeccompProfile should be delegated to the container-level v1.SecurityContext`) } }) } func TestRestrictedSecurityContext(t *testing.T) { - sc := initialize.RestrictedSecurityContext() + sc := initialize.RestrictedSecurityContext(true) // Kubernetes describes recommended security profiles: // - https://docs.k8s.io/concepts/security/pod-security-standards/ diff --git a/internal/pgadmin/reconcile.go b/internal/pgadmin/reconcile.go index 6a237f9118..f07788c42e 100644 --- a/internal/pgadmin/reconcile.go +++ b/internal/pgadmin/reconcile.go @@ -244,7 +244,7 @@ func Pod( ImagePullPolicy: inCluster.Spec.ImagePullPolicy, Resources: inCluster.Spec.UserInterface.PGAdmin.Resources, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(inCluster.CompareVersion("2.5.0") >= 0), Ports: []corev1.ContainerPort{{ Name: naming.PortPGAdmin, @@ -294,7 +294,7 @@ func Pod( Image: container.Image, ImagePullPolicy: container.ImagePullPolicy, Resources: container.Resources, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(inCluster.CompareVersion("2.5.0") >= 0), VolumeMounts: []corev1.VolumeMount{ startupVolumeMount, configVolumeMount, diff --git a/internal/pgadmin/reconcile_test.go b/internal/pgadmin/reconcile_test.go index c94d90c637..c60f77e638 100644 --- a/internal/pgadmin/reconcile_test.go +++ b/internal/pgadmin/reconcile_test.go @@ -22,6 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "github.com/percona/percona-postgresql-operator/internal/naming" "github.com/percona/percona-postgresql-operator/internal/testing/cmp" "github.com/percona/percona-postgresql-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -85,6 +86,10 @@ func TestPod(t *testing.T) { pod := new(corev1.PodSpec) pvc := new(corev1.PersistentVolumeClaim) + cluster.SetLabels(map[string]string{ + naming.LabelVersion: "2.5.0", + }) + call := func() { Pod(cluster, config, pod, pvc) } t.Run("Disabled", func(t *testing.T) { diff --git a/internal/pgbackrest/config.go b/internal/pgbackrest/config.go index cc976790ee..dbd1b071c0 100644 --- a/internal/pgbackrest/config.go +++ b/internal/pgbackrest/config.go @@ -159,7 +159,7 @@ func MakePGBackrestLogDir(template *corev1.PodTemplateSpec, Image: config.PGBackRestContainerImage(cluster), ImagePullPolicy: cluster.Spec.ImagePullPolicy, Name: naming.ContainerPGBackRestLogDirInit, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(cluster.CompareVersion("2.5.0") >= 0), } // Set the container resources to the 'pgbackrest' container configuration. @@ -444,7 +444,7 @@ func getExternalRepoConfigs(repo v1beta1.PGBackRestRepo) map[string]string { // reloadCommand returns an entrypoint that convinces the pgBackRest TLS server // to reload its options and certificate files when they change. The process // will appear as name in `ps` and `top`. -func reloadCommand(name string) []string { +func reloadCommand(name string, post250 bool) []string { // Use a Bash loop to periodically check the mtime of the mounted server // volume and configuration file. When either changes, signal pgBackRest // and print the observed timestamp. @@ -460,7 +460,29 @@ func reloadCommand(name string) []string { // descriptor gets closed and reopened to use the builtin `[ -nt` to check // mtimes. // - https://unix.stackexchange.com/a/407383 - const script = ` + script := ` +exec {fd}<> <(:) +until read -r -t 5 -u "${fd}"; do + if + [ "${filename}" -nt "/proc/self/fd/${fd}" ] && + pkill -HUP --exact --parent=0 pgbackrest + then + exec {fd}>&- && exec {fd}<> <(:) + stat --dereference --format='Loaded configuration dated %y' "${filename}" + elif + { [ "${directory}" -nt "/proc/self/fd/${fd}" ] || + [ "${authority}" -nt "/proc/self/fd/${fd}" ] + } && + pkill -HUP --exact --parent=0 pgbackrest + then + exec {fd}>&- && exec {fd}<> <(:) + stat --format='Loaded certificates dated %y' "${directory}" + fi +done +` + + if post250 { + script = ` exec {fd}<> <(:||:) until read -r -t 5 -u "${fd}"; do if @@ -480,6 +502,7 @@ until read -r -t 5 -u "${fd}"; do fi done ` + } // Elide the above script from `ps` and `top` by wrapping it in a function // and calling that. diff --git a/internal/pgbackrest/config_test.go b/internal/pgbackrest/config_test.go index b7d3d319ee..e7bdfb75a5 100644 --- a/internal/pgbackrest/config_test.go +++ b/internal/pgbackrest/config_test.go @@ -320,7 +320,7 @@ func TestMakePGBackrestLogDir(t *testing.T) { func TestReloadCommand(t *testing.T) { shellcheck := require.ShellCheck(t) - command := reloadCommand("some-name") + command := reloadCommand("some-name", true) // Expect a bash command with an inline script. assert.DeepEqual(t, command[:3], []string{"bash", "-ceu", "--"}) @@ -338,7 +338,7 @@ func TestReloadCommand(t *testing.T) { } func TestReloadCommandPrettyYAML(t *testing.T) { - b, err := yaml.Marshal(reloadCommand("any")) + b, err := yaml.Marshal(reloadCommand("any", true)) assert.NilError(t, err) assert.Assert(t, strings.Contains(string(b), "\n- |"), "expected literal block scalar, got:\n%s", b) diff --git a/internal/pgbackrest/reconcile.go b/internal/pgbackrest/reconcile.go index 78a6ebf967..5e57865f85 100644 --- a/internal/pgbackrest/reconcile.go +++ b/internal/pgbackrest/reconcile.go @@ -125,6 +125,10 @@ func AddConfigToInstancePod( Path: serverConfigProjectionPath, }) secret.Secret.Items = append(secret.Secret.Items, clientCertificates()...) + if cluster.CompareVersion("2.5.0") < 0 { + t := true + secret.Secret.Optional = &t + } // Start with a copy of projections specified in the cluster. Items later in // the list take precedence over earlier items (that is, last write wins). @@ -304,7 +308,7 @@ func addServerContainerAndVolume( Command: []string{"pgbackrest", "server"}, Image: config.PGBackRestContainerImage(cluster), ImagePullPolicy: cluster.Spec.ImagePullPolicy, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(cluster.CompareVersion("2.5.0") >= 0), LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ @@ -342,10 +346,10 @@ func addServerContainerAndVolume( reloader := corev1.Container{ Name: naming.ContainerPGBackRestConfig, - Command: reloadCommand(naming.ContainerPGBackRestConfig), + Command: reloadCommand(naming.ContainerPGBackRestConfig, cluster.CompareVersion("2.5.0") >= 0), Image: container.Image, ImagePullPolicy: container.ImagePullPolicy, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(cluster.CompareVersion("2.5.0") >= 0), // The configuration mount is appended by [addConfigVolumeAndMounts]. VolumeMounts: []corev1.VolumeMount{serverVolumeMount}, diff --git a/internal/pgbackrest/reconcile_test.go b/internal/pgbackrest/reconcile_test.go index 66fc2b62e3..da8a4631b0 100644 --- a/internal/pgbackrest/reconcile_test.go +++ b/internal/pgbackrest/reconcile_test.go @@ -180,7 +180,13 @@ func TestAddRepoVolumesToPod(t *testing.T) { } func TestAddConfigToInstancePod(t *testing.T) { - cluster := v1beta1.PostgresCluster{} + cluster := v1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + naming.LabelVersion: "2.5.0", + }, + }, + } cluster.Name = "hippo" cluster.Default() @@ -580,6 +586,9 @@ func TestAddServerToInstancePod(t *testing.T) { cluster := v1beta1.PostgresCluster{} cluster.Name = "hippo" cluster.Default() + cluster.SetLabels(map[string]string{ + naming.LabelVersion: "2.5.0", + }) pod := corev1.PodSpec{ Containers: []corev1.Container{ @@ -840,6 +849,9 @@ func TestAddServerToRepoPod(t *testing.T) { cluster := v1beta1.PostgresCluster{} cluster.Name = "hippo" cluster.Default() + cluster.SetLabels(map[string]string{ + naming.LabelVersion: "2.5.0", + }) pod := corev1.PodSpec{ Containers: []corev1.Container{ diff --git a/internal/pgbouncer/config.go b/internal/pgbouncer/config.go index 2beb348b67..399f5745ae 100644 --- a/internal/pgbouncer/config.go +++ b/internal/pgbouncer/config.go @@ -239,7 +239,7 @@ func podConfigFiles( // reloadCommand returns an entrypoint that convinces PgBouncer to reload // configuration files. The process will appear as name in `ps` and `top`. -func reloadCommand(name string) []string { +func reloadCommand(name string, post250 bool) []string { // Use a Bash loop to periodically check the mtime of the mounted // configuration volume. When it changes, signal PgBouncer and print the // observed timestamp. @@ -249,7 +249,18 @@ func reloadCommand(name string) []string { // descriptor gets closed and reopened to use the builtin `[ -nt` to check // mtimes. // - https://unix.stackexchange.com/a/407383 - const script = ` + script := ` +exec {fd}<> <(:) +while read -r -t 5 -u "${fd}" || true; do + if [ "${directory}" -nt "/proc/self/fd/${fd}" ] && pkill -HUP --exact pgbouncer + then + exec {fd}>&- && exec {fd}<> <(:) + stat --format='Loaded configuration dated %y' "${directory}" + fi +done +` + if post250 { + script = ` exec {fd}<> <(:||:) while read -r -t 5 -u "${fd}" ||:; do if [[ "${directory}" -nt "/proc/self/fd/${fd}" ]] && pkill -HUP --exact pgbouncer @@ -259,6 +270,7 @@ while read -r -t 5 -u "${fd}" ||:; do fi done ` + } // Elide the above script from `ps` and `top` by wrapping it in a function // and calling that. diff --git a/internal/pgbouncer/config_test.go b/internal/pgbouncer/config_test.go index 3f3b8320f5..3c5dddf5f6 100644 --- a/internal/pgbouncer/config_test.go +++ b/internal/pgbouncer/config_test.go @@ -212,7 +212,7 @@ func TestPodConfigFiles(t *testing.T) { func TestReloadCommand(t *testing.T) { shellcheck := require.ShellCheck(t) - command := reloadCommand("some-name") + command := reloadCommand("some-name", true) // Expect a bash command with an inline script. assert.DeepEqual(t, command[:3], []string{"bash", "-ceu", "--"}) diff --git a/internal/pgbouncer/reconcile.go b/internal/pgbouncer/reconcile.go index f617af6e79..37439756fb 100644 --- a/internal/pgbouncer/reconcile.go +++ b/internal/pgbouncer/reconcile.go @@ -145,7 +145,7 @@ func Pod( Image: config.PGBouncerContainerImage(inCluster), ImagePullPolicy: inCluster.Spec.ImagePullPolicy, Resources: inCluster.Spec.Proxy.PGBouncer.Resources, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(inCluster.CompareVersion("2.5.0") >= 0), Ports: []corev1.ContainerPort{{ Name: naming.PortPGBouncer, @@ -162,10 +162,10 @@ func Pod( reloader := corev1.Container{ Name: naming.ContainerPGBouncerConfig, - Command: reloadCommand(naming.ContainerPGBouncerConfig), + Command: reloadCommand(naming.ContainerPGBouncerConfig, inCluster.CompareVersion("2.5.0") >= 0), Image: container.Image, ImagePullPolicy: container.ImagePullPolicy, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(inCluster.CompareVersion("2.5.0") >= 0), VolumeMounts: []corev1.VolumeMount{configVolumeMount}, } diff --git a/internal/pgbouncer/reconcile_test.go b/internal/pgbouncer/reconcile_test.go index 53e54b0ace..ebd97432c8 100644 --- a/internal/pgbouncer/reconcile_test.go +++ b/internal/pgbouncer/reconcile_test.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "github.com/percona/percona-postgresql-operator/internal/feature" + "github.com/percona/percona-postgresql-operator/internal/naming" "github.com/percona/percona-postgresql-operator/internal/pki" "github.com/percona/percona-postgresql-operator/internal/postgres" "github.com/percona/percona-postgresql-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" @@ -112,6 +113,10 @@ func TestPod(t *testing.T) { secret := new(corev1.Secret) pod := new(corev1.PodSpec) + cluster.SetLabels(map[string]string{ + naming.LabelVersion: "2.5.0", + }) + call := func() { Pod(ctx, cluster, configMap, primaryCertificate, secret, pod) } t.Run("Disabled", func(t *testing.T) { diff --git a/internal/postgres/config.go b/internal/postgres/config.go index c4dd5621eb..3ab3fe26e2 100644 --- a/internal/postgres/config.go +++ b/internal/postgres/config.go @@ -52,6 +52,15 @@ recreate() ( // bashSafeLink is a Bash function that moves an existing file or directory // and replaces it with a symbolic link. + bashSafeLinkPre250 = ` +safelink() ( + local desired="$1" name="$2" current + current=$(realpath "${name}") + if [ "${current}" = "${desired}" ]; then return; fi + set -x; mv --no-target-directory "${current}" "${desired}" + ln --no-dereference --force --symbolic "${desired}" "${name}" +) +` bashSafeLink = ` safelink() ( local desired="$1" name="$2" current @@ -154,7 +163,7 @@ func Environment(cluster *v1beta1.PostgresCluster) []corev1.EnvVar { // reloadCommand returns an entrypoint that convinces PostgreSQL to reload // certificate files when they change. The process will appear as name in `ps` // and `top`. -func reloadCommand(name string) []string { +func reloadCommand(name string, post250 bool) []string { // Use a Bash loop to periodically check the mtime of the mounted // certificate volume. When it changes, copy the replication certificate, // signal PostgreSQL, and print the observed timestamp. @@ -178,6 +187,27 @@ func reloadCommand(name string) []string { // mtimes. // - https://unix.stackexchange.com/a/407383 script := fmt.Sprintf(` +declare -r directory=%q +exec {fd}<> <(:) +while read -r -t 5 -u "${fd}" || true; do + if [ "${directory}" -nt "/proc/self/fd/${fd}" ] && + install -D --mode=0600 -t %q "${directory}"/{%s,%s,%s} && + pkill -HUP --exact --parent=1 postgres + then + exec {fd}>&- && exec {fd}<> <(:) + stat --format='Loaded certificates dated %%y' "${directory}" + fi +done +`, + naming.CertMountPath, + naming.ReplicationTmp, + naming.ReplicationCertPath, + naming.ReplicationPrivateKeyPath, + naming.ReplicationCACertPath, + ) + + if post250 { + script = fmt.Sprintf(` # Parameters for curl when managing autogrow annotation. APISERVER="https://kubernetes.default.svc" SERVICEACCOUNT="/var/run/secrets/kubernetes.io/serviceaccount" @@ -213,12 +243,13 @@ while read -r -t 5 -u "${fd}" ||:; do fi done `, - naming.CertMountPath, - naming.ReplicationTmp, - naming.ReplicationCertPath, - naming.ReplicationPrivateKeyPath, - naming.ReplicationCACertPath, - ) + naming.CertMountPath, + naming.ReplicationTmp, + naming.ReplicationCertPath, + naming.ReplicationPrivateKeyPath, + naming.ReplicationCACertPath, + ) + } // Elide the above script from `ps` and `top` by wrapping it in a function // and calling that. @@ -231,7 +262,9 @@ done // PostgreSQL. func startupCommand( ctx context.Context, - cluster *v1beta1.PostgresCluster, instance *v1beta1.PostgresInstanceSetSpec, + cluster *v1beta1.PostgresCluster, + instance *v1beta1.PostgresInstanceSetSpec, + post250 bool, ) []string { version := fmt.Sprint(cluster.Spec.PostgresVersion) walDir := WALDirectory(cluster, instance) @@ -290,7 +323,7 @@ chmod +x /tmp/pg_rewind_tde.sh } args := []string{version, walDir, naming.PGBackRestPGDataLogPath} - script := strings.Join([]string{ + statements := []string{ `declare -r expected_major_version="$1" pgwal_directory="$2" pgbrLog_directory="$3"`, // Function to print the permissions of a file or directory and its parents. @@ -306,36 +339,94 @@ chmod +x /tmp/pg_rewind_tde.sh strings.TrimSpace(bashRecreateDirectory), // Function to change a directory symlink while keeping the directory contents. - strings.TrimSpace(bashSafeLink), + func() string { + if !post250 { + return strings.TrimSpace(bashSafeLinkPre250) + } + return strings.TrimSpace(bashSafeLink) + }(), // Log the effective user ID and all the group IDs. `echo Initializing ...`, - `results 'uid' "$(id -u ||:)" 'gid' "$(id -G ||:)"`, - - // The pgbackrest spool path should be co-located with wal. If a wal volume exists, symlink the spool-path to it. - `if [[ "${pgwal_directory}" == *"pgwal/"* ]] && [[ ! -d "/pgwal/pgbackrest-spool" ]];then rm -rf "/pgdata/pgbackrest-spool" && mkdir -p "/pgwal/pgbackrest-spool" && ln --force --symbolic "/pgwal/pgbackrest-spool" "/pgdata/pgbackrest-spool";fi`, - // When a pgwal volume is removed, the symlink will be broken; force pgbackrest to recreate spool-path. - `if [[ ! -e "/pgdata/pgbackrest-spool" ]];then rm -rf /pgdata/pgbackrest-spool;fi`, + func() string { + if !post250 { + return `results 'uid' "$(id -u)" 'gid' "$(id -G)"` + } + return `results 'uid' "$(id -u ||:)" 'gid' "$(id -G ||:)"` + }(), + + func() string { + if !post250 { + return "remove" + } + + // The pgbackrest spool path should be co-located with wal. If a wal volume exists, symlink the spool-path to it. + return `if [[ "${pgwal_directory}" == *"pgwal/"* ]] && [[ ! -d "/pgwal/pgbackrest-spool" ]];then rm -rf "/pgdata/pgbackrest-spool" && mkdir -p "/pgwal/pgbackrest-spool" && ln --force --symbolic "/pgwal/pgbackrest-spool" "/pgdata/pgbackrest-spool";fi` + }(), + + func() string { + if !post250 { + return "remove" + } + + // When a pgwal volume is removed, the symlink will be broken; force pgbackrest to recreate spool-path. + return `if [[ ! -e "/pgdata/pgbackrest-spool" ]];then rm -rf /pgdata/pgbackrest-spool;fi` + }(), + + func() string { + if !post250 { + return `results 'postgres path' "$(command -v postgres)"` + } + + return `results 'postgres path' "$(command -v postgres ||:)"` + }(), + + func() string { + if !post250 { + return `results 'postgres version' "${postgres_version:=$(postgres --version)}"` + } + + return `results 'postgres version' "${postgres_version:=$(postgres --version ||:)}"` + }(), // Abort when the PostgreSQL version installed in the image does not // match the cluster spec. - `results 'postgres path' "$(command -v postgres ||:)"`, - `results 'postgres version' "${postgres_version:=$(postgres --version ||:)}"`, `[[ "${postgres_version}" =~ ") ${expected_major_version}"($|[^0-9]) ]] ||`, `halt Expected PostgreSQL version "${expected_major_version}"`, - // Abort when the configured data directory is not $PGDATA. - // - https://www.postgresql.org/docs/current/runtime-config-file-locations.html `results 'config directory' "${PGDATA:?}"`, - `postgres_data_directory=$([[ -d "${PGDATA}" ]] && postgres -C data_directory || echo "${PGDATA}")`, + + func() string { + if !post250 { + return `postgres_data_directory=$([ -d "${PGDATA}" ] && postgres -C data_directory || echo "${PGDATA}")` + } + + return `postgres_data_directory=$([[ -d "${PGDATA}" ]] && postgres -C data_directory || echo "${PGDATA}")` + }(), + `results 'data directory' "${postgres_data_directory}"`, + // Abort when the configured data directory is not $PGDATA. + // - https://www.postgresql.org/docs/current/runtime-config-file-locations.html `[[ "${postgres_data_directory}" == "${PGDATA}" ]] ||`, `halt Expected matching config and data directories`, // Determine if the data directory has been prepared for bootstrapping the cluster `bootstrap_dir="${postgres_data_directory}_bootstrap"`, - `[[ -d "${bootstrap_dir}" ]] && results 'bootstrap directory' "${bootstrap_dir}"`, - `[[ -d "${bootstrap_dir}" ]] && postgres_data_directory="${bootstrap_dir}"`, + func() string { + if !post250 { + return `[ -d "${bootstrap_dir}" ] && results 'bootstrap directory' "${bootstrap_dir}"` + } + + return `[[ -d "${bootstrap_dir}" ]] && results 'bootstrap directory' "${bootstrap_dir}"` + }(), + + func() string { + if !post250 { + return `[ -d "${bootstrap_dir}" ] && postgres_data_directory="${bootstrap_dir}"` + } + + return `[[ -d "${bootstrap_dir}" ]] && postgres_data_directory="${bootstrap_dir}"` + }(), // PostgreSQL requires its directory to be writable by only itself. // Pod "securityContext.fsGroup" sets g+w on directories for *some* @@ -381,11 +472,24 @@ chmod +x /tmp/pg_rewind_tde.sh naming.ReplicationCACert), // Add the pg_rewind wrapper script, if TDE is enabled. - pg_rewind_override, + func() string { + if len(pg_rewind_override) < 1 { + return "remove" + } + + return pg_rewind_override + }(), tablespaceCmd, + // When the data directory is empty, there's nothing more to do. - `[[ -f "${postgres_data_directory}/PG_VERSION" ]] || exit 0`, + func() string { + if !post250 { + return `[ -f "${postgres_data_directory}/PG_VERSION" ] || exit 0` + } + + return `[[ -f "${postgres_data_directory}/PG_VERSION" ]] || exit 0` + }(), // Abort when the data directory is not empty and its version does not // match the cluster spec. @@ -409,7 +513,13 @@ chmod +x /tmp/pg_rewind_tde.sh // - https://git.postgresql.org/gitweb/?p=postgresql.git;f=src/bin/initdb/initdb.c;hb=REL_13_0#l2718 // - https://git.postgresql.org/gitweb/?p=postgresql.git;f=src/bin/pg_basebackup/pg_basebackup.c;hb=REL_13_0#l2621 `safelink "${pgwal_directory}" "${postgres_data_directory}/pg_wal"`, - `results 'wal directory' "$(realpath "${postgres_data_directory}/pg_wal" ||:)"`, + func() string { + if !post250 { + return `results 'wal directory' "$(realpath "${postgres_data_directory}/pg_wal")"` + } + + return `results 'wal directory' "$(realpath "${postgres_data_directory}/pg_wal" ||:)"` + }(), // Early versions of PGO create replicas with a recovery signal file. // Patroni also creates a standby signal file before starting Postgres, @@ -419,7 +529,17 @@ chmod +x /tmp/pg_rewind_tde.sh // - https://git.postgresql.org/gitweb/?p=postgresql.git;f=src/backend/access/transam/xlog.c;hb=REL_12_0#l5318 // TODO(cbandy): Remove this after 5.0 is EOL. `rm -f "${postgres_data_directory}/recovery.signal"`, - }, "\n") + } + script := strings.Join(filter(statements, func(s string) bool { return s != "remove" }), "\n") return append([]string{"bash", "-ceu", "--", script, "startup"}, args...) } + +func filter(ss []string, test func(string) bool) (ret []string) { + for _, s := range ss { + if test(s) { + ret = append(ret, s) + } + } + return ret +} diff --git a/internal/postgres/config_test.go b/internal/postgres/config_test.go index f03887a474..03849485f2 100644 --- a/internal/postgres/config_test.go +++ b/internal/postgres/config_test.go @@ -473,7 +473,7 @@ func TestStartupCommand(t *testing.T) { instance := new(v1beta1.PostgresInstanceSetSpec) ctx := context.Background() - command := startupCommand(ctx, cluster, instance) + command := startupCommand(ctx, cluster, instance, true) // Expect a bash command with an inline script. assert.DeepEqual(t, command[:3], []string{"bash", "-ceu", "--"}) @@ -508,7 +508,7 @@ func TestStartupCommand(t *testing.T) { }, }, } - command := startupCommand(ctx, cluster, instance) + command := startupCommand(ctx, cluster, instance, true) assert.Assert(t, len(command) > 3) assert.Assert(t, strings.Contains(command[3], `cat << "EOF" > /tmp/pg_rewind_tde.sh #!/bin/sh diff --git a/internal/postgres/reconcile.go b/internal/postgres/reconcile.go index a898a4cba4..b84bd0a73a 100644 --- a/internal/postgres/reconcile.go +++ b/internal/postgres/reconcile.go @@ -185,20 +185,24 @@ func InstancePod(ctx context.Context, Protocol: corev1.ProtocolTCP, }}, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(inCluster.CompareVersion("2.5.0") >= 0), VolumeMounts: dbContainerMounts, } reloader := corev1.Container{ Name: naming.ContainerClientCertCopy, - Command: reloadCommand(naming.ContainerClientCertCopy), + Command: reloadCommand(naming.ContainerClientCertCopy, inCluster.CompareVersion("2.5.0") >= 0), Image: container.Image, ImagePullPolicy: container.ImagePullPolicy, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(inCluster.CompareVersion("2.5.0") >= 0), - VolumeMounts: []corev1.VolumeMount{certVolumeMount, dataVolumeMount}, + VolumeMounts: []corev1.VolumeMount{certVolumeMount}, + } + + if inCluster.CompareVersion("2.5.0") >= 0 { + reloader.VolumeMounts = append(reloader.VolumeMounts, dataVolumeMount) } if inInstanceSpec.Sidecars != nil && @@ -210,13 +214,13 @@ func InstancePod(ctx context.Context, startup := corev1.Container{ Name: naming.ContainerPostgresStartup, - Command: startupCommand(ctx, inCluster, inInstanceSpec), + Command: startupCommand(ctx, inCluster, inInstanceSpec, inCluster.CompareVersion("2.5.0") >= 0), Env: Environment(inCluster), Image: container.Image, ImagePullPolicy: container.ImagePullPolicy, Resources: container.Resources, - SecurityContext: initialize.RestrictedSecurityContext(), + SecurityContext: initialize.RestrictedSecurityContext(inCluster.CompareVersion("2.5.0") >= 0), VolumeMounts: []corev1.VolumeMount{certVolumeMount, dataVolumeMount}, } diff --git a/internal/postgres/reconcile_test.go b/internal/postgres/reconcile_test.go index 67ae10cd4c..fab30edd03 100644 --- a/internal/postgres/reconcile_test.go +++ b/internal/postgres/reconcile_test.go @@ -77,6 +77,9 @@ func TestInstancePod(t *testing.T) { cluster.Default() cluster.Spec.ImagePullPolicy = corev1.PullAlways cluster.Spec.PostgresVersion = 11 + cluster.SetLabels(map[string]string{ + naming.LabelVersion: "2.5.0", + }) dataVolume := new(corev1.PersistentVolumeClaim) dataVolume.Name = "datavol" @@ -280,7 +283,6 @@ initContainers: halt "$(permissions "${pgbrLog_directory}" ||:)" install -D --mode=0600 -t "/tmp/replication" "/pgconf/tls/replication"/{tls.crt,tls.key,ca.crt} - [[ -f "${postgres_data_directory}/PG_VERSION" ]] || exit 0 results 'data version' "${postgres_data_version:=$(< "${postgres_data_directory}/PG_VERSION")}" [[ "${postgres_data_version}" == "${expected_major_version}" ]] || diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go index 58939e0e75..66151d121b 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go @@ -5,7 +5,6 @@ import ( "os" gover "github.com/hashicorp/go-version" - v "github.com/hashicorp/go-version" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -334,12 +333,12 @@ func (cr *PerconaPGCluster) ToCrunchy(ctx context.Context, postgresCluster *crun return postgresCluster, nil } -func (cr *PerconaPGCluster) Version() *v.Version { - return v.Must(v.NewVersion(cr.Spec.CRVersion)) +func (cr *PerconaPGCluster) Version() *gover.Version { + return gover.Must(gover.NewVersion(cr.Spec.CRVersion)) } func (cr *PerconaPGCluster) CompareVersion(ver string) int { - return cr.Version().Compare(v.Must(v.NewVersion(ver))) + return cr.Version().Compare(gover.Must(gover.NewVersion(ver))) } type AppState string diff --git a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go index 4dcf64ac5e..39d903e47d 100644 --- a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go +++ b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( "fmt" + gover "github.com/hashicorp/go-version" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -733,3 +734,13 @@ func NewPostgresCluster() *PostgresCluster { cluster.SetGroupVersionKind(GroupVersion.WithKind("PostgresCluster")) return cluster } + +const LabelVersion = "pgv2.percona.com/version" + +func (cr *PostgresCluster) CompareVersion(ver string) int { + crVersion, err := gover.NewVersion(cr.Labels[LabelVersion]) + if err != nil { + return -2 + } + return crVersion.Compare(gover.Must(gover.NewVersion(ver))) +} From e413d3ac11ff2b7967b3e0c053ee98bed29de10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 1 Oct 2024 14:43:32 +0300 Subject: [PATCH 2/6] fix envtest --- .../postgrescluster/pgbackrest_test.go | 17 ++++++++++++++++- .../controller/postgrescluster/volumes_test.go | 8 +++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/controller/postgrescluster/pgbackrest_test.go b/internal/controller/postgrescluster/pgbackrest_test.go index 3089ae3f41..1ab3f8c5c5 100644 --- a/internal/controller/postgrescluster/pgbackrest_test.go +++ b/internal/controller/postgrescluster/pgbackrest_test.go @@ -2683,7 +2683,13 @@ func TestGenerateRestoreJobIntent(t *testing.T) { } t.Run("empty", func(t *testing.T) { - err := r.generateRestoreJobIntent(&v1beta1.PostgresCluster{}, "", "", + err := r.generateRestoreJobIntent(&v1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + naming.LabelVersion: "2.5.0", + }, + }, + }, "", "", []string{}, []corev1.VolumeMount{}, []corev1.Volume{}, &v1beta1.PostgresClusterDataSource{}, &batchv1.Job{}) assert.NilError(t, err) @@ -2727,6 +2733,9 @@ func TestGenerateRestoreJobIntent(t *testing.T) { cluster := &v1beta1.PostgresCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "test", + Labels: map[string]string{ + naming.LabelVersion: "2.5.0", + }, }, Spec: v1beta1.PostgresClusterSpec{ Metadata: &v1beta1.Metadata{ @@ -2893,6 +2902,9 @@ func TestObserveRestoreEnv(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: clusterName, Namespace: namespace, + Labels: map[string]string{ + naming.LabelVersion: "2.5.0", + }, }, } meta := naming.PGBackRestRestoreJob(cluster) @@ -3107,6 +3119,9 @@ func TestPrepareForRestore(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: clusterName, Namespace: namespace, + Labels: map[string]string{ + naming.LabelVersion: "2.5.0", + }, }, } meta := naming.PGBackRestRestoreJob(cluster) diff --git a/internal/controller/postgrescluster/volumes_test.go b/internal/controller/postgrescluster/volumes_test.go index 4431d13946..f5437c9485 100644 --- a/internal/controller/postgrescluster/volumes_test.go +++ b/internal/controller/postgrescluster/volumes_test.go @@ -49,6 +49,9 @@ func TestHandlePersistentVolumeClaimError(t *testing.T) { cluster := new(v1beta1.PostgresCluster) cluster.Namespace = "ns1" cluster.Name = "pg2" + cluster.SetLabels(map[string]string{ + naming.LabelVersion: "2.5.0", + }) reset := func() { cluster.Status.Conditions = cluster.Status.Conditions[:0] @@ -395,7 +398,7 @@ func TestReconcileConfigureExistingPVCs(t *testing.T) { Name: "testcluster", Namespace: ns.GetName(), Labels: map[string]string{ - naming.LabelVersion: "2.3.0", + naming.LabelVersion: "2.5.0", }, }, Spec: v1beta1.PostgresClusterSpec{ @@ -657,6 +660,9 @@ func TestReconcileMoveDirectories(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "testcluster", Namespace: ns.GetName(), + Labels: map[string]string{ + naming.LabelVersion: "2.5.0", + }, }, Spec: v1beta1.PostgresClusterSpec{ PostgresVersion: 13, From 899605488617227146c113d7f6c45719f5616be0 Mon Sep 17 00:00:00 2001 From: Eleonora Zinchenko Date: Tue, 1 Oct 2024 21:54:46 +0300 Subject: [PATCH 3/6] Add cluster restart check to upgrade-minor tests --- e2e-tests/functions | 21 ++- e2e-tests/tests/upgrade-minor/01-assert.yaml | 1 + e2e-tests/tests/upgrade-minor/04-assert.yaml | 2 +- .../upgrade-minor/04-upgrade-operator.yaml | 2 +- e2e-tests/tests/upgrade-minor/05-assert.yaml | 61 ++----- .../05-sleep-after-operator-update.yaml | 6 + e2e-tests/tests/upgrade-minor/06-assert.yaml | 167 ++++++++++++++++++ ...e-cluster.yaml => 06-upgrade-cluster.yaml} | 0 ...{06-write-data.yaml => 07-write-data.yaml} | 0 .../{07-assert.yaml => 08-assert.yaml} | 0 ...primary.yaml => 08-read-from-primary.yaml} | 0 11 files changed, 212 insertions(+), 48 deletions(-) create mode 100644 e2e-tests/tests/upgrade-minor/05-sleep-after-operator-update.yaml create mode 100644 e2e-tests/tests/upgrade-minor/06-assert.yaml rename e2e-tests/tests/upgrade-minor/{05-upgrade-cluster.yaml => 06-upgrade-cluster.yaml} (100%) rename e2e-tests/tests/upgrade-minor/{06-write-data.yaml => 07-write-data.yaml} (100%) rename e2e-tests/tests/upgrade-minor/{07-assert.yaml => 08-assert.yaml} (100%) rename e2e-tests/tests/upgrade-minor/{07-read-from-primary.yaml => 08-read-from-primary.yaml} (100%) diff --git a/e2e-tests/functions b/e2e-tests/functions index 3f9ceca9ee..e32ac221d7 100644 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -2,7 +2,7 @@ # set root repo relatively to a test dir ROOT_REPO=${ROOT_REPO:-$(realpath ../../..)} -CERT_MANAGER_VER="1.14.2" +CERT_MANAGER_VER="1.15.3" test_name=$(basename "$(pwd)") source "${ROOT_REPO}/e2e-tests/vars.sh" @@ -58,6 +58,25 @@ deploy_operator() { | kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f - } +update_operator() { + local cw_prefix="" + + if [[ $OPERATOR_NS ]]; then + cw_prefix="cw-" + fi + + kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/crd.yaml" + kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/${cw_prefix}rbac.yaml" + + local disable_telemetry=true + if [ "${test_name}" == "telemetry-transfer" ]; then + disable_telemetry=false + fi + + kubectl -n "${OPERATOR_NS:-$NAMESPACE}" patch deployment percona-postgresql-operator -p \ + '{"spec":{"template":{"spec":{"containers":[{"name":"operator","image":"'${IMAGE}'"}]}}}}' +} + deploy_operator_gh() { local git_tag="$1" local cw_prefix="" diff --git a/e2e-tests/tests/upgrade-minor/01-assert.yaml b/e2e-tests/tests/upgrade-minor/01-assert.yaml index 553681c646..19551ad384 100644 --- a/e2e-tests/tests/upgrade-minor/01-assert.yaml +++ b/e2e-tests/tests/upgrade-minor/01-assert.yaml @@ -138,6 +138,7 @@ apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: name: upgrade-minor + generation: 1 status: pgbouncer: ready: 3 diff --git a/e2e-tests/tests/upgrade-minor/04-assert.yaml b/e2e-tests/tests/upgrade-minor/04-assert.yaml index 064501ea0a..93804ea3e2 100644 --- a/e2e-tests/tests/upgrade-minor/04-assert.yaml +++ b/e2e-tests/tests/upgrade-minor/04-assert.yaml @@ -21,4 +21,4 @@ metadata: name: check-operator-deploy-status timeout: 45 commands: - - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 + - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1,status.observedGeneration=2 diff --git a/e2e-tests/tests/upgrade-minor/04-upgrade-operator.yaml b/e2e-tests/tests/upgrade-minor/04-upgrade-operator.yaml index bc93c023cd..7d1a67b0e1 100644 --- a/e2e-tests/tests/upgrade-minor/04-upgrade-operator.yaml +++ b/e2e-tests/tests/upgrade-minor/04-upgrade-operator.yaml @@ -8,5 +8,5 @@ commands: source ../../functions - deploy_operator + update_operator kubectl wait -n ${OPERATOR_NS:-$NAMESPACE} --timeout 30s --for=jsonpath='{.spec.template.spec.containers[0].image}'=$IMAGE deployment/percona-postgresql-operator diff --git a/e2e-tests/tests/upgrade-minor/05-assert.yaml b/e2e-tests/tests/upgrade-minor/05-assert.yaml index 25808acbe7..478ec3551c 100644 --- a/e2e-tests/tests/upgrade-minor/05-assert.yaml +++ b/e2e-tests/tests/upgrade-minor/05-assert.yaml @@ -1,6 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 300 +timeout: 10 --- kind: StatefulSet apiVersion: apps/v1 @@ -22,7 +22,7 @@ metadata: controller: true blockOwnerDeletion: true status: - observedGeneration: 3 + observedGeneration: 1 replicas: 1 readyReplicas: 1 currentReplicas: 1 @@ -32,7 +32,7 @@ status: kind: StatefulSet apiVersion: apps/v1 metadata: - generation: 3 + generation: 1 labels: app.kubernetes.io/component: pg app.kubernetes.io/instance: upgrade-minor @@ -48,26 +48,13 @@ metadata: name: upgrade-minor controller: true blockOwnerDeletion: true -spec: - template: - metadata: - labels: - app.kubernetes.io/component: pg - app.kubernetes.io/instance: upgrade-minor - app.kubernetes.io/managed-by: percona-postgresql-operator - app.kubernetes.io/name: percona-postgresql - app.kubernetes.io/part-of: percona-postgresql - postgres-operator.crunchydata.com/cluster: upgrade-minor - postgres-operator.crunchydata.com/data: postgres - postgres-operator.crunchydata.com/instance-set: instance1 - postgres-operator.crunchydata.com/patroni: upgrade-minor-ha status: - availableReplicas: 1 - collisionCount: 0 - observedGeneration: 3 - readyReplicas: 1 + observedGeneration: 1 replicas: 1 + readyReplicas: 1 + currentReplicas: 1 updatedReplicas: 1 + collisionCount: 0 --- kind: Deployment apiVersion: apps/v1 @@ -82,7 +69,7 @@ metadata: postgres-operator.crunchydata.com/cluster: upgrade-minor postgres-operator.crunchydata.com/role: pgbouncer annotations: - deployment.kubernetes.io/revision: '3' + deployment.kubernetes.io/revision: '1' ownerReferences: - apiVersion: postgres-operator.crunchydata.com/v1beta1 kind: PostgresCluster @@ -90,31 +77,15 @@ metadata: controller: true blockOwnerDeletion: true status: - observedGeneration: 3 + observedGeneration: 1 replicas: 3 updatedReplicas: 3 readyReplicas: 3 --- -kind: Job -apiVersion: batch/v1 -metadata: - labels: - postgres-operator.crunchydata.com/cluster: upgrade-minor - postgres-operator.crunchydata.com/pgbackrest: '' - postgres-operator.crunchydata.com/pgbackrest-backup: replica-create - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true -status: - succeeded: 1 ---- apiVersion: postgres-operator.crunchydata.com/v1beta1 kind: PostgresCluster metadata: - generation: 2 + generation: 1 labels: e2e: upgrade-minor name: upgrade-minor @@ -132,7 +103,7 @@ status: readyReplicas: 3 replicas: 3 updatedReplicas: 3 - observedGeneration: 2 + observedGeneration: 1 pgbackrest: repoHost: apiVersion: apps/v1 @@ -151,17 +122,17 @@ status: apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: - generation: 2 name: upgrade-minor + generation: 1 status: pgbouncer: ready: 3 size: 3 postgres: instances: - - name: instance1 - ready: 3 - size: 3 + - name: instance1 + ready: 3 + size: 3 ready: 3 size: 3 - state: ready + state: ready \ No newline at end of file diff --git a/e2e-tests/tests/upgrade-minor/05-sleep-after-operator-update.yaml b/e2e-tests/tests/upgrade-minor/05-sleep-after-operator-update.yaml new file mode 100644 index 0000000000..506a371d4e --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/05-sleep-after-operator-update.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 30 +commands: + - script: |- + sleep 30 diff --git a/e2e-tests/tests/upgrade-minor/06-assert.yaml b/e2e-tests/tests/upgrade-minor/06-assert.yaml new file mode 100644 index 0000000000..25808acbe7 --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/06-assert.yaml @@ -0,0 +1,167 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: upgrade-minor-repo-host + labels: + app.kubernetes.io/instance: upgrade-minor + app.kubernetes.io/managed-by: percona-postgresql-operator + app.kubernetes.io/name: percona-postgresql + app.kubernetes.io/part-of: percona-postgresql + postgres-operator.crunchydata.com/cluster: upgrade-minor + postgres-operator.crunchydata.com/data: pgbackrest + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-dedicated: '' + ownerReferences: + - apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: upgrade-minor + controller: true + blockOwnerDeletion: true +status: + observedGeneration: 3 + replicas: 1 + readyReplicas: 1 + currentReplicas: 1 + updatedReplicas: 1 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + generation: 3 + labels: + app.kubernetes.io/component: pg + app.kubernetes.io/instance: upgrade-minor + app.kubernetes.io/managed-by: percona-postgresql-operator + app.kubernetes.io/name: percona-postgresql + app.kubernetes.io/part-of: percona-postgresql + postgres-operator.crunchydata.com/cluster: upgrade-minor + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + ownerReferences: + - apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: upgrade-minor + controller: true + blockOwnerDeletion: true +spec: + template: + metadata: + labels: + app.kubernetes.io/component: pg + app.kubernetes.io/instance: upgrade-minor + app.kubernetes.io/managed-by: percona-postgresql-operator + app.kubernetes.io/name: percona-postgresql + app.kubernetes.io/part-of: percona-postgresql + postgres-operator.crunchydata.com/cluster: upgrade-minor + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 + postgres-operator.crunchydata.com/patroni: upgrade-minor-ha +status: + availableReplicas: 1 + collisionCount: 0 + observedGeneration: 3 + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: upgrade-minor-pgbouncer + labels: + app.kubernetes.io/component: pgbouncer + app.kubernetes.io/instance: upgrade-minor + app.kubernetes.io/managed-by: percona-postgresql-operator + app.kubernetes.io/name: percona-postgresql + app.kubernetes.io/part-of: percona-postgresql + postgres-operator.crunchydata.com/cluster: upgrade-minor + postgres-operator.crunchydata.com/role: pgbouncer + annotations: + deployment.kubernetes.io/revision: '3' + ownerReferences: + - apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: upgrade-minor + controller: true + blockOwnerDeletion: true +status: + observedGeneration: 3 + replicas: 3 + updatedReplicas: 3 + readyReplicas: 3 +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: upgrade-minor + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 +--- +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + generation: 2 + labels: + e2e: upgrade-minor + name: upgrade-minor + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster + name: upgrade-minor + controller: true + blockOwnerDeletion: true + finalizers: + - postgres-operator.crunchydata.com/finalizer +status: + instances: + - name: instance1 + readyReplicas: 3 + replicas: 3 + updatedReplicas: 3 + observedGeneration: 2 + pgbackrest: + repoHost: + apiVersion: apps/v1 + kind: StatefulSet + ready: true + repos: + - bound: true + name: repo1 + replicaCreateBackupComplete: true + stanzaCreated: true + proxy: + pgBouncer: + readyReplicas: 3 + replicas: 3 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + generation: 2 + name: upgrade-minor +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/upgrade-minor/05-upgrade-cluster.yaml b/e2e-tests/tests/upgrade-minor/06-upgrade-cluster.yaml similarity index 100% rename from e2e-tests/tests/upgrade-minor/05-upgrade-cluster.yaml rename to e2e-tests/tests/upgrade-minor/06-upgrade-cluster.yaml diff --git a/e2e-tests/tests/upgrade-minor/06-write-data.yaml b/e2e-tests/tests/upgrade-minor/07-write-data.yaml similarity index 100% rename from e2e-tests/tests/upgrade-minor/06-write-data.yaml rename to e2e-tests/tests/upgrade-minor/07-write-data.yaml diff --git a/e2e-tests/tests/upgrade-minor/07-assert.yaml b/e2e-tests/tests/upgrade-minor/08-assert.yaml similarity index 100% rename from e2e-tests/tests/upgrade-minor/07-assert.yaml rename to e2e-tests/tests/upgrade-minor/08-assert.yaml diff --git a/e2e-tests/tests/upgrade-minor/07-read-from-primary.yaml b/e2e-tests/tests/upgrade-minor/08-read-from-primary.yaml similarity index 100% rename from e2e-tests/tests/upgrade-minor/07-read-from-primary.yaml rename to e2e-tests/tests/upgrade-minor/08-read-from-primary.yaml From 1543c96816ff7c72c474fb49e714317f1a6cc91f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Wed, 2 Oct 2024 09:47:18 +0300 Subject: [PATCH 4/6] K8SPG-643: Always try stanza-upgrade after a failed stanza-create --- .../controller/postgrescluster/pgbackrest.go | 3 +- internal/pgbackrest/pgbackrest.go | 28 +++---------------- internal/pgbackrest/pgbackrest_test.go | 7 ++--- 3 files changed, 8 insertions(+), 30 deletions(-) diff --git a/internal/controller/postgrescluster/pgbackrest.go b/internal/controller/postgrescluster/pgbackrest.go index 3ac264a5a3..5cf4b72e26 100644 --- a/internal/controller/postgrescluster/pgbackrest.go +++ b/internal/controller/postgrescluster/pgbackrest.go @@ -2649,8 +2649,7 @@ func (r *Reconciler) reconcileStanzaCreate(ctx context.Context, } // Always attempt to create pgBackRest stanza first - configHashMismatch, err := pgbackrest.Executor(exec).StanzaCreateOrUpgrade(ctx, configHash, - false, postgresCluster) + configHashMismatch, err := pgbackrest.Executor(exec).StanzaCreateOrUpgrade(ctx, configHash, postgresCluster) if err != nil { // record and log any errors resulting from running the stanza-create command r.Recorder.Event(postgresCluster, corev1.EventTypeWarning, EventUnableToCreateStanzas, diff --git a/internal/pgbackrest/pgbackrest.go b/internal/pgbackrest/pgbackrest.go index 675d1b7b74..6dfbad7eb8 100644 --- a/internal/pgbackrest/pgbackrest.go +++ b/internal/pgbackrest/pgbackrest.go @@ -20,11 +20,9 @@ import ( "context" "fmt" "io" - "strings" "github.com/pkg/errors" - "github.com/percona/percona-postgresql-operator/internal/logging" "github.com/percona/percona-postgresql-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -36,10 +34,6 @@ const ( // errMsgStaleReposWithVolumesConfig is the error message displayed when a volume-backed repo has been // configured, but the configuration has not yet propagated into the container. errMsgStaleReposWithVolumesConfig = "postgres operator error: pgBackRest stale volume-backed repo configuration" - - // errMsgBackupDbMismatch is the error message returned from pgBackRest when PG versions - // or PG system identifiers do not match between the PG instance and the existing stanza - errMsgBackupDbMismatch = "backup and archive info files exist but do not match the database" ) // Executor calls "pgbackrest" commands @@ -58,15 +52,10 @@ type Executor func( // from running (with a config mismatch indicating that the pgBackRest configuration as stored in // the cluster's pgBackRest ConfigMap has not yet propagated to the Pod). func (exec Executor) StanzaCreateOrUpgrade(ctx context.Context, configHash string, - upgrade bool, postgresCluster *v1beta1.PostgresCluster) (bool, error) { + postgresCluster *v1beta1.PostgresCluster) (bool, error) { var stdout, stderr bytes.Buffer - stanzaCmd := "create" - if upgrade { - stanzaCmd = "upgrade" - } - var reposWithVolumes []v1beta1.PGBackRestRepo for _, repo := range postgresCluster.Spec.Backups.PGBackRest.Repos { if repo.Volume != nil { @@ -95,24 +84,21 @@ func (exec Executor) StanzaCreateOrUpgrade(ctx context.Context, configHash strin // Otherwise, it runs the pgbackrest command, which will either be "stanza-create" or // "stanza-upgrade", depending on the value of the boolean "upgrade" parameter. const script = ` -declare -r hash="$1" stanza="$2" hash_msg="$3" vol_msg="$4" cmd="$5" check_repo_cmd="$6" +declare -r hash="$1" stanza="$2" hash_msg="$3" vol_msg="$4" check_repo_cmd="$5" if [[ "$(< /etc/pgbackrest/conf.d/config-hash)" != "${hash}" ]]; then printf >&2 "%s" "${hash_msg}"; exit 1; elif ! bash -c "${check_repo_cmd}"; then printf >&2 "%s" "${vol_msg}"; exit 1; else - pgbackrest "${cmd}" --stanza="${stanza}" + pgbackrest stanza-create --stanza="${stanza}" || pgbackrest stanza-upgrade --stanza="${stanza}" fi ` - log := logging.FromContext(ctx) if err := exec(ctx, nil, &stdout, &stderr, "bash", "-ceu", "--", script, "-", configHash, DefaultStanzaName, errMsgConfigHashMismatch, errMsgStaleReposWithVolumesConfig, - fmt.Sprintf("stanza-%s", stanzaCmd), checkRepoCmd); err != nil { + checkRepoCmd); err != nil { errReturn := stdout.String() + " " + stderr.String() - log.Error(err, "stanza command failed", "cmd", stanzaCmd, "err", errReturn) - // if the config hashes didn't match, return true and don't return an error since this is // expected while waiting for config changes in ConfigMaps and Secrets to make it to the // container @@ -126,12 +112,6 @@ fi return true, nil } - // if the err returned from pgbackrest command is about a version mismatch - // then we should run upgrade rather than create - if strings.Contains(errReturn, errMsgBackupDbMismatch) { - return exec.StanzaCreateOrUpgrade(ctx, configHash, true, postgresCluster) - } - // if none of the above errors, return the err return false, errors.WithStack(fmt.Errorf("%w: %v", err, errReturn)) } diff --git a/internal/pgbackrest/pgbackrest_test.go b/internal/pgbackrest/pgbackrest_test.go index 85ae724a67..f86b423d68 100644 --- a/internal/pgbackrest/pgbackrest_test.go +++ b/internal/pgbackrest/pgbackrest_test.go @@ -37,18 +37,17 @@ func TestStanzaCreateOrUpgrade(t *testing.T) { ctx := context.Background() configHash := "7f5d4d5bdc" expectedCommand := []string{"bash", "-ceu", "--", ` -declare -r hash="$1" stanza="$2" hash_msg="$3" vol_msg="$4" cmd="$5" check_repo_cmd="$6" +declare -r hash="$1" stanza="$2" hash_msg="$3" vol_msg="$4" check_repo_cmd="$5" if [[ "$(< /etc/pgbackrest/conf.d/config-hash)" != "${hash}" ]]; then printf >&2 "%s" "${hash_msg}"; exit 1; elif ! bash -c "${check_repo_cmd}"; then printf >&2 "%s" "${vol_msg}"; exit 1; else - pgbackrest "${cmd}" --stanza="${stanza}" + pgbackrest stanza-create --stanza="${stanza}" || pgbackrest stanza-upgrade --stanza="${stanza}" fi `, "-", "7f5d4d5bdc", "db", "postgres operator error: pgBackRest config hash mismatch", "postgres operator error: pgBackRest stale volume-backed repo configuration", - "stanza-create", "grep repo1-path /etc/pgbackrest/conf.d/pgbackrest_instance.conf", } @@ -93,7 +92,7 @@ fi }, } - configHashMismatch, err := Executor(stanzaExec).StanzaCreateOrUpgrade(ctx, configHash, false, postgresCluster) + configHashMismatch, err := Executor(stanzaExec).StanzaCreateOrUpgrade(ctx, configHash, postgresCluster) assert.NilError(t, err) assert.Assert(t, !configHashMismatch) From 419808095eb3dad83ba3ca41cd12e4c1fa86da94 Mon Sep 17 00:00:00 2001 From: Viacheslav Sarzhan Date: Wed, 2 Oct 2024 11:04:39 +0300 Subject: [PATCH 5/6] fix upgrade-minor test --- e2e-tests/tests/upgrade-minor/06-assert.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/e2e-tests/tests/upgrade-minor/06-assert.yaml b/e2e-tests/tests/upgrade-minor/06-assert.yaml index 25808acbe7..5367bb784c 100644 --- a/e2e-tests/tests/upgrade-minor/06-assert.yaml +++ b/e2e-tests/tests/upgrade-minor/06-assert.yaml @@ -22,7 +22,7 @@ metadata: controller: true blockOwnerDeletion: true status: - observedGeneration: 3 + observedGeneration: 2 replicas: 1 readyReplicas: 1 currentReplicas: 1 @@ -32,7 +32,7 @@ status: kind: StatefulSet apiVersion: apps/v1 metadata: - generation: 3 + generation: 2 labels: app.kubernetes.io/component: pg app.kubernetes.io/instance: upgrade-minor @@ -64,7 +64,7 @@ spec: status: availableReplicas: 1 collisionCount: 0 - observedGeneration: 3 + observedGeneration: 2 readyReplicas: 1 replicas: 1 updatedReplicas: 1 @@ -81,8 +81,6 @@ metadata: app.kubernetes.io/part-of: percona-postgresql postgres-operator.crunchydata.com/cluster: upgrade-minor postgres-operator.crunchydata.com/role: pgbouncer - annotations: - deployment.kubernetes.io/revision: '3' ownerReferences: - apiVersion: postgres-operator.crunchydata.com/v1beta1 kind: PostgresCluster @@ -90,7 +88,7 @@ metadata: controller: true blockOwnerDeletion: true status: - observedGeneration: 3 + observedGeneration: 2 replicas: 3 updatedReplicas: 3 readyReplicas: 3 From 522c655e6891794075cc468b309de98d726c3d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Thu, 3 Oct 2024 11:29:44 +0300 Subject: [PATCH 6/6] improve major-upgrade --- e2e-tests/functions | 14 +++++ .../major-upgrade/01-create-cluster.yaml | 2 +- e2e-tests/tests/major-upgrade/10-assert.yaml | 20 +++++++ e2e-tests/tests/major-upgrade/11-assert.yaml | 40 ++++++-------- .../tests/major-upgrade/11-change-repo.yaml | 12 +++++ e2e-tests/tests/major-upgrade/12-assert.yaml | 43 +++++++-------- ...{11-run-backup.yaml => 12-run-backup.yaml} | 2 +- e2e-tests/tests/major-upgrade/13-assert.yaml | 54 ++++++++++++++++--- ...2-run-restore.yaml => 13-run-restore.yaml} | 2 +- e2e-tests/tests/major-upgrade/14-assert.yaml | 10 ++++ ...primary.yaml => 14-read-from-primary.yaml} | 0 e2e-tests/tests/major-upgrade/20-assert.yaml | 20 +++++++ e2e-tests/tests/major-upgrade/21-assert.yaml | 40 ++++++-------- .../tests/major-upgrade/21-change-repo.yaml | 12 +++++ e2e-tests/tests/major-upgrade/22-assert.yaml | 43 +++++++-------- ...{21-run-backup.yaml => 22-run-backup.yaml} | 2 +- e2e-tests/tests/major-upgrade/23-assert.yaml | 54 ++++++++++++++++--- ...2-run-restore.yaml => 23-run-restore.yaml} | 2 +- e2e-tests/tests/major-upgrade/24-assert.yaml | 10 ++++ ...primary.yaml => 24-read-from-primary.yaml} | 0 e2e-tests/tests/major-upgrade/30-assert.yaml | 20 +++++++ e2e-tests/tests/major-upgrade/31-assert.yaml | 40 ++++++-------- .../tests/major-upgrade/31-change-repo.yaml | 12 +++++ e2e-tests/tests/major-upgrade/32-assert.yaml | 43 +++++++-------- ...{31-run-backup.yaml => 32-run-backup.yaml} | 2 +- e2e-tests/tests/major-upgrade/33-assert.yaml | 54 ++++++++++++++++--- ...2-run-restore.yaml => 33-run-restore.yaml} | 2 +- e2e-tests/tests/major-upgrade/34-assert.yaml | 10 ++++ ...primary.yaml => 34-read-from-primary.yaml} | 0 e2e-tests/tests/major-upgrade/40-assert.yaml | 20 +++++++ e2e-tests/tests/major-upgrade/41-assert.yaml | 40 ++++++-------- .../tests/major-upgrade/41-change-repo.yaml | 12 +++++ e2e-tests/tests/major-upgrade/42-assert.yaml | 41 +++++++------- ...{41-run-backup.yaml => 42-run-backup.yaml} | 0 e2e-tests/tests/major-upgrade/43-assert.yaml | 54 ++++++++++++++++--- ...2-run-restore.yaml => 43-run-restore.yaml} | 0 e2e-tests/tests/major-upgrade/44-assert.yaml | 10 ++++ ...primary.yaml => 44-read-from-primary.yaml} | 0 38 files changed, 528 insertions(+), 214 deletions(-) create mode 100644 e2e-tests/tests/major-upgrade/11-change-repo.yaml rename e2e-tests/tests/major-upgrade/{11-run-backup.yaml => 12-run-backup.yaml} (89%) rename e2e-tests/tests/major-upgrade/{12-run-restore.yaml => 13-run-restore.yaml} (87%) create mode 100644 e2e-tests/tests/major-upgrade/14-assert.yaml rename e2e-tests/tests/major-upgrade/{13-read-from-primary.yaml => 14-read-from-primary.yaml} (100%) create mode 100644 e2e-tests/tests/major-upgrade/21-change-repo.yaml rename e2e-tests/tests/major-upgrade/{21-run-backup.yaml => 22-run-backup.yaml} (89%) rename e2e-tests/tests/major-upgrade/{22-run-restore.yaml => 23-run-restore.yaml} (87%) create mode 100644 e2e-tests/tests/major-upgrade/24-assert.yaml rename e2e-tests/tests/major-upgrade/{23-read-from-primary.yaml => 24-read-from-primary.yaml} (100%) create mode 100644 e2e-tests/tests/major-upgrade/31-change-repo.yaml rename e2e-tests/tests/major-upgrade/{31-run-backup.yaml => 32-run-backup.yaml} (89%) rename e2e-tests/tests/major-upgrade/{32-run-restore.yaml => 33-run-restore.yaml} (87%) create mode 100644 e2e-tests/tests/major-upgrade/34-assert.yaml rename e2e-tests/tests/major-upgrade/{33-read-from-primary.yaml => 34-read-from-primary.yaml} (100%) create mode 100644 e2e-tests/tests/major-upgrade/41-change-repo.yaml rename e2e-tests/tests/major-upgrade/{41-run-backup.yaml => 42-run-backup.yaml} (100%) rename e2e-tests/tests/major-upgrade/{42-run-restore.yaml => 43-run-restore.yaml} (100%) create mode 100644 e2e-tests/tests/major-upgrade/44-assert.yaml rename e2e-tests/tests/major-upgrade/{43-read-from-primary.yaml => 44-read-from-primary.yaml} (100%) diff --git a/e2e-tests/functions b/e2e-tests/functions index e32ac221d7..0c16637126 100644 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -759,3 +759,17 @@ get_container_image() { echo "${IMAGE_BASE}:${operatorVersion}-ppg${pgVersion}-${component}" } + +get_postgresql_logs() { + local pgVersion=$1 + + for pod in $(kubectl get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + local phase=$(kubectl -n ${NAMESPACE} get ${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "find /pgdata/pg${pgVersion}/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done +} diff --git a/e2e-tests/tests/major-upgrade/01-create-cluster.yaml b/e2e-tests/tests/major-upgrade/01-create-cluster.yaml index 1380fae4aa..729fbcc51f 100644 --- a/e2e-tests/tests/major-upgrade/01-create-cluster.yaml +++ b/e2e-tests/tests/major-upgrade/01-create-cluster.yaml @@ -7,7 +7,7 @@ commands: set -o xtrace source ../../functions - + get_cr \ | yq eval ' .spec.postgresVersion = 12 | diff --git a/e2e-tests/tests/major-upgrade/10-assert.yaml b/e2e-tests/tests/major-upgrade/10-assert.yaml index 66d99bc294..ae8dcb18f8 100644 --- a/e2e-tests/tests/major-upgrade/10-assert.yaml +++ b/e2e-tests/tests/major-upgrade/10-assert.yaml @@ -1,6 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 360 +commands: +- script: |- + kubectl -n ${NAMESPACE} get pg,pod,job + sleep 5 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -20,3 +24,19 @@ status: ready: 3 size: 3 state: ready +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: major-upgrade + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 diff --git a/e2e-tests/tests/major-upgrade/11-assert.yaml b/e2e-tests/tests/major-upgrade/11-assert.yaml index 525d522912..47ccba03b5 100644 --- a/e2e-tests/tests/major-upgrade/11-assert.yaml +++ b/e2e-tests/tests/major-upgrade/11-assert.yaml @@ -1,31 +1,21 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 660 +commands: +- script: |- + kubectl get postgrescluster major-upgrade \ + -n ${NAMESPACE} \ + -o yaml \ + | yq eval '.status.pgbackrest.repos' - --- -kind: Job -apiVersion: batch/v1 +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster metadata: - annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-12-to-13 - labels: - postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true + name: major-upgrade status: - succeeded: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - name: backup-after-12-to-13 -spec: - pgCluster: major-upgrade - repoName: repo1 - options: - - --type=full -status: - state: Succeeded + pgbackrest: + repos: + - bound: true + name: repo2 + replicaCreateBackupComplete: true + stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/11-change-repo.yaml b/e2e-tests/tests/major-upgrade/11-change-repo.yaml new file mode 100644 index 0000000000..22094b19f4 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/11-change-repo.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + kubectl patch \ + -n $NAMESPACE \ + perconapgcluster major-upgrade \ + --type='json' \ + -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo2"}]' diff --git a/e2e-tests/tests/major-upgrade/12-assert.yaml b/e2e-tests/tests/major-upgrade/12-assert.yaml index cb8a90c13f..d1790eabf5 100644 --- a/e2e-tests/tests/major-upgrade/12-assert.yaml +++ b/e2e-tests/tests/major-upgrade/12-assert.yaml @@ -1,30 +1,31 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 +timeout: 660 --- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGRestore +kind: Job +apiVersion: batch/v1 metadata: - name: restore-after-12-to-13 -spec: - pgCluster: major-upgrade - repoName: repo1 + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-12-to-13 + labels: + postgres-operator.crunchydata.com/pgbackrest-backup: manual + postgres-operator.crunchydata.com/pgbackrest-repo: repo2 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true status: - state: Succeeded + succeeded: 1 --- apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster +kind: PerconaPGBackup metadata: - name: major-upgrade + name: backup-after-12-to-13 +spec: + pgCluster: major-upgrade + repoName: repo2 + options: + - --type=full status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready + state: Succeeded diff --git a/e2e-tests/tests/major-upgrade/11-run-backup.yaml b/e2e-tests/tests/major-upgrade/12-run-backup.yaml similarity index 89% rename from e2e-tests/tests/major-upgrade/11-run-backup.yaml rename to e2e-tests/tests/major-upgrade/12-run-backup.yaml index 0cfac94a95..601008d4ca 100644 --- a/e2e-tests/tests/major-upgrade/11-run-backup.yaml +++ b/e2e-tests/tests/major-upgrade/12-run-backup.yaml @@ -4,6 +4,6 @@ metadata: name: backup-after-12-to-13 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo2 options: - --type=full diff --git a/e2e-tests/tests/major-upgrade/13-assert.yaml b/e2e-tests/tests/major-upgrade/13-assert.yaml index 7c65443fbe..2d05bb7d88 100644 --- a/e2e-tests/tests/major-upgrade/13-assert.yaml +++ b/e2e-tests/tests/major-upgrade/13-assert.yaml @@ -1,10 +1,52 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 30 +timeout: 720 +commands: +- script: |- + set -o errexit + + kubectl -n ${NAMESPACE} get pod + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "PostgreSQL logs from ${pod}:" + echo "find /pgdata/pg13/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done + + sleep 30 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 --- -kind: ConfigMap -apiVersion: v1 +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore metadata: - name: 05-read-from-primary -data: - data: ' 100500' + name: restore-after-12-to-13 +spec: + pgCluster: major-upgrade + repoName: repo2 +status: + state: Succeeded +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: major-upgrade +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/major-upgrade/12-run-restore.yaml b/e2e-tests/tests/major-upgrade/13-run-restore.yaml similarity index 87% rename from e2e-tests/tests/major-upgrade/12-run-restore.yaml rename to e2e-tests/tests/major-upgrade/13-run-restore.yaml index 6350f5d6f1..bba96fe985 100644 --- a/e2e-tests/tests/major-upgrade/12-run-restore.yaml +++ b/e2e-tests/tests/major-upgrade/13-run-restore.yaml @@ -4,4 +4,4 @@ metadata: name: restore-after-12-to-13 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo2 diff --git a/e2e-tests/tests/major-upgrade/14-assert.yaml b/e2e-tests/tests/major-upgrade/14-assert.yaml new file mode 100644 index 0000000000..7c65443fbe --- /dev/null +++ b/e2e-tests/tests/major-upgrade/14-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 05-read-from-primary +data: + data: ' 100500' diff --git a/e2e-tests/tests/major-upgrade/13-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/14-read-from-primary.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/13-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/14-read-from-primary.yaml diff --git a/e2e-tests/tests/major-upgrade/20-assert.yaml b/e2e-tests/tests/major-upgrade/20-assert.yaml index d60a84442f..c506a745f5 100644 --- a/e2e-tests/tests/major-upgrade/20-assert.yaml +++ b/e2e-tests/tests/major-upgrade/20-assert.yaml @@ -1,6 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 360 +commands: +- script: |- + kubectl -n ${NAMESPACE} get pg,pod,job + sleep 5 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -20,3 +24,19 @@ status: ready: 3 size: 3 state: ready +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: major-upgrade + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo2 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 diff --git a/e2e-tests/tests/major-upgrade/21-assert.yaml b/e2e-tests/tests/major-upgrade/21-assert.yaml index 788d3de044..301240b0f5 100644 --- a/e2e-tests/tests/major-upgrade/21-assert.yaml +++ b/e2e-tests/tests/major-upgrade/21-assert.yaml @@ -1,31 +1,21 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 660 +commands: +- script: |- + kubectl get postgrescluster major-upgrade \ + -n ${NAMESPACE} \ + -o yaml \ + | yq eval '.status.pgbackrest.repos' - --- -kind: Job -apiVersion: batch/v1 +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster metadata: - annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-13-to-14 - labels: - postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true + name: major-upgrade status: - succeeded: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - name: backup-after-13-to-14 -spec: - pgCluster: major-upgrade - repoName: repo1 - options: - - --type=full -status: - state: Succeeded + pgbackrest: + repos: + - bound: true + name: repo3 + replicaCreateBackupComplete: true + stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/21-change-repo.yaml b/e2e-tests/tests/major-upgrade/21-change-repo.yaml new file mode 100644 index 0000000000..57f9edc8a4 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/21-change-repo.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + kubectl patch \ + -n $NAMESPACE \ + perconapgcluster major-upgrade \ + --type='json' \ + -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo3"}]' diff --git a/e2e-tests/tests/major-upgrade/22-assert.yaml b/e2e-tests/tests/major-upgrade/22-assert.yaml index 2b8c164f6e..7a2f6c1c0f 100644 --- a/e2e-tests/tests/major-upgrade/22-assert.yaml +++ b/e2e-tests/tests/major-upgrade/22-assert.yaml @@ -1,30 +1,31 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 +timeout: 660 --- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGRestore +kind: Job +apiVersion: batch/v1 metadata: - name: restore-after-13-to-14 -spec: - pgCluster: major-upgrade - repoName: repo1 + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-13-to-14 + labels: + postgres-operator.crunchydata.com/pgbackrest-backup: manual + postgres-operator.crunchydata.com/pgbackrest-repo: repo3 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true status: - state: Succeeded + succeeded: 1 --- apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster +kind: PerconaPGBackup metadata: - name: major-upgrade + name: backup-after-13-to-14 +spec: + pgCluster: major-upgrade + repoName: repo3 + options: + - --type=full status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready + state: Succeeded diff --git a/e2e-tests/tests/major-upgrade/21-run-backup.yaml b/e2e-tests/tests/major-upgrade/22-run-backup.yaml similarity index 89% rename from e2e-tests/tests/major-upgrade/21-run-backup.yaml rename to e2e-tests/tests/major-upgrade/22-run-backup.yaml index 7f60f47386..a7a1050c53 100644 --- a/e2e-tests/tests/major-upgrade/21-run-backup.yaml +++ b/e2e-tests/tests/major-upgrade/22-run-backup.yaml @@ -4,6 +4,6 @@ metadata: name: backup-after-13-to-14 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo3 options: - --type=full diff --git a/e2e-tests/tests/major-upgrade/23-assert.yaml b/e2e-tests/tests/major-upgrade/23-assert.yaml index c86dbb55fc..75b542e689 100644 --- a/e2e-tests/tests/major-upgrade/23-assert.yaml +++ b/e2e-tests/tests/major-upgrade/23-assert.yaml @@ -1,10 +1,52 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 30 +timeout: 720 +commands: +- script: |- + set -o errexit + + kubectl -n ${NAMESPACE} get pod + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "PostgreSQL logs from ${pod}:" + echo "find /pgdata/pg14/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done + + sleep 30 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 --- -kind: ConfigMap -apiVersion: v1 +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore metadata: - name: 07-read-from-primary -data: - data: ' 100500' + name: restore-after-13-to-14 +spec: + pgCluster: major-upgrade + repoName: repo3 +status: + state: Succeeded +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: major-upgrade +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/major-upgrade/22-run-restore.yaml b/e2e-tests/tests/major-upgrade/23-run-restore.yaml similarity index 87% rename from e2e-tests/tests/major-upgrade/22-run-restore.yaml rename to e2e-tests/tests/major-upgrade/23-run-restore.yaml index 7ce7fed535..079ebf7b77 100644 --- a/e2e-tests/tests/major-upgrade/22-run-restore.yaml +++ b/e2e-tests/tests/major-upgrade/23-run-restore.yaml @@ -4,4 +4,4 @@ metadata: name: restore-after-13-to-14 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo3 diff --git a/e2e-tests/tests/major-upgrade/24-assert.yaml b/e2e-tests/tests/major-upgrade/24-assert.yaml new file mode 100644 index 0000000000..c86dbb55fc --- /dev/null +++ b/e2e-tests/tests/major-upgrade/24-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 07-read-from-primary +data: + data: ' 100500' diff --git a/e2e-tests/tests/major-upgrade/23-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/24-read-from-primary.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/23-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/24-read-from-primary.yaml diff --git a/e2e-tests/tests/major-upgrade/30-assert.yaml b/e2e-tests/tests/major-upgrade/30-assert.yaml index 1d058344ee..b871b4faf8 100644 --- a/e2e-tests/tests/major-upgrade/30-assert.yaml +++ b/e2e-tests/tests/major-upgrade/30-assert.yaml @@ -1,6 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 360 +commands: +- script: |- + kubectl -n ${NAMESPACE} get pg,pod,job + sleep 5 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -20,3 +24,19 @@ status: ready: 3 size: 3 state: ready +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: major-upgrade + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo3 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 diff --git a/e2e-tests/tests/major-upgrade/31-assert.yaml b/e2e-tests/tests/major-upgrade/31-assert.yaml index 14ca1327c2..6af985e4b7 100644 --- a/e2e-tests/tests/major-upgrade/31-assert.yaml +++ b/e2e-tests/tests/major-upgrade/31-assert.yaml @@ -1,31 +1,21 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 660 +commands: +- script: |- + kubectl get postgrescluster major-upgrade \ + -n ${NAMESPACE} \ + -o yaml \ + | yq eval '.status.pgbackrest.repos' - --- -kind: Job -apiVersion: batch/v1 +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster metadata: - annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-14-to-15 - labels: - postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true + name: major-upgrade status: - succeeded: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - name: backup-after-14-to-15 -spec: - pgCluster: major-upgrade - repoName: repo1 - options: - - --type=full -status: - state: Succeeded + pgbackrest: + repos: + - bound: true + name: repo4 + replicaCreateBackupComplete: true + stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/31-change-repo.yaml b/e2e-tests/tests/major-upgrade/31-change-repo.yaml new file mode 100644 index 0000000000..33711377c4 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/31-change-repo.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + kubectl patch \ + -n $NAMESPACE \ + perconapgcluster major-upgrade \ + --type='json' \ + -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo4"}]' diff --git a/e2e-tests/tests/major-upgrade/32-assert.yaml b/e2e-tests/tests/major-upgrade/32-assert.yaml index 0246ec0e79..75daf8364c 100644 --- a/e2e-tests/tests/major-upgrade/32-assert.yaml +++ b/e2e-tests/tests/major-upgrade/32-assert.yaml @@ -1,30 +1,31 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 +timeout: 660 --- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGRestore +kind: Job +apiVersion: batch/v1 metadata: - name: restore-after-14-to-15 -spec: - pgCluster: major-upgrade - repoName: repo1 + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-14-to-15 + labels: + postgres-operator.crunchydata.com/pgbackrest-backup: manual + postgres-operator.crunchydata.com/pgbackrest-repo: repo4 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true status: - state: Succeeded + succeeded: 1 --- apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster +kind: PerconaPGBackup metadata: - name: major-upgrade + name: backup-after-14-to-15 +spec: + pgCluster: major-upgrade + repoName: repo4 + options: + - --type=full status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready + state: Succeeded diff --git a/e2e-tests/tests/major-upgrade/31-run-backup.yaml b/e2e-tests/tests/major-upgrade/32-run-backup.yaml similarity index 89% rename from e2e-tests/tests/major-upgrade/31-run-backup.yaml rename to e2e-tests/tests/major-upgrade/32-run-backup.yaml index e787fedfca..792209c20c 100644 --- a/e2e-tests/tests/major-upgrade/31-run-backup.yaml +++ b/e2e-tests/tests/major-upgrade/32-run-backup.yaml @@ -4,6 +4,6 @@ metadata: name: backup-after-14-to-15 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo4 options: - --type=full diff --git a/e2e-tests/tests/major-upgrade/33-assert.yaml b/e2e-tests/tests/major-upgrade/33-assert.yaml index 7321c61abe..1d3a8a9aae 100644 --- a/e2e-tests/tests/major-upgrade/33-assert.yaml +++ b/e2e-tests/tests/major-upgrade/33-assert.yaml @@ -1,10 +1,52 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 30 +timeout: 720 +commands: +- script: |- + set -o errexit + + kubectl -n ${NAMESPACE} get pod + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "PostgreSQL logs from ${pod}:" + echo "find /pgdata/pg15/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done + + sleep 30 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 --- -kind: ConfigMap -apiVersion: v1 +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore metadata: - name: 09-read-from-primary -data: - data: ' 100500' + name: restore-after-14-to-15 +spec: + pgCluster: major-upgrade + repoName: repo4 +status: + state: Succeeded +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: major-upgrade +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/major-upgrade/32-run-restore.yaml b/e2e-tests/tests/major-upgrade/33-run-restore.yaml similarity index 87% rename from e2e-tests/tests/major-upgrade/32-run-restore.yaml rename to e2e-tests/tests/major-upgrade/33-run-restore.yaml index 24845cbb29..8e7c7c6dab 100644 --- a/e2e-tests/tests/major-upgrade/32-run-restore.yaml +++ b/e2e-tests/tests/major-upgrade/33-run-restore.yaml @@ -4,5 +4,5 @@ metadata: name: restore-after-14-to-15 spec: pgCluster: major-upgrade - repoName: repo1 + repoName: repo4 diff --git a/e2e-tests/tests/major-upgrade/34-assert.yaml b/e2e-tests/tests/major-upgrade/34-assert.yaml new file mode 100644 index 0000000000..7321c61abe --- /dev/null +++ b/e2e-tests/tests/major-upgrade/34-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 09-read-from-primary +data: + data: ' 100500' diff --git a/e2e-tests/tests/major-upgrade/33-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/34-read-from-primary.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/33-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/34-read-from-primary.yaml diff --git a/e2e-tests/tests/major-upgrade/40-assert.yaml b/e2e-tests/tests/major-upgrade/40-assert.yaml index 58f19f4e69..de3a338c83 100644 --- a/e2e-tests/tests/major-upgrade/40-assert.yaml +++ b/e2e-tests/tests/major-upgrade/40-assert.yaml @@ -1,6 +1,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 360 +commands: +- script: |- + kubectl -n ${NAMESPACE} get pg,pod,job + sleep 5 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -20,3 +24,19 @@ status: ready: 3 size: 3 state: ready +--- +kind: Job +apiVersion: batch/v1 +metadata: + labels: + postgres-operator.crunchydata.com/cluster: major-upgrade + postgres-operator.crunchydata.com/pgbackrest: '' + postgres-operator.crunchydata.com/pgbackrest-backup: replica-create + postgres-operator.crunchydata.com/pgbackrest-repo: repo4 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 diff --git a/e2e-tests/tests/major-upgrade/41-assert.yaml b/e2e-tests/tests/major-upgrade/41-assert.yaml index 2b003b69a9..1b95780c8f 100644 --- a/e2e-tests/tests/major-upgrade/41-assert.yaml +++ b/e2e-tests/tests/major-upgrade/41-assert.yaml @@ -1,31 +1,21 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 660 +commands: +- script: |- + kubectl get postgrescluster major-upgrade \ + -n ${NAMESPACE} \ + -o yaml \ + | yq eval '.status.pgbackrest.repos' - --- -kind: Job -apiVersion: batch/v1 +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster metadata: - annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-15-to-16 - labels: - postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true + name: major-upgrade status: - succeeded: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - name: backup-after-15-to-16 -spec: - pgCluster: major-upgrade - repoName: repo1 - options: - - --type=full -status: - state: Succeeded + pgbackrest: + repos: + - bound: true + name: repo1 + replicaCreateBackupComplete: true + stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/41-change-repo.yaml b/e2e-tests/tests/major-upgrade/41-change-repo.yaml new file mode 100644 index 0000000000..d3d277599f --- /dev/null +++ b/e2e-tests/tests/major-upgrade/41-change-repo.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + kubectl patch \ + -n $NAMESPACE \ + perconapgcluster major-upgrade \ + --type='json' \ + -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo1"}]' diff --git a/e2e-tests/tests/major-upgrade/42-assert.yaml b/e2e-tests/tests/major-upgrade/42-assert.yaml index ff07785690..2b003b69a9 100644 --- a/e2e-tests/tests/major-upgrade/42-assert.yaml +++ b/e2e-tests/tests/major-upgrade/42-assert.yaml @@ -1,30 +1,31 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 720 +timeout: 660 +--- +kind: Job +apiVersion: batch/v1 +metadata: + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-15-to-16 + labels: + postgres-operator.crunchydata.com/pgbackrest-backup: manual + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 + ownerReferences: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + controller: true + blockOwnerDeletion: true +status: + succeeded: 1 --- apiVersion: pgv2.percona.com/v2 -kind: PerconaPGRestore +kind: PerconaPGBackup metadata: - name: restore-after-15-to-16 + name: backup-after-15-to-16 spec: pgCluster: major-upgrade repoName: repo1 + options: + - --type=full status: state: Succeeded ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: major-upgrade -status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready diff --git a/e2e-tests/tests/major-upgrade/41-run-backup.yaml b/e2e-tests/tests/major-upgrade/42-run-backup.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/41-run-backup.yaml rename to e2e-tests/tests/major-upgrade/42-run-backup.yaml diff --git a/e2e-tests/tests/major-upgrade/43-assert.yaml b/e2e-tests/tests/major-upgrade/43-assert.yaml index 95d57c293f..3114c23118 100644 --- a/e2e-tests/tests/major-upgrade/43-assert.yaml +++ b/e2e-tests/tests/major-upgrade/43-assert.yaml @@ -1,10 +1,52 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 30 +timeout: 720 +commands: +- script: |- + set -o errexit + + kubectl -n ${NAMESPACE} get pod + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "PostgreSQL logs from ${pod}:" + echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done + + sleep 30 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 --- -kind: ConfigMap -apiVersion: v1 +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore metadata: - name: 11-read-from-primary -data: - data: ' 100500' + name: restore-after-15-to-16 +spec: + pgCluster: major-upgrade + repoName: repo1 +status: + state: Succeeded +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: major-upgrade +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/major-upgrade/42-run-restore.yaml b/e2e-tests/tests/major-upgrade/43-run-restore.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/42-run-restore.yaml rename to e2e-tests/tests/major-upgrade/43-run-restore.yaml diff --git a/e2e-tests/tests/major-upgrade/44-assert.yaml b/e2e-tests/tests/major-upgrade/44-assert.yaml new file mode 100644 index 0000000000..95d57c293f --- /dev/null +++ b/e2e-tests/tests/major-upgrade/44-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 11-read-from-primary +data: + data: ' 100500' diff --git a/e2e-tests/tests/major-upgrade/43-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/44-read-from-primary.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/43-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/44-read-from-primary.yaml