From 4f9d9a25e50f47e7aa36d98d0fbd6d05e1587d5f Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Wed, 19 Nov 2025 16:20:57 -0500
Subject: [PATCH 01/22] Fix naming for podlogs

---
 .../e2e/pgbackrest-restore/templates/create-cluster.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
index 89a72282ef..a50ef877e8 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
@@ -71,5 +71,5 @@ spec:
 
   catch:
     - podLogs:
-        selector: postgres-operator.crunchydata.com/cluster in (original)
+        selector: (concat('postgres-operator.crunchydata.com/cluster=', 'original'))
         tail: 50

From 7327b589a957173162fa9855b52a59eaa8f4fada Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Wed, 19 Nov 2025 16:47:23 -0500
Subject: [PATCH 02/22] more checks and output

---
 .../templates/create-cluster.yaml             | 22 ++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
index a50ef877e8..8e938c78fc 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
@@ -34,6 +34,22 @@ spec:
                   - name: repo1
                     volume:
                       volumeClaimSpec: ($volume)
+    - 
+      description: >
+        Wait for the cluster to come online
+      assert:
+        resource:
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            labels:
+              postgres-operator.crunchydata.com/cluster: original
+              postgres-operator.crunchydata.com/data: postgres
+          status:
+            phase: Running
+            (containerStatuses[?name == 'database']):
+            - name: database
+              ready: true
 
     -
       description: >
@@ -71,5 +87,9 @@ spec:
 
   catch:
     - podLogs:
-        selector: (concat('postgres-operator.crunchydata.com/cluster=', 'original'))
+        selector: postgres-operator.crunchydata.com/cluster in (original)
         tail: 50
+    - describe:
+        apiVersion: v1
+        kind: Pod
+        selector: postgres-operator.crunchydata.com/cluster in (original)

From 0c34eb169fb5415cff8a6cc84e67cbbf9e5ef809 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Wed, 19 Nov 2025 17:28:13 -0500
Subject: [PATCH 03/22] remove unnecessary images

---
 .github/workflows/test.yaml | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 26106c928e..53b6cb4a9a 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -103,14 +103,9 @@ jobs:
             registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi9-0.17.1-2542
             registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542
             registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-16.10-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-16.10-3.3-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-16.10-3.4-2542
             registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi9-18.0-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.4-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.5-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.6-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-18.0-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-18.0-3.6-2542
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi9-9.8-2542
+            registry.developers.crunchydata.com/crunchydata/postgres-operator:ubi9-5.8.4-0
 
       - name: Get pgMonitor files.
         run: make get-pgmonitor
@@ -134,14 +129,7 @@ jobs:
             --env 'RELATED_IMAGE_PGEXPORTER=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi9-0.17.1-2542' \
             --env 'RELATED_IMAGE_PGUPGRADE=registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi9-18.0-2542' \
             --env 'RELATED_IMAGE_POSTGRES_16=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-16.10-2542' \
-            --env 'RELATED_IMAGE_POSTGRES_16_GIS_3.3=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-16.10-3.3-2542' \
-            --env 'RELATED_IMAGE_POSTGRES_16_GIS_3.4=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-16.10-3.4-2542' \
             --env 'RELATED_IMAGE_POSTGRES_17=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542' \
-            --env 'RELATED_IMAGE_POSTGRES_17_GIS_3.4=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.4-2542' \
-            --env 'RELATED_IMAGE_POSTGRES_17_GIS_3.5=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.5-2542' \
-            --env 'RELATED_IMAGE_POSTGRES_17_GIS_3.6=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.6-2542' \
-            --env 'RELATED_IMAGE_POSTGRES_18=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-18.0-2542' \
-            --env 'RELATED_IMAGE_POSTGRES_18_GIS_3.6=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-18.0-3.6-2542' \
             --env 'RELATED_IMAGE_STANDALONE_PGADMIN=registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi9-9.8-2542' \
             --env 'RELATED_IMAGE_COLLECTOR=registry.developers.crunchydata.com/crunchydata/postgres-operator:ubi9-5.8.4-0' \
             --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \

From e0f9a1cf225c5d629364117ccafc608e52d1ca17 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Thu, 20 Nov 2025 18:01:14 -0500
Subject: [PATCH 04/22] use variable in template

---
 .../e2e/pgbackrest-restore/templates/clone-cluster.yaml   | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
index 193e7495c4..0c2c2bc3d6 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
@@ -53,14 +53,10 @@ spec:
   catch:
     - description: Read all log lines from job pods
       podLogs:
-        selector: >
-          batch.kubernetes.io/job-name,
-          postgres-operator.crunchydata.com/cluster in (clone-one)
+        selector: (join('', ['batch.kubernetes.io/job-name,', 'postgres-operator.crunchydata.com/cluster=', $name]))
         tail: -1
 
     - description: Read all log lines from postgres pods
       podLogs:
-        selector: >
-          postgres-operator.crunchydata.com/instance,
-          postgres-operator.crunchydata.com/cluster in (clone-one)
+        selector: (join('', ['postgres-operator.crunchydata.com/instance,', 'postgres-operator.crunchydata.com/cluster=', $name]))
         tail: -1

From f9a22530bf5dadcc355a83c9979e385a6da88daa Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Thu, 20 Nov 2025 18:32:47 -0500
Subject: [PATCH 05/22] add more logging

---
 .../templates/clone-cluster.yaml              | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
index 0c2c2bc3d6..c67ce1be87 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
@@ -51,6 +51,28 @@ spec:
                 updatedReplicas: 1
 
   catch:
+    - description: Describe the PostgresCluster to see its status
+      describe:
+        apiVersion: ($postgrescluster.apiVersion)
+        kind: PostgresCluster
+        name: ($name)
+
+    - description: Get all pods in the namespace to see what's running
+      describe:
+        apiVersion: v1
+        kind: Pod
+        selector: (join('', ['postgres-operator.crunchydata.com/cluster=', $name]))
+
+    - description: Get all jobs to check restore status
+      describe:
+        apiVersion: batch/v1
+        kind: Job
+        selector: (join('', ['postgres-operator.crunchydata.com/cluster=', $name]))
+
+    - description: Get events related to the cluster
+      script:
+        content: kubectl get events --field-selector (join('', ['involvedObject.name=', $name])) -o wide
+
     - description: Read all log lines from job pods
       podLogs:
         selector: (join('', ['batch.kubernetes.io/job-name,', 'postgres-operator.crunchydata.com/cluster=', $name]))
@@ -60,3 +82,9 @@ spec:
       podLogs:
         selector: (join('', ['postgres-operator.crunchydata.com/instance,', 'postgres-operator.crunchydata.com/cluster=', $name]))
         tail: -1
+
+    - description: Check PVCs for the cluster
+      describe:
+        apiVersion: v1
+        kind: PersistentVolumeClaim
+        selector: (join('', ['postgres-operator.crunchydata.com/cluster=', $name]))

From eed9964a2e76a65451efccb0369ff18a15995e2d Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Thu, 20 Nov 2025 19:25:18 -0500
Subject: [PATCH 06/22] Increase timeout to 5m for backup/restore operations

---
 .../chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml | 1 +
 .../chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml | 1 +
 .../e2e/pgbackrest-restore/templates/create-cluster.yaml         | 1 +
 .../e2e/pgbackrest-restore/templates/point-in-time-restore.yaml  | 1 +
 4 files changed, 4 insertions(+)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
index c67ce1be87..1e031c7d05 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
@@ -38,6 +38,7 @@ spec:
       description: >
         Wait for the cluster to come online
       assert:
+        timeout: 5m
         resource:
           apiVersion: ($postgrescluster.apiVersion)
           kind: PostgresCluster
diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml
index 6279f29248..8152700225 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml
@@ -23,6 +23,7 @@ spec:
       description: >
         Wait for the backup to complete
       assert:
+        timeout: 5m
         resource:
           apiVersion: batch/v1
           kind: Job
diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
index 8e938c78fc..c1e745c7da 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
@@ -55,6 +55,7 @@ spec:
       description: >
         Wait for the replica backup to complete
       assert:
+        timeout: 5m
         resource:
           apiVersion: ($postgrescluster.apiVersion)
           kind: PostgresCluster
diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/point-in-time-restore.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/point-in-time-restore.yaml
index b832cec5ae..f88a6e8eb8 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/point-in-time-restore.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/point-in-time-restore.yaml
@@ -45,6 +45,7 @@ spec:
       description: >
         Wait for the restore to complete and the cluster to come online
       assert:
+        timeout: 5m
         resource:
           apiVersion: ($postgrescluster.apiVersion)
           kind: PostgresCluster

From a531ce8387595605d170a5b53da87aa499782938 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Thu, 20 Nov 2025 20:21:25 -0500
Subject: [PATCH 07/22] Increate timeout for image prefetch

---
 .github/actions/k3d/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/k3d/action.yaml b/.github/actions/k3d/action.yaml
index 395d5f1116..b6e6ed5c2b 100644
--- a/.github/actions/k3d/action.yaml
+++ b/.github/actions/k3d/action.yaml
@@ -16,7 +16,7 @@ inputs:
     description: >
       Each line is the name of an image to fetch onto all Kubernetes nodes
   prefetch-timeout:
-    default: 90s
+    default: 3m
     required: true
     description: >
       Amount of time to wait for images to be fetched

From 910f94e1ff268e5208762f6fa8d66d807fea7d75 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Thu, 20 Nov 2025 22:17:39 -0500
Subject: [PATCH 08/22] Wait for postgres to be ready

---
 .../templates/change-parameters.yaml              | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/change-parameters.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/change-parameters.yaml
index 04b5a85e86..a741893a8c 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/change-parameters.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/change-parameters.yaml
@@ -26,6 +26,21 @@ spec:
             postgres-operator.crunchydata.com/cluster=original,
             postgres-operator.crunchydata.com/role=master
 
+    -
+      description: >
+        Wait for PostgreSQL to be ready
+      script:
+        skipCommandOutput: true
+        timeout: 2m
+        env:
+          - name: PRIMARY
+            value: ($primary)
+        content: |
+          until kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
+            -- psql -qAt --command 'SELECT 1' 2>/dev/null; do
+            sleep 1
+          done
+
     -
       description: >
         Read the timestamp at which PostgreSQL last started

From e7906eef5aa81146df28cb48035e7dabc22c25f8 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Thu, 20 Nov 2025 22:26:31 -0500
Subject: [PATCH 09/22] Add Postgres readiness checks before psql execs

Prevents race conditions where pod is found but Postgres isn't ready yet.
---
 .../e2e/pgbackrest-restore/templates/create-cluster.yaml    | 6 ++++++
 .../e2e/pgbackrest-restore/templates/lose-data.yaml         | 6 ++++++
 .../e2e/pgbackrest-restore/templates/verify-backup.yaml     | 6 ++++++
 3 files changed, 18 insertions(+)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
index c1e745c7da..2b570d9789 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml
@@ -81,6 +81,12 @@ spec:
                 postgres-operator.crunchydata.com/role=master'
           )
 
+          # Wait for PostgreSQL to be ready
+          until kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
+            -- psql -qAt --command 'SELECT 1' 2>/dev/null; do
+            sleep 1
+          done
+
           kubectl exec --stdin --namespace "${NAMESPACE}" "${PRIMARY}" -- psql -q --file=- <<'SQL'
             CREATE TABLESPACE barn LOCATION '/tablespaces/barn/data';
             GRANT ALL ON TABLESPACE barn TO public;
diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/lose-data.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/lose-data.yaml
index 39838099ff..570eaac04f 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/lose-data.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/lose-data.yaml
@@ -31,6 +31,12 @@ spec:
           - name: PRIMARY
             value: ($primary)
         content: |
+          # Wait for PostgreSQL to be ready
+          until kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
+            -- psql -qAt --command 'SELECT 1' 2>/dev/null; do
+            sleep 1
+          done
+
           OBJECTIVE=$(
             kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
               -- psql -qAt --command 'SELECT clock_timestamp()'
diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/verify-backup.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/verify-backup.yaml
index 1328f669c5..a81e7a908b 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/verify-backup.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/verify-backup.yaml
@@ -17,6 +17,12 @@ spec:
                 postgres-operator.crunchydata.com/role=master'
           )
 
+          # Wait for PostgreSQL to be ready
+          until kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
+            -- psql -qAt --command 'SELECT 1' 2>/dev/null; do
+            sleep 1
+          done
+
           kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
             -- psql --command 'SELECT pg_switch_wal()' --pset footer=off
 

From 0645472ec1b22fcef05a3fd195bb6e56271c910a Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Thu, 20 Nov 2025 23:06:29 -0500
Subject: [PATCH 10/22] Add storage provisioning diagnostics for CI failures

---
 .github/actions/k3d/monitor-resources.sh      | 43 +++++++++++++++++
 .github/workflows/test.yaml                   | 36 +++++++++++++++
 .../templates/clone-cluster.yaml              | 46 +++++++++++++++++++
 3 files changed, 125 insertions(+)
 create mode 100755 .github/actions/k3d/monitor-resources.sh

diff --git a/.github/actions/k3d/monitor-resources.sh b/.github/actions/k3d/monitor-resources.sh
new file mode 100755
index 0000000000..c70979ac9b
--- /dev/null
+++ b/.github/actions/k3d/monitor-resources.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+# Monitor k3d cluster resources during tests
+# Run this in the background to capture resource usage over time
+
+set -euo pipefail
+
+INTERVAL="${1:-30}"  # Check every N seconds
+DURATION="${2:-300}" # Run for N seconds (5 minutes)
+OUTPUT="${3:-resource-monitor.log}"
+
+echo "Starting resource monitoring (interval=${INTERVAL}s, duration=${DURATION}s)" > "$OUTPUT"
+echo "================================================================" >> "$OUTPUT"
+
+END_TIME=$(($(date +%s) + DURATION))
+
+while [ $(date +%s) -lt $END_TIME ]; do
+    TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
+    
+    echo "" >> "$OUTPUT"
+    echo "=== $TIMESTAMP ===" >> "$OUTPUT"
+    
+    # Docker disk usage
+    echo "--- Docker Disk Usage ---" >> "$OUTPUT"
+    docker system df >> "$OUTPUT" 2>&1 || true
+    
+    # k3d node disk usage
+    echo "--- k3d Node Disk Usage ---" >> "$OUTPUT"
+    docker exec k3d-k3s-default-server-0 df -h 2>> "$OUTPUT" | \
+        grep -E '(Filesystem|/$|/var/lib|overlay)' >> "$OUTPUT" || true
+    
+    # PVC counts
+    echo "--- PVC Status ---" >> "$OUTPUT"
+    kubectl get pvc --all-namespaces -o json 2>/dev/null | \
+        jq -r '.items | group_by(.status.phase) | map({phase: .[0].status.phase, count: length}) | .[]' >> "$OUTPUT" 2>&1 || true
+    
+    # Local path provisioner status
+    echo "--- Provisioner Pods ---" >> "$OUTPUT"
+    kubectl get pods -n kube-system -l app=local-path-provisioner -o wide >> "$OUTPUT" 2>&1 || true
+    
+    sleep "$INTERVAL"
+done
+
+echo "Resource monitoring complete" >> "$OUTPUT"
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 53b6cb4a9a..cf58e9cb9d 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -135,12 +135,48 @@ jobs:
             --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \
             --name 'postgres-operator' localhost/postgres-operator
 
+      - name: Check initial environment health
+        run: |
+          echo "::group::Docker system info"
+          docker system df
+          docker info | grep -E 'Storage Driver|Data Space'
+          echo "::endgroup::"
+          echo "::group::k3d node disk space"
+          docker exec k3d-k3s-default-server-0 df -h
+          echo "::endgroup::"
+          echo "::group::Storage provisioner status"
+          kubectl get storageclass -o wide
+          kubectl get pods -n kube-system -l app=local-path-provisioner -o wide
+          echo "::endgroup::"
+
+      - name: Start resource monitoring
+        run: |
+          nohup .github/actions/k3d/monitor-resources.sh 30 600 chainsaw-resource-monitor.log &
+          echo $! > monitor.pid
+
       - run: |
           make check-chainsaw && exit
           failed=$?
           echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
           exit $failed
 
+      - name: Stop resource monitoring
+        if: always()
+        run: |
+          if [ -f monitor.pid ]; then
+            kill $(cat monitor.pid) 2>/dev/null || true
+            rm monitor.pid
+          fi
+
+      - name: Upload resource monitoring logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: "chainsaw-resource-monitor-k8s=${{ matrix.kubernetes }}"
+          path: chainsaw-resource-monitor.log
+          retention-days: 7
+          if-no-files-found: ignore
+
       - run: make generate-kuttl
         env:
           KUTTL_PG_UPGRADE_FROM_VERSION: '16'
diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
index 1e031c7d05..c8f8875b59 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
@@ -34,6 +34,19 @@ spec:
                     volume:
                       volumeClaimSpec: ($volume)
 
+    - description: Wait for PVCs to be created
+      sleep:
+        duration: 5s
+
+    - description: Check PVC provisioning status early
+      script:
+        content: |
+          echo "=== PVCs for ($name) ==="
+          kubectl get pvc -l postgres-operator.crunchydata.com/cluster=($name) -o wide || true
+          echo ""
+          echo "=== All PVC Events ==="
+          kubectl get events --field-selector involvedObject.kind=PersistentVolumeClaim -o wide | tail -20 || true
+
     -
       description: >
         Wait for the cluster to come online
@@ -89,3 +102,36 @@ spec:
         apiVersion: v1
         kind: PersistentVolumeClaim
         selector: (join('', ['postgres-operator.crunchydata.com/cluster=', $name]))
+
+    - description: Get all PVC events to diagnose provisioning failures
+      script:
+        content: kubectl get events --all-namespaces --field-selector involvedObject.kind=PersistentVolumeClaim -o wide
+
+    - description: Check storage provisioner status
+      script:
+        content: |
+          echo "=== Storage Classes ==="
+          kubectl get storageclass -o wide
+          echo ""
+          echo "=== Local Path Provisioner Pods ==="
+          kubectl get pods -n kube-system -l app=local-path-provisioner -o wide
+          echo ""
+          echo "=== Local Path Provisioner Logs ==="
+          kubectl logs -n kube-system -l app=local-path-provisioner --tail=100 --prefix=true
+
+    - description: Check node resources and disk space
+      script:
+        content: |
+          echo "=== Node Resources ==="
+          kubectl describe nodes
+          echo ""
+          echo "=== Node Disk Usage (via docker) ==="
+          docker exec k3d-k3s-default-server-0 df -h
+
+    - description: Check all PVs and their status
+      script:
+        content: kubectl get pv -o wide
+
+    - description: Check for stuck PVCs across all namespaces
+      script:
+        content: kubectl get pvc --all-namespaces -o wide

From 872e8ad27890427dde8dc81bbb1ff0bb09b291f5 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Thu, 20 Nov 2025 23:47:31 -0500
Subject: [PATCH 11/22] test: reduce disk usage in pgbackrest-restore chainsaw
 test

Reduce PVC sizes from 1Gi to 256Mi and use Foreground deletion
with explicit PVC cleanup waits to prevent disk exhaustion on
GitHub-hosted runners.
---
 .../e2e/pgbackrest-restore/chainsaw-test.yaml | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
index a79dc6a193..b3334b0348 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
@@ -17,7 +17,7 @@ spec:
         connect: { name: PGCONNECT_TIMEOUT, value: '5' }
 
     - name: volume
-      value: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } }
+      value: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 256Mi } } }
 
     - name: postgrescluster
       value:
@@ -83,15 +83,19 @@ spec:
 
     - name: 'Delete Cluster #1'
       description: >
-        Delete this clone in the background to free up resources
+        Delete this clone and wait for PVCs to be cleaned up to free disk space
       try:
         - delete:
-            deletionPropagationPolicy: Background
-            expect: [{ check: { (`true`): true } }]
+            deletionPropagationPolicy: Foreground
             ref:
               apiVersion: ($postgrescluster.apiVersion)
               kind: PostgresCluster
               name: clone-one
+        - script:
+            content: |
+              echo "Waiting for clone-one PVCs to be deleted..."
+              kubectl wait --for=delete pvc -l postgres-operator.crunchydata.com/cluster=clone-one --timeout=2m || true
+              sleep 5
 
     - name: 'Update Cluster'
       description: >
@@ -160,15 +164,19 @@ spec:
 
     - name: 'Delete Cluster #2'
       description: >
-        Delete this clone in the background to free up resources
+        Delete this clone and wait for PVCs to be cleaned up to free disk space
       try:
         - delete:
-            deletionPropagationPolicy: Background
-            expect: [{ check: { (`true`): true } }]
+            deletionPropagationPolicy: Foreground
             ref:
               apiVersion: ($postgrescluster.apiVersion)
               kind: PostgresCluster
               name: clone-two
+        - script:
+            content: |
+              echo "Waiting for clone-two PVCs to be deleted..."
+              kubectl wait --for=delete pvc -l postgres-operator.crunchydata.com/cluster=clone-two --timeout=2m || true
+              sleep 5
 
     - name: 'Lose Data'
       description: >

From 136aa044314bcf475e21b1409c9e5ec2a6ed4093 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Fri, 21 Nov 2025 09:52:53 -0500
Subject: [PATCH 12/22] test: fix shell syntax error in clone-cluster template

Use env var instead of JMESPath expression in shell script.
---
 .../e2e/pgbackrest-restore/templates/clone-cluster.yaml    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
index c8f8875b59..a45758ef22 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
@@ -40,9 +40,12 @@ spec:
 
     - description: Check PVC provisioning status early
       script:
+        env:
+        - name: CLUSTER_NAME
+          value: ($name)
         content: |
-          echo "=== PVCs for ($name) ==="
-          kubectl get pvc -l postgres-operator.crunchydata.com/cluster=($name) -o wide || true
+          echo "=== PVCs for ${CLUSTER_NAME} ==="
+          kubectl get pvc -l postgres-operator.crunchydata.com/cluster=${CLUSTER_NAME} -o wide || true
           echo ""
           echo "=== All PVC Events ==="
           kubectl get events --field-selector involvedObject.kind=PersistentVolumeClaim -o wide | tail -20 || true

From c6fee50a31473ba8b3b303f6c34ba5421b7aa0e3 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Fri, 21 Nov 2025 10:39:11 -0500
Subject: [PATCH 13/22] test: add timeout to PVC deletion wait scripts

Prevent script timeout (5s default) from killing 2m kubectl wait.
---
 testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
index b3334b0348..68b13dce53 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
@@ -87,11 +87,13 @@ spec:
       try:
         - delete:
             deletionPropagationPolicy: Foreground
+            timeout: 3m
             ref:
               apiVersion: ($postgrescluster.apiVersion)
               kind: PostgresCluster
               name: clone-one
         - script:
+            timeout: 3m
             content: |
               echo "Waiting for clone-one PVCs to be deleted..."
               kubectl wait --for=delete pvc -l postgres-operator.crunchydata.com/cluster=clone-one --timeout=2m || true
@@ -168,11 +170,13 @@ spec:
       try:
         - delete:
             deletionPropagationPolicy: Foreground
+            timeout: 3m
             ref:
               apiVersion: ($postgrescluster.apiVersion)
               kind: PostgresCluster
               name: clone-two
         - script:
+            timeout: 3m
             content: |
               echo "Waiting for clone-two PVCs to be deleted..."
               kubectl wait --for=delete pvc -l postgres-operator.crunchydata.com/cluster=clone-two --timeout=2m || true

From 0566e113e476af49496aeb30c6a4260ca513a1aa Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Fri, 21 Nov 2025 15:41:05 -0500
Subject: [PATCH 14/22] test: remove unnecessary image prefetch for chainsaw
 tests

Only prefetch images actually used by the test: pgbackrest and postgres.
Removes ~500MB of unused images (pgbouncer, pgadmin, exporter, upgrade).
---
 .github/workflows/test.yaml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index cf58e9cb9d..cd3b37ac23 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -99,13 +99,8 @@ jobs:
           k3s-channel: "${{ matrix.kubernetes }}"
           prefetch-images: |
             registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi9-1.24-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi9-0.17.1-2542
             registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542
             registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-16.10-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi9-18.0-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi9-9.8-2542
-            registry.developers.crunchydata.com/crunchydata/postgres-operator:ubi9-5.8.4-0
 
       - name: Get pgMonitor files.
         run: make get-pgmonitor

From c257229ea032107c473606944238681563e012db Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Fri, 21 Nov 2025 16:29:30 -0500
Subject: [PATCH 15/22] Split e2e-k3d into separate chainsaw and kuttl jobs

---
 .github/workflows/test.yaml | 88 ++++++++++++++++++++++---------------
 1 file changed, 53 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index cd3b37ac23..375c7c60da 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -81,7 +81,7 @@ jobs:
           path: envtest-existing.coverage.gz
           retention-days: 1
 
-  e2e-k3d:
+  e2e-k3d-chainsaw:
     runs-on: ubuntu-24.04
     needs: [go-test]
     strategy:
@@ -130,47 +130,63 @@ jobs:
             --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \
             --name 'postgres-operator' localhost/postgres-operator
 
-      - name: Check initial environment health
-        run: |
-          echo "::group::Docker system info"
-          docker system df
-          docker info | grep -E 'Storage Driver|Data Space'
-          echo "::endgroup::"
-          echo "::group::k3d node disk space"
-          docker exec k3d-k3s-default-server-0 df -h
-          echo "::endgroup::"
-          echo "::group::Storage provisioner status"
-          kubectl get storageclass -o wide
-          kubectl get pods -n kube-system -l app=local-path-provisioner -o wide
-          echo "::endgroup::"
-
-      - name: Start resource monitoring
-        run: |
-          nohup .github/actions/k3d/monitor-resources.sh 30 600 chainsaw-resource-monitor.log &
-          echo $! > monitor.pid
-
       - run: |
           make check-chainsaw && exit
           failed=$?
           echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
           exit $failed
 
-      - name: Stop resource monitoring
-        if: always()
-        run: |
-          if [ -f monitor.pid ]; then
-            kill $(cat monitor.pid) 2>/dev/null || true
-            rm monitor.pid
-          fi
-
-      - name: Upload resource monitoring logs
-        if: always()
-        uses: actions/upload-artifact@v4
+      - name: Stop PGO
+        run: docker stop 'postgres-operator' || true
+
+  e2e-k3d-kuttl:
+    runs-on: ubuntu-24.04
+    needs: [go-test]
+    strategy:
+      fail-fast: false
+      matrix:
+        kubernetes: [v1.30, v1.33]
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-go@v6
+        with: { go-version: stable }
+
+      - name: Start k3s
+        uses: ./.github/actions/k3d
         with:
-          name: "chainsaw-resource-monitor-k8s=${{ matrix.kubernetes }}"
-          path: chainsaw-resource-monitor.log
-          retention-days: 7
-          if-no-files-found: ignore
+          k3s-channel: "${{ matrix.kubernetes }}"
+          prefetch-images: |
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2542
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-16.10-2542
+
+      - name: Get pgMonitor files.
+        run: make get-pgmonitor
+        env:
+          PGMONITOR_DIR: "${{ github.workspace }}/hack/tools/pgmonitor"
+          QUERIES_CONFIG_DIR: "${{ github.workspace }}/hack/tools/queries"
+
+      # Start a Docker container with the working directory mounted.
+      - run: make build BUILDAH=docker
+      - name: Start PGO
+        run: |
+          kubectl apply --server-side -k ./config/namespace
+          kubectl apply --server-side -k ./config/dev
+          hack/create-kubeconfig.sh postgres-operator pgo
+          docker run --detach --network host --read-only \
+            --volume "$(pwd):/mnt" --workdir '/mnt' \
+            --env 'QUERIES_CONFIG_DIR=/mnt/hack/tools/queries' \
+            --env 'KUBECONFIG=hack/.kube/postgres-operator/pgo' \
+            --env 'RELATED_IMAGE_PGBACKREST=registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2542' \
+            --env 'RELATED_IMAGE_PGBOUNCER=registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi9-1.24-2542' \
+            --env 'RELATED_IMAGE_PGEXPORTER=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi9-0.17.1-2542' \
+            --env 'RELATED_IMAGE_PGUPGRADE=registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi9-18.0-2542' \
+            --env 'RELATED_IMAGE_POSTGRES_16=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-16.10-2542' \
+            --env 'RELATED_IMAGE_POSTGRES_17=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542' \
+            --env 'RELATED_IMAGE_STANDALONE_PGADMIN=registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi9-9.8-2542' \
+            --env 'RELATED_IMAGE_COLLECTOR=registry.developers.crunchydata.com/crunchydata/postgres-operator:ubi9-5.8.4-0' \
+            --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \
+            --name 'postgres-operator' localhost/postgres-operator
 
       - run: make generate-kuttl
         env:
@@ -194,6 +210,8 @@ jobs:
     needs:
       - kubernetes-api
       - kubernetes-k3d
+      - e2e-k3d-chainsaw
+      - e2e-k3d-kuttl
     steps:
       - uses: actions/checkout@v5
       - uses: actions/setup-go@v6

From 569abc58ee88597e295168834da0c8683ead9029 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Fri, 21 Nov 2025 16:51:35 -0500
Subject: [PATCH 16/22] Fixup: remove extra debug

---
 .../e2e/pgbackrest-restore/chainsaw-test.yaml    | 12 ------------
 .../templates/clone-cluster.yaml                 | 16 ----------------
 2 files changed, 28 deletions(-)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
index 68b13dce53..9a5cf08693 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
@@ -92,12 +92,6 @@ spec:
               apiVersion: ($postgrescluster.apiVersion)
               kind: PostgresCluster
               name: clone-one
-        - script:
-            timeout: 3m
-            content: |
-              echo "Waiting for clone-one PVCs to be deleted..."
-              kubectl wait --for=delete pvc -l postgres-operator.crunchydata.com/cluster=clone-one --timeout=2m || true
-              sleep 5
 
     - name: 'Update Cluster'
       description: >
@@ -175,12 +169,6 @@ spec:
               apiVersion: ($postgrescluster.apiVersion)
               kind: PostgresCluster
               name: clone-two
-        - script:
-            timeout: 3m
-            content: |
-              echo "Waiting for clone-two PVCs to be deleted..."
-              kubectl wait --for=delete pvc -l postgres-operator.crunchydata.com/cluster=clone-two --timeout=2m || true
-              sleep 5
 
     - name: 'Lose Data'
       description: >
diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
index a45758ef22..4f510984fc 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
@@ -34,22 +34,6 @@ spec:
                     volume:
                       volumeClaimSpec: ($volume)
 
-    - description: Wait for PVCs to be created
-      sleep:
-        duration: 5s
-
-    - description: Check PVC provisioning status early
-      script:
-        env:
-        - name: CLUSTER_NAME
-          value: ($name)
-        content: |
-          echo "=== PVCs for ${CLUSTER_NAME} ==="
-          kubectl get pvc -l postgres-operator.crunchydata.com/cluster=${CLUSTER_NAME} -o wide || true
-          echo ""
-          echo "=== All PVC Events ==="
-          kubectl get events --field-selector involvedObject.kind=PersistentVolumeClaim -o wide | tail -20 || true
-
     -
       description: >
         Wait for the cluster to come online

From aa19cad7215458d75b8409be2392a91e0ab514d0 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Fri, 21 Nov 2025 17:53:00 -0500
Subject: [PATCH 17/22] test: add missing image prefetch and increase KUTTL
 timeout

- Add pgbouncer, exporter, upgrade, pgadmin images to prefetch
- Increase KUTTL timeout from 300s to 450s
- Increase prefetch timeout to 5m
---
 .github/workflows/test.yaml   | 5 +++++
 testing/kuttl/kuttl-test.yaml | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 375c7c60da..7786ff3e9f 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -155,10 +155,15 @@ jobs:
         uses: ./.github/actions/k3d
         with:
           k3s-channel: "${{ matrix.kubernetes }}"
+          prefetch-timeout: 5m
           prefetch-images: |
             registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2542
             registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542
             registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-16.10-2542
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi9-1.24-2542
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi9-0.17.1-2542
+            registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi9-18.0-2542
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi9-9.8-2542
 
       - name: Get pgMonitor files.
         run: make get-pgmonitor
diff --git a/testing/kuttl/kuttl-test.yaml b/testing/kuttl/kuttl-test.yaml
index 6733707507..446e6c7b7a 100644
--- a/testing/kuttl/kuttl-test.yaml
+++ b/testing/kuttl/kuttl-test.yaml
@@ -2,7 +2,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
 testDirs:
 - testing/kuttl/e2e-generated/
-timeout: 300
+timeout: 450
 parallel: 2
 # by default kuttl will run in a generated namespace to override
 # that functionality simply uncomment the line below and replace

From 6cfdeeb6e47e4b2e45369cce3ff14006654c5d75 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Mon, 24 Nov 2025 12:22:10 -0500
Subject: [PATCH 18/22] Remove some debug logging

---
 .github/actions/k3d/monitor-resources.sh      | 43 -------------------
 .../templates/clone-cluster.yaml              | 33 --------------
 2 files changed, 76 deletions(-)
 delete mode 100755 .github/actions/k3d/monitor-resources.sh

diff --git a/.github/actions/k3d/monitor-resources.sh b/.github/actions/k3d/monitor-resources.sh
deleted file mode 100755
index c70979ac9b..0000000000
--- a/.github/actions/k3d/monitor-resources.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/bash
-# Monitor k3d cluster resources during tests
-# Run this in the background to capture resource usage over time
-
-set -euo pipefail
-
-INTERVAL="${1:-30}"  # Check every N seconds
-DURATION="${2:-300}" # Run for N seconds (5 minutes)
-OUTPUT="${3:-resource-monitor.log}"
-
-echo "Starting resource monitoring (interval=${INTERVAL}s, duration=${DURATION}s)" > "$OUTPUT"
-echo "================================================================" >> "$OUTPUT"
-
-END_TIME=$(($(date +%s) + DURATION))
-
-while [ $(date +%s) -lt $END_TIME ]; do
-    TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
-    
-    echo "" >> "$OUTPUT"
-    echo "=== $TIMESTAMP ===" >> "$OUTPUT"
-    
-    # Docker disk usage
-    echo "--- Docker Disk Usage ---" >> "$OUTPUT"
-    docker system df >> "$OUTPUT" 2>&1 || true
-    
-    # k3d node disk usage
-    echo "--- k3d Node Disk Usage ---" >> "$OUTPUT"
-    docker exec k3d-k3s-default-server-0 df -h 2>> "$OUTPUT" | \
-        grep -E '(Filesystem|/$|/var/lib|overlay)' >> "$OUTPUT" || true
-    
-    # PVC counts
-    echo "--- PVC Status ---" >> "$OUTPUT"
-    kubectl get pvc --all-namespaces -o json 2>/dev/null | \
-        jq -r '.items | group_by(.status.phase) | map({phase: .[0].status.phase, count: length}) | .[]' >> "$OUTPUT" 2>&1 || true
-    
-    # Local path provisioner status
-    echo "--- Provisioner Pods ---" >> "$OUTPUT"
-    kubectl get pods -n kube-system -l app=local-path-provisioner -o wide >> "$OUTPUT" 2>&1 || true
-    
-    sleep "$INTERVAL"
-done
-
-echo "Resource monitoring complete" >> "$OUTPUT"
diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
index 4f510984fc..1e031c7d05 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml
@@ -89,36 +89,3 @@ spec:
         apiVersion: v1
         kind: PersistentVolumeClaim
         selector: (join('', ['postgres-operator.crunchydata.com/cluster=', $name]))
-
-    - description: Get all PVC events to diagnose provisioning failures
-      script:
-        content: kubectl get events --all-namespaces --field-selector involvedObject.kind=PersistentVolumeClaim -o wide
-
-    - description: Check storage provisioner status
-      script:
-        content: |
-          echo "=== Storage Classes ==="
-          kubectl get storageclass -o wide
-          echo ""
-          echo "=== Local Path Provisioner Pods ==="
-          kubectl get pods -n kube-system -l app=local-path-provisioner -o wide
-          echo ""
-          echo "=== Local Path Provisioner Logs ==="
-          kubectl logs -n kube-system -l app=local-path-provisioner --tail=100 --prefix=true
-
-    - description: Check node resources and disk space
-      script:
-        content: |
-          echo "=== Node Resources ==="
-          kubectl describe nodes
-          echo ""
-          echo "=== Node Disk Usage (via docker) ==="
-          docker exec k3d-k3s-default-server-0 df -h
-
-    - description: Check all PVs and their status
-      script:
-        content: kubectl get pv -o wide
-
-    - description: Check for stuck PVCs across all namespaces
-      script:
-        content: kubectl get pvc --all-namespaces -o wide

From e8b0ac484ea94eb44050aa79adc99ab030d0691a Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Mon, 24 Nov 2025 12:22:27 -0500
Subject: [PATCH 19/22] Revert timeout bump

---
 testing/kuttl/kuttl-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/kuttl/kuttl-test.yaml b/testing/kuttl/kuttl-test.yaml
index 446e6c7b7a..6733707507 100644
--- a/testing/kuttl/kuttl-test.yaml
+++ b/testing/kuttl/kuttl-test.yaml
@@ -2,7 +2,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
 testDirs:
 - testing/kuttl/e2e-generated/
-timeout: 450
+timeout: 300
 parallel: 2
 # by default kuttl will run in a generated namespace to override
 # that functionality simply uncomment the line below and replace

From a2d12f5f3efb64fa9f3ac15f722c1e39c3585991 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Mon, 24 Nov 2025 12:23:13 -0500
Subject: [PATCH 20/22] Revert foreground deletion of pvc

---
 .../chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml    | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
index 9a5cf08693..02e776c8c4 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
@@ -83,11 +83,9 @@ spec:
 
     - name: 'Delete Cluster #1'
       description: >
-        Delete this clone and wait for PVCs to be cleaned up to free disk space
+        Delete this clone
       try:
         - delete:
-            deletionPropagationPolicy: Foreground
-            timeout: 3m
             ref:
               apiVersion: ($postgrescluster.apiVersion)
               kind: PostgresCluster
@@ -160,11 +158,9 @@ spec:
 
     - name: 'Delete Cluster #2'
       description: >
-        Delete this clone and wait for PVCs to be cleaned up to free disk space
+        Delete this clone
       try:
         - delete:
-            deletionPropagationPolicy: Foreground
-            timeout: 3m
             ref:
               apiVersion: ($postgrescluster.apiVersion)
               kind: PostgresCluster

From bd01f635cf0c69dec4212b25d1d0ea9695733657 Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Mon, 24 Nov 2025 12:30:46 -0500
Subject: [PATCH 21/22] Restore background deletion policy for clone clusters

---
 .../chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
index 02e776c8c4..5bc2e993f8 100644
--- a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
+++ b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml
@@ -83,9 +83,11 @@ spec:
 
     - name: 'Delete Cluster #1'
       description: >
-        Delete this clone
+        Delete this clone in the background to free up resources
       try:
         - delete:
+            deletionPropagationPolicy: Background
+            expect: [{ check: { (`true`): true } }]
             ref:
               apiVersion: ($postgrescluster.apiVersion)
               kind: PostgresCluster
@@ -158,9 +160,11 @@ spec:
 
     - name: 'Delete Cluster #2'
       description: >
-        Delete this clone
+        Delete this clone in the background to free up resources
       try:
         - delete:
+            deletionPropagationPolicy: Background
+            expect: [{ check: { (`true`): true } }]
             ref:
               apiVersion: ($postgrescluster.apiVersion)
               kind: PostgresCluster

From 51f6ea05bdf7026278ce870ad93fe57948f7602a Mon Sep 17 00:00:00 2001
From: jmckulk <joseph.mckulka@snowflake.com>
Date: Tue, 25 Nov 2025 10:32:20 -0500
Subject: [PATCH 22/22] Remove more prefetch images from chainsaw action

---
 .github/workflows/test.yaml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 7786ff3e9f..ad781ba97f 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -100,7 +100,6 @@ jobs:
           prefetch-images: |
             registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2542
             registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542
-            registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-16.10-2542
 
       - name: Get pgMonitor files.
         run: make get-pgmonitor
@@ -120,13 +119,7 @@ jobs:
             --env 'QUERIES_CONFIG_DIR=/mnt/hack/tools/queries' \
             --env 'KUBECONFIG=hack/.kube/postgres-operator/pgo' \
             --env 'RELATED_IMAGE_PGBACKREST=registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2542' \
-            --env 'RELATED_IMAGE_PGBOUNCER=registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi9-1.24-2542' \
-            --env 'RELATED_IMAGE_PGEXPORTER=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi9-0.17.1-2542' \
-            --env 'RELATED_IMAGE_PGUPGRADE=registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi9-18.0-2542' \
-            --env 'RELATED_IMAGE_POSTGRES_16=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-16.10-2542' \
             --env 'RELATED_IMAGE_POSTGRES_17=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542' \
-            --env 'RELATED_IMAGE_STANDALONE_PGADMIN=registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi9-9.8-2542' \
-            --env 'RELATED_IMAGE_COLLECTOR=registry.developers.crunchydata.com/crunchydata/postgres-operator:ubi9-5.8.4-0' \
             --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \
             --name 'postgres-operator' localhost/postgres-operator