diff --git a/.github/actions/k3d/action.yaml b/.github/actions/k3d/action.yaml index 395d5f1116..b6e6ed5c2b 100644 --- a/.github/actions/k3d/action.yaml +++ b/.github/actions/k3d/action.yaml @@ -16,7 +16,7 @@ inputs: description: > Each line is the name of an image to fetch onto all Kubernetes nodes prefetch-timeout: - default: 90s + default: 3m required: true description: > Amount of time to wait for images to be fetched diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 26106c928e..ad781ba97f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -81,7 +81,7 @@ jobs: path: envtest-existing.coverage.gz retention-days: 1 - e2e-k3d: + e2e-k3d-chainsaw: runs-on: ubuntu-24.04 needs: [go-test] strategy: @@ -99,18 +99,64 @@ jobs: k3s-channel: "${{ matrix.kubernetes }}" prefetch-images: | registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2542 - registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi9-1.24-2542 - registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi9-0.17.1-2542 + registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542 + + - name: Get pgMonitor files. + run: make get-pgmonitor + env: + PGMONITOR_DIR: "${{ github.workspace }}/hack/tools/pgmonitor" + QUERIES_CONFIG_DIR: "${{ github.workspace }}/hack/tools/queries" + + # Start a Docker container with the working directory mounted. + - run: make build BUILDAH=docker + - name: Start PGO + run: | + kubectl apply --server-side -k ./config/namespace + kubectl apply --server-side -k ./config/dev + hack/create-kubeconfig.sh postgres-operator pgo + docker run --detach --network host --read-only \ + --volume "$(pwd):/mnt" --workdir '/mnt' \ + --env 'QUERIES_CONFIG_DIR=/mnt/hack/tools/queries' \ + --env 'KUBECONFIG=hack/.kube/postgres-operator/pgo' \ + --env 'RELATED_IMAGE_PGBACKREST=registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2542' \ + --env 'RELATED_IMAGE_POSTGRES_17=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542' \ + --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \ + --name 'postgres-operator' localhost/postgres-operator + + - run: | + make check-chainsaw && exit + failed=$? + echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::' + exit $failed + + - name: Stop PGO + run: docker stop 'postgres-operator' || true + + e2e-k3d-kuttl: + runs-on: ubuntu-24.04 + needs: [go-test] + strategy: + fail-fast: false + matrix: + kubernetes: [v1.30, v1.33] + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: { go-version: stable } + + - name: Start k3s + uses: ./.github/actions/k3d + with: + k3s-channel: "${{ matrix.kubernetes }}" + prefetch-timeout: 5m + prefetch-images: | + registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2542 registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542 registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-16.10-2542 - registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-16.10-3.3-2542 - registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-16.10-3.4-2542 + registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi9-1.24-2542 + registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi9-0.17.1-2542 registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi9-18.0-2542 - registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.4-2542 - registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.5-2542 - registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.6-2542 - registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-18.0-2542 - registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-18.0-3.6-2542 + registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi9-9.8-2542 - name: Get pgMonitor files. run: make get-pgmonitor @@ -134,25 +180,12 @@ jobs: --env 'RELATED_IMAGE_PGEXPORTER=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi9-0.17.1-2542' \ --env 'RELATED_IMAGE_PGUPGRADE=registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi9-18.0-2542' \ --env 'RELATED_IMAGE_POSTGRES_16=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-16.10-2542' \ - --env 'RELATED_IMAGE_POSTGRES_16_GIS_3.3=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-16.10-3.3-2542' \ - --env 'RELATED_IMAGE_POSTGRES_16_GIS_3.4=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-16.10-3.4-2542' \ --env 'RELATED_IMAGE_POSTGRES_17=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.6-2542' \ - --env 'RELATED_IMAGE_POSTGRES_17_GIS_3.4=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.4-2542' \ - --env 'RELATED_IMAGE_POSTGRES_17_GIS_3.5=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.5-2542' \ - --env 'RELATED_IMAGE_POSTGRES_17_GIS_3.6=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-17.6-3.6-2542' \ - --env 'RELATED_IMAGE_POSTGRES_18=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-18.0-2542' \ - --env 'RELATED_IMAGE_POSTGRES_18_GIS_3.6=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi9-18.0-3.6-2542' \ --env 'RELATED_IMAGE_STANDALONE_PGADMIN=registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi9-9.8-2542' \ --env 'RELATED_IMAGE_COLLECTOR=registry.developers.crunchydata.com/crunchydata/postgres-operator:ubi9-5.8.4-0' \ --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \ --name 'postgres-operator' localhost/postgres-operator - - run: | - make check-chainsaw && exit - failed=$? - echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::' - exit $failed - - run: make generate-kuttl env: KUTTL_PG_UPGRADE_FROM_VERSION: '16' @@ -175,6 +208,8 @@ jobs: needs: - kubernetes-api - kubernetes-k3d + - e2e-k3d-chainsaw + - e2e-k3d-kuttl steps: - uses: actions/checkout@v5 - uses: actions/setup-go@v6 diff --git a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml index a79dc6a193..5bc2e993f8 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml @@ -17,7 +17,7 @@ spec: connect: { name: PGCONNECT_TIMEOUT, value: '5' } - name: volume - value: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } } + value: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 256Mi } } } - name: postgrescluster value: diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/change-parameters.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/change-parameters.yaml index 04b5a85e86..a741893a8c 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/templates/change-parameters.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/change-parameters.yaml @@ -26,6 +26,21 @@ spec: postgres-operator.crunchydata.com/cluster=original, postgres-operator.crunchydata.com/role=master + - + description: > + Wait for PostgreSQL to be ready + script: + skipCommandOutput: true + timeout: 2m + env: + - name: PRIMARY + value: ($primary) + content: | + until kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT 1' 2>/dev/null; do + sleep 1 + done + - description: > Read the timestamp at which PostgreSQL last started diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml index 193e7495c4..1e031c7d05 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml @@ -38,6 +38,7 @@ spec: description: > Wait for the cluster to come online assert: + timeout: 5m resource: apiVersion: ($postgrescluster.apiVersion) kind: PostgresCluster @@ -51,16 +52,40 @@ spec: updatedReplicas: 1 catch: + - description: Describe the PostgresCluster to see its status + describe: + apiVersion: ($postgrescluster.apiVersion) + kind: PostgresCluster + name: ($name) + + - description: Get all pods in the namespace to see what's running + describe: + apiVersion: v1 + kind: Pod + selector: (join('', ['postgres-operator.crunchydata.com/cluster=', $name])) + + - description: Get all jobs to check restore status + describe: + apiVersion: batch/v1 + kind: Job + selector: (join('', ['postgres-operator.crunchydata.com/cluster=', $name])) + + - description: Get events related to the cluster + script: + content: kubectl get events --field-selector (join('', ['involvedObject.name=', $name])) -o wide + - description: Read all log lines from job pods podLogs: - selector: > - batch.kubernetes.io/job-name, - postgres-operator.crunchydata.com/cluster in (clone-one) + selector: (join('', ['batch.kubernetes.io/job-name,', 'postgres-operator.crunchydata.com/cluster=', $name])) tail: -1 - description: Read all log lines from postgres pods podLogs: - selector: > - postgres-operator.crunchydata.com/instance, - postgres-operator.crunchydata.com/cluster in (clone-one) + selector: (join('', ['postgres-operator.crunchydata.com/instance,', 'postgres-operator.crunchydata.com/cluster=', $name])) tail: -1 + + - description: Check PVCs for the cluster + describe: + apiVersion: v1 + kind: PersistentVolumeClaim + selector: (join('', ['postgres-operator.crunchydata.com/cluster=', $name])) diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml index 6279f29248..8152700225 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml @@ -23,6 +23,7 @@ spec: description: > Wait for the backup to complete assert: + timeout: 5m resource: apiVersion: batch/v1 kind: Job diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml index 89a72282ef..2b570d9789 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/create-cluster.yaml @@ -34,11 +34,28 @@ spec: - name: repo1 volume: volumeClaimSpec: ($volume) + - + description: > + Wait for the cluster to come online + assert: + resource: + apiVersion: v1 + kind: Pod + metadata: + labels: + postgres-operator.crunchydata.com/cluster: original + postgres-operator.crunchydata.com/data: postgres + status: + phase: Running + (containerStatuses[?name == 'database']): + - name: database + ready: true - description: > Wait for the replica backup to complete assert: + timeout: 5m resource: apiVersion: ($postgrescluster.apiVersion) kind: PostgresCluster @@ -64,6 +81,12 @@ spec: postgres-operator.crunchydata.com/role=master' ) + # Wait for PostgreSQL to be ready + until kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT 1' 2>/dev/null; do + sleep 1 + done + kubectl exec --stdin --namespace "${NAMESPACE}" "${PRIMARY}" -- psql -q --file=- <<'SQL' CREATE TABLESPACE barn LOCATION '/tablespaces/barn/data'; GRANT ALL ON TABLESPACE barn TO public; @@ -73,3 +96,7 @@ spec: - podLogs: selector: postgres-operator.crunchydata.com/cluster in (original) tail: 50 + - describe: + apiVersion: v1 + kind: Pod + selector: postgres-operator.crunchydata.com/cluster in (original) diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/lose-data.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/lose-data.yaml index 39838099ff..570eaac04f 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/templates/lose-data.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/lose-data.yaml @@ -31,6 +31,12 @@ spec: - name: PRIMARY value: ($primary) content: | + # Wait for PostgreSQL to be ready + until kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT 1' 2>/dev/null; do + sleep 1 + done + OBJECTIVE=$( kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ -- psql -qAt --command 'SELECT clock_timestamp()' diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/point-in-time-restore.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/point-in-time-restore.yaml index b832cec5ae..f88a6e8eb8 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/templates/point-in-time-restore.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/point-in-time-restore.yaml @@ -45,6 +45,7 @@ spec: description: > Wait for the restore to complete and the cluster to come online assert: + timeout: 5m resource: apiVersion: ($postgrescluster.apiVersion) kind: PostgresCluster diff --git a/testing/chainsaw/e2e/pgbackrest-restore/templates/verify-backup.yaml b/testing/chainsaw/e2e/pgbackrest-restore/templates/verify-backup.yaml index 1328f669c5..a81e7a908b 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/templates/verify-backup.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/templates/verify-backup.yaml @@ -17,6 +17,12 @@ spec: postgres-operator.crunchydata.com/role=master' ) + # Wait for PostgreSQL to be ready + until kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT 1' 2>/dev/null; do + sleep 1 + done + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ -- psql --command 'SELECT pg_switch_wal()' --pset footer=off