From e09505ff12170dac59e82f8ca7a0510f6dfe3833 Mon Sep 17 00:00:00 2001
From: Tom Ashley <tom.ashley@gmail.com>
Date: Thu, 19 Jun 2025 10:44:27 +0100
Subject: [PATCH 01/17] feat: Install gandalf, salt-wrapper for infra

---
 .../files/gandalf_config/gandalf.sudoers.conf |  2 +
 .../files/gandalf_config/gandalf_salt.service | 19 +++++
 .../files/gandalf_config/gandalf_salt.timer   | 13 +++
 ansible/manifest-playbook.yml                 | 18 +++++
 ansible/tasks/internal/gandalf.yml            | 79 +++++++++++++++++++
 ansible/tasks/setup-supabase-internal.yml     |  5 ++
 ansible/vars.yml                              |  2 +
 7 files changed, 138 insertions(+)
 create mode 100644 ansible/files/gandalf_config/gandalf.sudoers.conf
 create mode 100644 ansible/files/gandalf_config/gandalf_salt.service
 create mode 100644 ansible/files/gandalf_config/gandalf_salt.timer
 create mode 100644 ansible/tasks/internal/gandalf.yml

diff --git a/ansible/files/gandalf_config/gandalf.sudoers.conf b/ansible/files/gandalf_config/gandalf.sudoers.conf
new file mode 100644
index 000000000..bf2517490
--- /dev/null
+++ b/ansible/files/gandalf_config/gandalf.sudoers.conf
@@ -0,0 +1,2 @@
+%gandalf ALL= NOPASSWD: /usr/bin/salt-call
+%gandalf ALL= NOPASSWD: /usr/bin/gpg --homedir /etc/salt/gpgkeys --import, /usr/bin/gpg --homedir /etc/salt/gpgkeys --list-secret-keys *
diff --git a/ansible/files/gandalf_config/gandalf_salt.service b/ansible/files/gandalf_config/gandalf_salt.service
new file mode 100644
index 000000000..c70db7ed8
--- /dev/null
+++ b/ansible/files/gandalf_config/gandalf_salt.service
@@ -0,0 +1,19 @@
+[Unit]
+Description=Configuration management via gandalf salt
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/gandalf/gandalf --config /opt/gandalf/config.yaml salt --apply --store-result
+User=gandalf
+Group=gandalf
+StandardOutput=journal
+StandardError=journal
+StateDirectory=gandalf
+CacheDirectory=gandalf
+
+# Security hardening
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/files/gandalf_config/gandalf_salt.timer b/ansible/files/gandalf_config/gandalf_salt.timer
new file mode 100644
index 000000000..6862ca402
--- /dev/null
+++ b/ansible/files/gandalf_config/gandalf_salt.timer
@@ -0,0 +1,13 @@
+[Unit]
+Description=Run Supabase gandalf salt on a schedule
+Requires=gandalf_salt.service
+
+[Timer]
+OnCalendar=*:0/10
+# Random delay up to 30 seconds splay
+RandomizedDelaySec=30
+AccuracySec=1s
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/ansible/manifest-playbook.yml b/ansible/manifest-playbook.yml
index 5c1c65053..583e3596c 100644
--- a/ansible/manifest-playbook.yml
+++ b/ansible/manifest-playbook.yml
@@ -61,6 +61,22 @@
       shell: |
         cd /tmp && tar -cJf admin-mgr-{{ adminmgr_release }}-arm64.tar.xz admin-mgr
 
+    - name: Download gandalf archive
+      get_url:
+        url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/gandalf/v{{ gandalf_release }}/gandalf_{{ gandalf_release }}_linux_arm64.tar.gz"
+        dest: "/tmp/gandalf.tar.gz"
+        timeout: 90
+
+    - name: gandalf - unpack archive in /tmp
+      unarchive:
+        remote_src: yes
+        src: /tmp/gandalf.tar.gz
+        dest: /tmp
+
+    - name: gandalf - pack archive
+      shell: |
+        cd /tmp && tar -cJf gandalf-{{ gandalf_release }}-arm64.tar.xz gandalf
+
     - name: upload archives
       shell: |
         aws s3 cp /tmp/{{ item.file }} s3://{{ internal_artifacts_bucket }}/upgrades/{{ item.service }}/{{ item.file }}
@@ -73,3 +89,5 @@
           file: supabase-admin-api-{{ adminapi_release }}-arm64.tar.xz
         - service: admin-mgr
           file: admin-mgr-{{ adminmgr_release }}-arm64.tar.xz
+        - service: gandalf
+          file: gandalf-{{ gandalf_release }}-arm64.tar.xz
diff --git a/ansible/tasks/internal/gandalf.yml b/ansible/tasks/internal/gandalf.yml
new file mode 100644
index 000000000..c6ffc9673
--- /dev/null
+++ b/ansible/tasks/internal/gandalf.yml
@@ -0,0 +1,79 @@
+- name: gandalf - system user
+  user:
+    name: gandalf
+    groups: admin,gandalf,salt
+    append: yes
+    system: yes
+    shell: /bin/sh
+
+- name: Setting arch (x86)
+  set_fact:
+    arch: "x86"
+  when: platform == "amd64"
+
+- name: Setting arch (arm)
+  set_fact:
+    arch: "arm64"
+  when: platform == "arm64"
+
+- name: Download gandalf archive
+  get_url:
+    url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/gandalf/v{{ gandalf_release }}/gandalf_{{ gandalf_release }}_linux_{{ arch }}.tar.gz"
+    dest: "/tmp/gandalf.tar.gz"
+    timeout: 90
+
+- name: gandalf - unpack archive in /opt
+  unarchive:
+    remote_src: yes
+    src: /tmp/gandalf.tar.gz
+    dest: /opt
+    owner: gandalf
+
+- name: gandalf - create symlink
+  ansible.builtin.file:
+    path: /opt/gandalf/gandalf
+    src: "/opt/gandalf/gandalf-linux-{{ arch }}"
+    state: link
+    owner: gandalf
+    mode: '0755'
+    force: yes
+
+- name: gandalf - config dir
+  file:
+    path: /opt/gandalf
+    owner: gandalf
+    state: directory
+
+- name: gandalf - gpg dir
+  file:
+    path: /etc/salt/gpgkeys
+    owner: root
+    group: salt
+    state: directory
+
+- name: give gandalf user permissions
+  copy:
+    src: files/gandalf.sudoers.conf
+    dest: /etc/sudoers.d/gandalf
+    mode: "0644"
+
+- name: gandalf - create salt systemd timer file 
+  copy:
+     src: files/gandalf_config/gandalf_salt.timer
+     dest: /etc/systemd/system/gandalf_salt.timer
+
+- name: gandalf - create salt service file
+  copy:
+    src: files/gandalf_config/gandalf_salt.service
+    dest: /etc/systemd/system/gandalf_salt.service
+
+- name: gandalf - reload systemd
+  systemd:
+    daemon_reload: yes
+
+# Initially ensure gandalf is installed but not started
+- name: gandalf - DISABLE service
+  systemd:
+    name: gandalf_salt
+    enabled: no
+    state: stopped
diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml
index 7aa931763..5a37dbaa0 100644
--- a/ansible/tasks/setup-supabase-internal.yml
+++ b/ansible/tasks/setup-supabase-internal.yml
@@ -115,5 +115,10 @@
   tags:
     - aws-only
 
+- name: Install gandalf
+  import_tasks: internal/gandalf.yml
+  tags:
+    - aws-only
+
 - name: Envoy - use lds.supabase.yaml for /etc/envoy/lds.yaml
   command: mv /etc/envoy/lds.supabase.yaml /etc/envoy/lds.yaml
diff --git a/ansible/vars.yml b/ansible/vars.yml
index 2838d6f92..f1e633c8a 100644
--- a/ansible/vars.yml
+++ b/ansible/vars.yml
@@ -57,3 +57,5 @@ adminmgr_release: 0.25.1
 
 vector_x86_deb: "https://packages.timber.io/vector/0.22.3/vector_0.22.3-1_amd64.deb"
 vector_arm_deb: "https://packages.timber.io/vector/0.22.3/vector_0.22.3-1_arm64.deb"
+
+gandalf_release: 1.4.18

From c7492f518ff06c22d9945eac3527b21b6b5f983e Mon Sep 17 00:00:00 2001
From: Tom Ashley <tom.ashley@gmail.com>
Date: Mon, 23 Jun 2025 11:38:16 +0100
Subject: [PATCH 02/17] chore: version bumps

---
 ansible/tasks/internal/gandalf.yml | 2 +-
 ansible/vars.yml                   | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ansible/tasks/internal/gandalf.yml b/ansible/tasks/internal/gandalf.yml
index c6ffc9673..7ef2bd0af 100644
--- a/ansible/tasks/internal/gandalf.yml
+++ b/ansible/tasks/internal/gandalf.yml
@@ -57,7 +57,7 @@
     dest: /etc/sudoers.d/gandalf
     mode: "0644"
 
-- name: gandalf - create salt systemd timer file 
+- name: gandalf - create salt systemd timer file
   copy:
      src: files/gandalf_config/gandalf_salt.timer
      dest: /etc/systemd/system/gandalf_salt.timer
diff --git a/ansible/vars.yml b/ansible/vars.yml
index f1e633c8a..4d54e10bc 100644
--- a/ansible/vars.yml
+++ b/ansible/vars.yml
@@ -10,8 +10,8 @@ postgres_major:
 # Full version strings for each major version
 postgres_release:
   postgresorioledb-17: "17.0.1.093-orioledb"
-  postgres17: "17.4.1.043"
-  postgres15: "15.8.1.100"
+  postgres17: "17.4.1.044"
+  postgres15: "15.8.1.101"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"
@@ -58,4 +58,4 @@ adminmgr_release: 0.25.1
 vector_x86_deb: "https://packages.timber.io/vector/0.22.3/vector_0.22.3-1_amd64.deb"
 vector_arm_deb: "https://packages.timber.io/vector/0.22.3/vector_0.22.3-1_arm64.deb"
 
-gandalf_release: 1.4.18
+gandalf_release: 1.4.26

From d609e224b79b8437947895d984f2d48539b4832d Mon Sep 17 00:00:00 2001
From: Tom Ashley <tom.ashley@gmail.com>
Date: Mon, 23 Jun 2025 12:20:21 +0100
Subject: [PATCH 03/17] bumping oriole build also

---
 ansible/vars.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/vars.yml b/ansible/vars.yml
index 4d54e10bc..18895df58 100644
--- a/ansible/vars.yml
+++ b/ansible/vars.yml
@@ -9,7 +9,7 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.093-orioledb"
+  postgresorioledb-17: "17.0.1.094-orioledb"
   postgres17: "17.4.1.044"
   postgres15: "15.8.1.101"
 

From 6021a156011ec65d420595e6c9f0c22bf8cadf18 Mon Sep 17 00:00:00 2001
From: Tom Ashley <tom.ashley@gmail.com>
Date: Mon, 23 Jun 2025 14:19:24 +0100
Subject: [PATCH 04/17] fix: create the Gandalf group

---
 ansible/tasks/internal/gandalf.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ansible/tasks/internal/gandalf.yml b/ansible/tasks/internal/gandalf.yml
index 7ef2bd0af..29027eb93 100644
--- a/ansible/tasks/internal/gandalf.yml
+++ b/ansible/tasks/internal/gandalf.yml
@@ -1,7 +1,8 @@
 - name: gandalf - system user
   user:
     name: gandalf
-    groups: admin,gandalf,salt
+    group: gandalf
+    groups: admin,salt
     append: yes
     system: yes
     shell: /bin/sh

From 4c177538a1c1f084ca0b3d9a39d03314774dac72 Mon Sep 17 00:00:00 2001
From: Tom Ashley <tom.ashley@gmail.com>
Date: Mon, 23 Jun 2025 14:55:45 +0100
Subject: [PATCH 05/17] fix: implicitly create the gandalf group

---
 ansible/tasks/internal/gandalf.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ansible/tasks/internal/gandalf.yml b/ansible/tasks/internal/gandalf.yml
index 29027eb93..6c8358ce1 100644
--- a/ansible/tasks/internal/gandalf.yml
+++ b/ansible/tasks/internal/gandalf.yml
@@ -1,3 +1,8 @@
+- name: gandalf - system group
+  group:
+    name: gandalf
+    system: yes
+
 - name: gandalf - system user
   user:
     name: gandalf

From 43fea2e40a0951ab8a538637dab1c3b7ae54ab93 Mon Sep 17 00:00:00 2001
From: Tom Ashley <tom.ashley@gmail.com>
Date: Mon, 23 Jun 2025 15:48:01 +0100
Subject: [PATCH 06/17] fix: correct gandalf archive path

---
 ansible/tasks/internal/gandalf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/tasks/internal/gandalf.yml b/ansible/tasks/internal/gandalf.yml
index 6c8358ce1..b0b701bcb 100644
--- a/ansible/tasks/internal/gandalf.yml
+++ b/ansible/tasks/internal/gandalf.yml
@@ -24,7 +24,7 @@
 
 - name: Download gandalf archive
   get_url:
-    url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/gandalf/v{{ gandalf_release }}/gandalf_{{ gandalf_release }}_linux_{{ arch }}.tar.gz"
+    url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/gandalf/v{{ gandalf_release }}/gandalf-{{ gandalf_release }}-linux-{{ arch }}.tar.gz"
     dest: "/tmp/gandalf.tar.gz"
     timeout: 90
 

From 43eed3d0d91b5faee850d44c42a6d77d331337e9 Mon Sep 17 00:00:00 2001
From: Tom Ashley <tom.ashley@gmail.com>
Date: Mon, 23 Jun 2025 16:16:52 +0100
Subject: [PATCH 07/17] fix: unpack gandalf build correctly

---
 ansible/tasks/internal/gandalf.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ansible/tasks/internal/gandalf.yml b/ansible/tasks/internal/gandalf.yml
index b0b701bcb..daa9cf524 100644
--- a/ansible/tasks/internal/gandalf.yml
+++ b/ansible/tasks/internal/gandalf.yml
@@ -32,8 +32,10 @@
   unarchive:
     remote_src: yes
     src: /tmp/gandalf.tar.gz
-    dest: /opt
+    dest: /opt/gandalf/
     owner: gandalf
+    extra_opts:
+      - --strip-components=1
 
 - name: gandalf - create symlink
   ansible.builtin.file:

From ed4933411ecc121eb783ae782e6be1f88517ade0 Mon Sep 17 00:00:00 2001
From: Tom Ashley <tom.ashley@gmail.com>
Date: Mon, 23 Jun 2025 16:54:03 +0100
Subject: [PATCH 08/17] fix: reorganise order for gandalf installs

---
 ansible/tasks/internal/gandalf.yml | 38 +++++++++++++++---------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/ansible/tasks/internal/gandalf.yml b/ansible/tasks/internal/gandalf.yml
index daa9cf524..2821998f3 100644
--- a/ansible/tasks/internal/gandalf.yml
+++ b/ansible/tasks/internal/gandalf.yml
@@ -12,6 +12,25 @@
     system: yes
     shell: /bin/sh
 
+- name: gandalf - config dir
+  file:
+    path: /opt/gandalf
+    owner: gandalf
+    state: directory
+
+- name: gandalf - gpg dir
+  file:
+    path: /etc/salt/gpgkeys
+    owner: root
+    group: salt
+    state: directory
+
+- name: give gandalf user permissions
+  copy:
+    src: files/gandalf.sudoers.conf
+    dest: /etc/sudoers.d/gandalf
+    mode: "0644"
+
 - name: Setting arch (x86)
   set_fact:
     arch: "x86"
@@ -46,25 +65,6 @@
     mode: '0755'
     force: yes
 
-- name: gandalf - config dir
-  file:
-    path: /opt/gandalf
-    owner: gandalf
-    state: directory
-
-- name: gandalf - gpg dir
-  file:
-    path: /etc/salt/gpgkeys
-    owner: root
-    group: salt
-    state: directory
-
-- name: give gandalf user permissions
-  copy:
-    src: files/gandalf.sudoers.conf
-    dest: /etc/sudoers.d/gandalf
-    mode: "0644"
-
 - name: gandalf - create salt systemd timer file
   copy:
      src: files/gandalf_config/gandalf_salt.timer

From 0ca5760e798ce28d8afa9b0cf4eaf728989ebdb8 Mon Sep 17 00:00:00 2001
From: Tom Ashley <tom.ashley@gmail.com>
Date: Mon, 23 Jun 2025 17:16:30 +0100
Subject: [PATCH 09/17] fix: point to the correct path

---
 ansible/tasks/internal/gandalf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/tasks/internal/gandalf.yml b/ansible/tasks/internal/gandalf.yml
index 2821998f3..47231daad 100644
--- a/ansible/tasks/internal/gandalf.yml
+++ b/ansible/tasks/internal/gandalf.yml
@@ -27,7 +27,7 @@
 
 - name: give gandalf user permissions
   copy:
-    src: files/gandalf.sudoers.conf
+    src: files/gandalf_config/gandalf.sudoers.conf
     dest: /etc/sudoers.d/gandalf
     mode: "0644"
 

From eaa2bb50d0a25b0c0ccfb6256066bfab07e836ba Mon Sep 17 00:00:00 2001
From: Crispy1975 <12525875+Crispy1975@users.noreply.github.com>
Date: Tue, 24 Jun 2025 17:42:09 +0100
Subject: [PATCH 10/17] chore: bump vars due to conflicts

---
 ansible/vars.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ansible/vars.yml b/ansible/vars.yml
index 18895df58..939c892d1 100644
--- a/ansible/vars.yml
+++ b/ansible/vars.yml
@@ -9,9 +9,9 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.094-orioledb"
-  postgres17: "17.4.1.044"
-  postgres15: "15.8.1.101"
+  postgresorioledb-17: "17.0.1.096-orioledb"
+  postgres17: "17.4.1.046"
+  postgres15: "15.8.1.103"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"
@@ -58,4 +58,4 @@ adminmgr_release: 0.25.1
 vector_x86_deb: "https://packages.timber.io/vector/0.22.3/vector_0.22.3-1_amd64.deb"
 vector_arm_deb: "https://packages.timber.io/vector/0.22.3/vector_0.22.3-1_arm64.deb"
 
-gandalf_release: 1.4.26
+gandalf_release: 1.4.30

From 4eb81c4afba98731250cb97bdfda262617c63cfb Mon Sep 17 00:00:00 2001
From: samrose <samuel@supabase.io>
Date: Mon, 23 Jun 2025 17:22:16 -0400
Subject: [PATCH 11/17] fix: pin version of libpq to 17.x (#1649)

* fix: pin version of libpq to the major version installed

* fix: the variable is postgresql_major_version

* fix: using the correct var

* fix: default to postgres 15 major version

* fix: default to 15

* fix: supposedly main contains all releases for libpq5

* fix: syntax

* fix: different syntax

* fix: rely on ppa

* fix: ppa removal

* fix: change to trigger workflow

* fix: adding a test to see postgres version

* fix: testing for failure related to libpq5 version issue

* test: include read-only test for postgrest

* test: reduce log access time

* test: make sure to fail if version wrong

* chore: bump vars
---
 ansible/tasks/setup-postgrest.yml |  29 ++--
 testinfra/test_ami_nix.py         | 230 ++++++++++++++++++++++++++++++
 2 files changed, 247 insertions(+), 12 deletions(-)

diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml
index a98d1990f..b6e0f951c 100644
--- a/ansible/tasks/setup-postgrest.yml
+++ b/ansible/tasks/setup-postgrest.yml
@@ -6,22 +6,28 @@
     url: https://www.postgresql.org/media/keys/ACCC4CF8.asc
     state: present
 
-- name: PostgREST - add Postgres PPA
+- name: PostgREST - add Postgres PPA main
   apt_repository:
-    repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg {{ postgresql_major }}"
+    repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main"
     state: present
+    filename: postgresql-pgdg
 
-- name: PostgREST - update apt cache
+- name: PostgREST - install system dependencies
   apt:
-    update_cache: yes
-
-# libpq is a C library that enables user programs to communicate with
-# the PostgreSQL database server.
-- name: PostgREST - system dependencies
-  apt:
-    pkg:
+    package:
       - libpq5
       - libnuma-dev
+    update_cache: yes
+    state: present
+
+- name: PostgREST - verify libpq5 version
+  shell: dpkg -l libpq5 | grep '^ii' | awk '{print $3}'
+  register: libpq5_version
+  changed_when: false
+
+- name: Show installed libpq5 version
+  debug:
+    msg: "Installed libpq5 version: {{ libpq5_version.stdout }}"
 
 - name: PostgREST - remove Postgres PPA gpg key
   apt_key:
@@ -30,7 +36,7 @@
 
 - name: PostgREST - remove Postgres PPA
   apt_repository:
-    repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg {{ postgresql_major }}"
+    repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main"
     state: absent
 
 - name: postgis - ensure dependencies do not get autoremoved
@@ -88,7 +94,6 @@
       #! /usr/bin/env bash
       set -euo pipefail
       set -x
-
       cd "$(dirname "$0")"
       cat $@ > merged.conf
     dest: /etc/postgrest/merge.sh
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index 1975818d6..864ab2861 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -507,3 +507,233 @@ def test_postgrest_ending_empty_key_query_parameter_is_removed(host):
         },
     )
     assert res.ok
+
+
+def test_postgresql_version(host):
+    """Print the PostgreSQL version being tested and ensure it's >= 14."""
+    result = run_ssh_command(host['ssh'], "sudo -u postgres psql -c 'SELECT version();'")
+    if result['succeeded']:
+        print(f"\nPostgreSQL Version:\n{result['stdout']}")
+        # Extract version number from the output
+        version_line = result['stdout'].strip().split('\n')[2]  # Skip header and get the actual version
+        # Extract major version number (e.g., "15.8" -> 15)
+        import re
+        version_match = re.search(r'PostgreSQL (\d+)\.', version_line)
+        if version_match:
+            major_version = int(version_match.group(1))
+            print(f"PostgreSQL major version: {major_version}")
+            assert major_version >= 14, f"PostgreSQL version {major_version} is less than 14"
+        else:
+            assert False, "Could not parse PostgreSQL version number"
+    else:
+        print(f"\nFailed to get PostgreSQL version: {result['stderr']}")
+        assert False, "Failed to get PostgreSQL version"
+    
+    # Also get the version from the command line
+    result = run_ssh_command(host['ssh'], "sudo -u postgres psql --version")
+    if result['succeeded']:
+        print(f"PostgreSQL Client Version: {result['stdout'].strip()}")
+    else:
+        print(f"Failed to get PostgreSQL client version: {result['stderr']}")
+    
+    print("✓ PostgreSQL version is >= 14")
+
+
+def test_libpq5_version(host):
+    """Print the libpq5 version installed and ensure it's >= 14."""
+    # Try different package managers to find libpq5
+    result = run_ssh_command(host['ssh'], "dpkg -l | grep libpq5 || true")
+    if result['succeeded'] and result['stdout'].strip():
+        print(f"\nlibpq5 package info:\n{result['stdout']}")
+        # Extract version from dpkg output (format: ii libpq5:arm64 17.5-1.pgdg20.04+1)
+        import re
+        version_match = re.search(r'libpq5[^ ]* +(\d+)\.', result['stdout'])
+        if version_match:
+            major_version = int(version_match.group(1))
+            print(f"libpq5 major version: {major_version}")
+            assert major_version >= 14, f"libpq5 version {major_version} is less than 14"
+        else:
+            print("Could not parse libpq5 version from dpkg output")
+    else:
+        print("\nlibpq5 not found via dpkg")
+    
+    # Also try to find libpq.so files
+    result = run_ssh_command(host['ssh'], "find /usr -name '*libpq*' -type f 2>/dev/null | head -10")
+    if result['succeeded'] and result['stdout'].strip():
+        print(f"\nlibpq files found:\n{result['stdout']}")
+    else:
+        print("\nNo libpq files found")
+    
+    # Check if we can get version from a libpq file
+    result = run_ssh_command(host['ssh'], "ldd /usr/bin/psql | grep libpq || true")
+    if result['succeeded'] and result['stdout'].strip():
+        print(f"\npsql libpq dependency:\n{result['stdout']}")
+    else:
+        print("\nCould not find libpq dependency for psql")
+    
+    # Try to get version from libpq directly
+    result = run_ssh_command(host['ssh'], "psql --version 2>&1 | head -1")
+    if result['succeeded'] and result['stdout'].strip():
+        print(f"\npsql version output: {result['stdout'].strip()}")
+        # The psql version should match the libpq version
+        import re
+        version_match = re.search(r'psql \(PostgreSQL\) (\d+)\.', result['stdout'])
+        if version_match:
+            major_version = int(version_match.group(1))
+            print(f"psql/libpq major version: {major_version}")
+            assert major_version >= 14, f"psql/libpq version {major_version} is less than 14"
+        else:
+            print("Could not parse psql version")
+    
+    print("✓ libpq5 version is >= 14")
+
+
+def test_postgrest_read_only_session_attrs(host):
+    """Test PostgREST with target_session_attrs=read-only and check for session errors."""
+    # First, check if PostgreSQL is configured for read-only mode
+    result = run_ssh_command(host['ssh'], "sudo -u postgres psql -c \"SHOW default_transaction_read_only;\"")
+    if result['succeeded']:
+        default_read_only = result['stdout'].strip()
+        print(f"PostgreSQL default_transaction_read_only: {default_read_only}")
+    else:
+        print("Could not check PostgreSQL read-only setting")
+        default_read_only = "unknown"
+    
+    # Check if PostgreSQL is in recovery mode (standby)
+    result = run_ssh_command(host['ssh'], "sudo -u postgres psql -c \"SELECT pg_is_in_recovery();\"")
+    if result['succeeded']:
+        in_recovery = result['stdout'].strip()
+        print(f"PostgreSQL pg_is_in_recovery: {in_recovery}")
+    else:
+        print("Could not check PostgreSQL recovery status")
+        in_recovery = "unknown"
+    
+    # Find PostgreSQL configuration file
+    result = run_ssh_command(host['ssh'], "sudo -u postgres psql -c \"SHOW config_file;\"")
+    if result['succeeded']:
+        config_file = result['stdout'].strip().split('\n')[2].strip()  # Skip header and get the actual path
+        print(f"PostgreSQL config file: {config_file}")
+    else:
+        print("Could not find PostgreSQL config file")
+        config_file = "/etc/postgresql/15/main/postgresql.conf"  # Default fallback
+    
+    # Backup PostgreSQL config
+    result = run_ssh_command(host['ssh'], f"sudo cp {config_file} {config_file}.backup")
+    assert result['succeeded'], "Failed to backup PostgreSQL config"
+    
+    # Add read-only setting to PostgreSQL config
+    result = run_ssh_command(host['ssh'], f"echo 'default_transaction_read_only = on' | sudo tee -a {config_file}")
+    assert result['succeeded'], "Failed to add read-only setting to PostgreSQL config"
+    
+    # Restart PostgreSQL to apply the new configuration
+    result = run_ssh_command(host['ssh'], "sudo systemctl restart postgresql")
+    assert result['succeeded'], "Failed to restart PostgreSQL"
+    
+    # Wait for PostgreSQL to start up
+    sleep(5)
+    
+    # Verify the change took effect
+    result = run_ssh_command(host['ssh'], "sudo -u postgres psql -c \"SHOW default_transaction_read_only;\"")
+    if result['succeeded']:
+        new_default_read_only = result['stdout'].strip()
+        print(f"PostgreSQL default_transaction_read_only after change: {new_default_read_only}")
+    else:
+        print("Could not verify PostgreSQL read-only setting change")
+    
+    # First, backup the current PostgREST config
+    result = run_ssh_command(host['ssh'], "sudo cp /etc/postgrest/base.conf /etc/postgrest/base.conf.backup")
+    assert result['succeeded'], "Failed to backup PostgREST config"
+    
+    try:
+        # Read the current config to get the db-uri
+        result = run_ssh_command(host['ssh'], "sudo cat /etc/postgrest/base.conf | grep '^db-uri'")
+        assert result['succeeded'], "Failed to read current db-uri"
+        
+        current_db_uri = result['stdout'].strip()
+        print(f"Current db-uri: {current_db_uri}")
+        
+        # Extract just the URI part (remove the db-uri = " prefix and trailing quote)
+        uri_start = current_db_uri.find('"') + 1
+        uri_end = current_db_uri.rfind('"')
+        base_uri = current_db_uri[uri_start:uri_end]
+        
+        # Modify the URI to add target_session_attrs=read-only
+        if '?' in base_uri:
+            # URI already has parameters, add target_session_attrs
+            modified_uri = base_uri + "&target_session_attrs=read-only"
+        else:
+            # URI has no parameters, add target_session_attrs
+            modified_uri = base_uri + "?target_session_attrs=read-only"
+        
+        print(f"Modified URI: {modified_uri}")
+        
+        # Use awk to replace the db-uri line more reliably
+        result = run_ssh_command(host['ssh'], f"sudo awk '{{if ($1 == \"db-uri\") print \"db-uri = \\\"{modified_uri}\\\"\"; else print $0}}' /etc/postgrest/base.conf > /tmp/new_base.conf && sudo mv /tmp/new_base.conf /etc/postgrest/base.conf")
+        assert result['succeeded'], "Failed to update db-uri in config"
+        
+        # Verify the change was made correctly
+        result = run_ssh_command(host['ssh'], "sudo cat /etc/postgrest/base.conf | grep '^db-uri'")
+        print(f"Updated db-uri line: {result['stdout'].strip()}")
+        
+        # Also show the full config to debug
+        result = run_ssh_command(host['ssh'], "sudo cat /etc/postgrest/base.conf")
+        print(f"Full config after change:\n{result['stdout']}")
+        
+        # Restart PostgREST to apply the new configuration
+        result = run_ssh_command(host['ssh'], "sudo systemctl restart postgrest")
+        assert result['succeeded'], "Failed to restart PostgREST"
+        
+        # Wait a moment for PostgREST to start up
+        sleep(5)
+        
+        # Check if PostgREST is running
+        result = run_ssh_command(host['ssh'], "sudo systemctl is-active postgrest")
+        if not (result['succeeded'] and result['stdout'].strip() == 'active'):
+            # If PostgREST failed to start, check the logs to see why
+            log_result = run_ssh_command(host['ssh'], "sudo journalctl -u postgrest --since '5 seconds ago' --no-pager")
+            print(f"PostgREST failed to start. Recent logs:\n{log_result['stdout']}")
+            assert False, "PostgREST failed to start after config change"
+        
+        # Make a test request to trigger any potential session errors
+        try:
+            response = requests.get(
+                f"http://{host['ip']}/rest/v1/",
+                headers={"apikey": anon_key, "authorization": f"Bearer {anon_key}"},
+                timeout=10
+            )
+            print(f"Test request status: {response.status_code}")
+        except Exception as e:
+            print(f"Test request failed: {str(e)}")
+        
+        # Check PostgREST logs for "session is not read-only" errors
+        result = run_ssh_command(host['ssh'], "sudo journalctl -u postgrest --since '5 seconds ago' | grep -i 'session is not read-only' || true")
+        
+        if result['stdout'].strip():
+            print(f"\nFound 'session is not read-only' errors in PostgREST logs:\n{result['stdout']}")
+            assert False, "PostgREST logs contain 'session is not read-only' errors even though PostgreSQL is configured for read-only mode"
+        else:
+            print("\nNo 'session is not read-only' errors found in PostgREST logs")
+            
+    finally:
+        # Restore the original configuration
+        result = run_ssh_command(host['ssh'], "sudo cp /etc/postgrest/base.conf.backup /etc/postgrest/base.conf")
+        if result['succeeded']:
+            result = run_ssh_command(host['ssh'], "sudo systemctl restart postgrest")
+            if result['succeeded']:
+                print("Restored original PostgREST configuration")
+            else:
+                print("Warning: Failed to restart PostgREST after restoring config")
+        else:
+            print("Warning: Failed to restore original PostgREST configuration")
+        
+        # Restore PostgreSQL to original configuration
+        result = run_ssh_command(host['ssh'], f"sudo cp {config_file}.backup {config_file}")
+        if result['succeeded']:
+            result = run_ssh_command(host['ssh'], "sudo systemctl restart postgresql")
+            if result['succeeded']:
+                print("Restored PostgreSQL to original configuration")
+            else:
+                print("Warning: Failed to restart PostgreSQL after restoring config")
+        else:
+            print("Warning: Failed to restore PostgreSQL configuration")
+

From d6e11599c52daf0ee23677340b620ab58ac5361e Mon Sep 17 00:00:00 2001
From: Div Arora <root@darora.com>
Date: Tue, 24 Jun 2025 11:30:49 +0800
Subject: [PATCH 12/17] ci: no longer build Oriole QEMU artifact

Said artifact is no longer being used. If needed in the future, it can
be trivially reintroduced, but in the meantime there's no reason to
expend resources on them.
---
 .github/workflows/qemu-image-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml
index 8abee1461..97bc92a1c 100644
--- a/.github/workflows/qemu-image-build.yml
+++ b/.github/workflows/qemu-image-build.yml
@@ -30,7 +30,7 @@ jobs:
       - name: Set PostgreSQL versions - only builds pg17 atm
         id: set-versions
         run: |
-          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[1,2]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[1]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
           echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
 
   build:

From 605f9de9f4ef4de8cac44fd035dd9daabb26be5e Mon Sep 17 00:00:00 2001
From: Div Arora <root@darora.com>
Date: Tue, 24 Jun 2025 11:18:59 +0800
Subject: [PATCH 13/17] chore: disable auth config reloading within qemu
 artifact

The QEMU artifact is not running with a filesystem that supports the
inotify API, resulting in spurious error messages, and broken config
reloading.
---
 ansible/files/gotrue.service.j2 | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ansible/files/gotrue.service.j2 b/ansible/files/gotrue.service.j2
index 272e5b871..2478e99e6 100644
--- a/ansible/files/gotrue.service.j2
+++ b/ansible/files/gotrue.service.j2
@@ -4,7 +4,12 @@ Description=Gotrue
 [Service]
 Type=simple
 WorkingDirectory=/opt/gotrue
+{% if qemu_mode is defined and qemu_mode %}
+ExecStart=/opt/gotrue/gotrue
+{% else %}
 ExecStart=/opt/gotrue/gotrue --config-dir /etc/auth.d
+{% endif %}
+
 User=gotrue
 Restart=always
 RestartSec=3

From c14a67d1130b2a4c149bbaeb1d9d20ab7bfc318a Mon Sep 17 00:00:00 2001
From: Div Arora <root@darora.com>
Date: Sun, 4 May 2025 11:25:49 +0800
Subject: [PATCH 14/17] chore: strip out unnecessary dependencies from QEMU
 builds

Also upgrades to Alpine 3.22.
---
 .github/workflows/qemu-image-build.yml     |  1 +
 Dockerfile-kubernetes                      | 11 +++++++----
 ebssurrogate/scripts/qemu-bootstrap-nix.sh |  2 +-
 scripts/90-cleanup-qemu.sh                 |  5 ++++-
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml
index 97bc92a1c..ffd12683c 100644
--- a/.github/workflows/qemu-image-build.yml
+++ b/.github/workflows/qemu-image-build.yml
@@ -10,6 +10,7 @@ on:
       - 'qemu-arm64-nix.pkr.hcl'
       - 'common-nix.vars.pkr.hcl'
       - 'ansible/vars.yml'
+      - 'scripts/*'
   workflow_dispatch:
 
 permissions:
diff --git a/Dockerfile-kubernetes b/Dockerfile-kubernetes
index a1a39e498..2a5e1ca1e 100644
--- a/Dockerfile-kubernetes
+++ b/Dockerfile-kubernetes
@@ -1,9 +1,12 @@
-FROM alpine:3.21
+FROM alpine:3.22
 
 ADD ./output-cloudimg/packer-cloudimg /disk/focal.qcow2
 
-RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh-client nftables cloud-utils-localds aavmf virtiofsd
-# dev stuff
-# RUN apk add --no-cache iproute2
+RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh-client aavmf virtiofsd \
+    && truncate -s 64M /root/varstore.img \
+    && truncate -s 64M /root/efi.img \
+    && dd if=/usr/share/AAVMF/QEMU_EFI.fd of=/root/efi.img conv=notrunc \
+    && qemu-img create -f qcow2 /tmp/disk.qcow2 -b /disk/focal.qcow2 -F qcow2 \
+    && apk del --no-cache aavmf qemu-img
 
 CMD exec /bin/sh -c "trap : TERM INT; sleep 9999999999d & wait"
diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh
index ad5c2744c..b52b85789 100755
--- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh
+++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh
@@ -155,6 +155,6 @@ execute_stage2_playbook
 # we do not want to ship an initialized DB as this is performed as needed
 mkdir -p /db/template
 mv /data/pgdata /db/template
+cloud-init clean --logs
 clean_legacy_things
 clean_system
-cloud-init clean --logs
diff --git a/scripts/90-cleanup-qemu.sh b/scripts/90-cleanup-qemu.sh
index d6c6ade05..50d8d956f 100644
--- a/scripts/90-cleanup-qemu.sh
+++ b/scripts/90-cleanup-qemu.sh
@@ -43,7 +43,10 @@ elif [ -n "$(command -v apt-get)" ]; then
   add-apt-repository --yes --remove ppa:ansible/ansible
 
   source /etc/os-release
-  apt-get -y remove --purge linux-headers-5.11.0-1021-aws
+
+  apt-mark manual libevent-2.1-7 # required for pgbouncer
+  apt-get -y remove --purge ansible-core apport appstream bash-completion bcache-tools bind9-dnsutils bind9-host bind9-libs bolt btrfs-progs byobu command-not-found console-setup distro-info eject fonts-ubuntu-console friendly-recovery ftp fwupd gawk gdisk keyboard-configuration libvolume-key1 libicu66 libssl-dev lvm2 lxd-agent-loader man-db mdadm modemmanager mtd-utils nano netcat-openbsd nfs-common ntfs-3g parted pastebinit screen strace thin-provisioning-tools tmux usb-modeswitch vim vim-runtime wget whiptail xfsprogs
+  apt remove -y --purge libc6-dev linux-libc-dev libevent-dev libpcre3-dev libsystemd-dev
 
   apt-get -y update
   apt-get -y upgrade

From fc0095796d1fa316eab87086f0bcb40425164e68 Mon Sep 17 00:00:00 2001
From: Tom Ashley <tom.ashley@gmail.com>
Date: Mon, 23 Jun 2025 11:38:16 +0100
Subject: [PATCH 15/17] chore: version bumps

---
 ansible/tasks/internal/gandalf.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ansible/tasks/internal/gandalf.yml b/ansible/tasks/internal/gandalf.yml
index 47231daad..2c0d03fba 100644
--- a/ansible/tasks/internal/gandalf.yml
+++ b/ansible/tasks/internal/gandalf.yml
@@ -62,13 +62,13 @@
     src: "/opt/gandalf/gandalf-linux-{{ arch }}"
     state: link
     owner: gandalf
-    mode: '0755'
+    mode: "0755"
     force: yes
 
 - name: gandalf - create salt systemd timer file
   copy:
-     src: files/gandalf_config/gandalf_salt.timer
-     dest: /etc/systemd/system/gandalf_salt.timer
+    src: files/gandalf_config/gandalf_salt.timer
+    dest: /etc/systemd/system/gandalf_salt.timer
 
 - name: gandalf - create salt service file
   copy:

From 65cc7434d3b5999b4a8bf9b8ef7875887f49ca02 Mon Sep 17 00:00:00 2001
From: Crispy1975 <12525875+Crispy1975@users.noreply.github.com>
Date: Tue, 24 Jun 2025 18:11:18 +0100
Subject: [PATCH 16/17] tests: update user and group test adding gandalf

---
 ansible/files/permission_check.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/ansible/files/permission_check.py b/ansible/files/permission_check.py
index a753f69ec..90e9d52de 100644
--- a/ansible/files/permission_check.py
+++ b/ansible/files/permission_check.py
@@ -94,11 +94,17 @@
     "systemd-coredump": [
         {"groupname": "systemd-coredump", "username": "systemd-coredump"}
     ],
+    "gandalf": [
+        {"groupname": "gandalf", "username": "gandalf"},
+        {"groupname": "admin", "username": "gandalf"},
+        {"groupname": "salt", "username": "gandalf"},
+    ],
 }
 
 # postgresql.service is expected to mount /etc as read-only
 expected_mount = "/etc ro"
 
+
 # This program depends on osquery being installed on the system
 # Function to run osquery
 def run_osquery(query):
@@ -154,6 +160,7 @@ def check_nixbld_users():
 
     print("All nixbld users are in the 'nixbld' group.")
 
+
 def check_postgresql_mount():
     # processes table has the nix .postgres-wrapped path as the
     # binary path, rather than /usr/lib/postgresql/bin/postgres which
@@ -182,6 +189,7 @@ def check_postgresql_mount():
 
     print("postgresql.service mounts /etc as read-only.")
 
+
 def main():
     parser = argparse.ArgumentParser(
         prog="Supabase Postgres Artifact Permissions Checker",
@@ -251,5 +259,6 @@ def main():
     # Check if postgresql.service is using a read-only mount for /etc
     check_postgresql_mount()
 
+
 if __name__ == "__main__":
     main()

From 9dc85fd7f79a08792ffcb9c333984b7659e78538 Mon Sep 17 00:00:00 2001
From: Crispy1975 <12525875+Crispy1975@users.noreply.github.com>
Date: Tue, 24 Jun 2025 20:56:17 +0100
Subject: [PATCH 17/17] chore: add gandalf user to user list

---
 ansible/files/permission_check.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ansible/files/permission_check.py b/ansible/files/permission_check.py
index 90e9d52de..b34a56bb4 100644
--- a/ansible/files/permission_check.py
+++ b/ansible/files/permission_check.py
@@ -242,6 +242,7 @@ def main():
         "postgrest",
         "tcpdump",
         "systemd-coredump",
+        "gandalf",
     ]
     if not qemu_artifact:
         usernames.append("ec2-instance-connect")