Skip to content

Commit f24e0d0

Browse files
hunleydhunleyd
committed
Merge remote-tracking branch 'origin' into INDATA-152
* origin: refactor: Move read-replica.conf to conf.d (#1956) ci: Custom GitHub Runners for Nix Builds (#1745) fix: search path and migration grants (#1939) chore: bump admin api version (#1964)
2 parents d9a8841 + 75c71e3 commit f24e0d0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+1396
-212
lines changed

‎.github/actionlint.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
self-hosted-runner:
2+
labels:
3+
- aarch64-darwin
4+
- aarch64-linux
5+
- blacksmith-32vcpu-ubuntu-2404

‎.github/actions/nix-install-self-hosted/action.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: 'Configure Nix on self hosted runners'
2+
description: 'Sets up AWS credentials to push to the Nix binary cache'
3+
inputs:
4+
aws-role-duration:
5+
description: 'AWS role session duration in seconds'
6+
required: false
7+
default: '18000'
8+
9+
runs:
10+
using: 'composite'
11+
steps:
12+
- name: aws-creds
13+
uses: aws-actions/configure-aws-credentials@v4.3.1
14+
with:
15+
disable-retry: true
16+
aws-region: us-east-2
17+
role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev
18+
role-session-name: gha-oidc-${{ github.run_id }}
19+
role-duration-seconds: ${{ inputs.aws-role-duration }}
20+
21+
- name: Write creds files
22+
shell: bash
23+
run: |
24+
umask 006
25+
cat > /etc/nix/aws/nix-aws-credentials <<EOF
26+
[ci-uploader]
27+
aws_access_key_id = ${AWS_ACCESS_KEY_ID}
28+
aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}
29+
aws_session_token = ${AWS_SESSION_TOKEN}
30+
EOF

‎.github/workflows/nix-build.yml‎

Lines changed: 180 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -14,77 +14,201 @@ permissions:
1414
contents: write
1515
packages: write
1616

17-
concurrency:
18-
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
19-
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
20-
2117
jobs:
22-
build-run-image:
18+
nix-eval:
19+
uses: ./.github/workflows/nix-eval.yml
20+
secrets:
21+
DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
22+
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
23+
24+
nix-build-packages-aarch64-linux:
25+
name: >-
26+
${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }}
27+
(aarch64-linux)
28+
needs: nix-eval
29+
runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
30+
if: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_linux != null }}
31+
strategy:
32+
fail-fast: false
33+
max-parallel: 5
34+
matrix: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_linux }}
35+
steps:
36+
- name: Checkout Repo
37+
if: ${{ matrix.attr != '' }}
38+
uses: actions/checkout@v4
39+
- name: Install nix (ephemeral)
40+
if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }}
41+
uses: ./.github/actions/nix-install-ephemeral
42+
with:
43+
push-to-cache: 'true'
44+
env:
45+
DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
46+
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
47+
- name: Install nix (self-hosted)
48+
if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }}
49+
uses: ./.github/actions/nix-install-self-hosted
50+
- name: nix build
51+
if: ${{ matrix.attr != '' }}
52+
shell: bash
53+
run: nix build --accept-flake-config -L .#${{ matrix.attr }}
54+
55+
nix-build-checks-aarch64-linux:
56+
name: >-
57+
${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }}
58+
(aarch64-linux)
59+
needs: [nix-eval, nix-build-packages-aarch64-linux]
60+
runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
61+
if: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_linux != null }}
62+
strategy:
63+
fail-fast: false
64+
max-parallel: 5
65+
matrix: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_linux }}
66+
steps:
67+
- name: Checkout Repo
68+
if: ${{ matrix.attr != '' }}
69+
uses: actions/checkout@v4
70+
- name: Install nix (ephemeral)
71+
if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }}
72+
uses: ./.github/actions/nix-install-ephemeral
73+
with:
74+
push-to-cache: 'true'
75+
env:
76+
DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
77+
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
78+
- name: Install nix (self-hosted)
79+
if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }}
80+
uses: ./.github/actions/nix-install-self-hosted
81+
- name: nix build
82+
if: ${{ matrix.attr != '' }}
83+
shell: bash
84+
run: nix build --accept-flake-config -L .#${{ matrix.attr }}
85+
86+
nix-build-packages-aarch64-darwin:
87+
name: >-
88+
${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }}
89+
(aarch64-darwin)
90+
needs: nix-eval
91+
runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
92+
if: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_darwin != null }}
93+
strategy:
94+
fail-fast: false
95+
max-parallel: 5
96+
matrix: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_darwin }}
97+
steps:
98+
- name: Checkout Repo
99+
if: ${{ matrix.attr != '' }}
100+
uses: actions/checkout@v4
101+
- name: Install nix
102+
if: ${{ matrix.attr != '' }}
103+
uses: ./.github/actions/nix-install-self-hosted
104+
- name: nix build
105+
if: ${{ matrix.attr != '' }}
106+
shell: bash
107+
run: nix build --accept-flake-config -L .#${{ matrix.attr }}
108+
109+
nix-build-checks-aarch64-darwin:
110+
name: >-
111+
${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }}
112+
(aarch64-darwin)
113+
needs: [nix-eval, nix-build-packages-aarch64-darwin]
114+
runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
115+
if: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_darwin != null }}
116+
strategy:
117+
fail-fast: false
118+
max-parallel: 5
119+
matrix: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_darwin }}
120+
steps:
121+
- name: Checkout Repo
122+
if: ${{ matrix.attr != '' }}
123+
uses: actions/checkout@v4
124+
- name: Install nix
125+
if: ${{ matrix.attr != '' }}
126+
uses: ./.github/actions/nix-install-self-hosted
127+
- name: nix build
128+
if: ${{ matrix.attr != '' }}
129+
shell: bash
130+
run: nix build --accept-flake-config -L .#${{ matrix.attr }}
131+
132+
nix-build-packages-x86_64-linux:
133+
name: >-
134+
${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }}
135+
(x86_64-linux)
136+
needs: nix-eval
137+
runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
138+
if: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).x86_64_linux != null }}
23139
strategy:
24140
fail-fast: false
25-
matrix:
26-
include:
27-
- runner: blacksmith-32vcpu-ubuntu-2404
28-
arch: amd64
29-
- runner: blacksmith-32vcpu-ubuntu-2404-arm
30-
arch: arm64
31-
- runner: macos-latest-xlarge
32-
arch: arm64
33-
runs-on: ${{ matrix.runner }}
34-
timeout-minutes: 180
141+
max-parallel: 5
142+
matrix: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).x86_64_linux }}
35143
steps:
36144
- name: Checkout Repo
37-
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
38-
- uses: ./.github/actions/nix-install-ephemeral
145+
if: ${{ matrix.attr != '' }}
146+
uses: actions/checkout@v4
147+
- name: Install nix
148+
if: ${{ matrix.attr != '' }}
149+
uses: ./.github/actions/nix-install-ephemeral
39150
with:
40-
push-to-cache: ${{ github.secret_source == 'Actions' && 'true' || 'false' }}
151+
push-to-cache: 'true'
41152
env:
42153
DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
43154
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
44-
- name: Aggressive disk cleanup for DuckDB build
45-
if: matrix.runner == 'macos-latest-xlarge'
46-
run: |
47-
nix --version
48-
echo "=== BEFORE CLEANUP ==="
49-
df -h
50-
# Remove major space consumers
51-
sudo rm -rf /usr/share/dotnet || true
52-
sudo rm -rf /usr/local/lib/android || true
53-
sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform || true
54-
sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/watchOS.platform || true
55-
sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/tvOS.platform || true
56-
# Clean everything possible
57-
sudo rm -rf /opt/ghc || true
58-
sudo rm -rf /usr/local/share/boost || true
59-
sudo rm -rf /opt/homebrew || true
60-
sudo xcrun simctl delete all 2>/dev/null || true
61-
# Aggressive cache cleanup
62-
sudo rm -rf /System/Library/Caches/* 2>/dev/null || true
63-
sudo rm -rf /Library/Caches/* 2>/dev/null || true
64-
sudo rm -rf ~/Library/Caches/* 2>/dev/null || true
65-
sudo rm -rf /private/var/log/* 2>/dev/null || true
66-
sudo rm -rf /tmp/* 2>/dev/null || true
67-
echo "=== AFTER CLEANUP ==="
68-
df -h
69-
-
70-
name: Build psql bundle
71-
run: >
72-
nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48"
73-
-- --skip-cached --no-nom ${{ matrix.runner == 'macos-latest-xlarge' && '--max-jobs 1' || '' }} --copy-to "s3://nix-postgres-artifacts?secret-key=/etc/nix/nix-secret-key"
74-
--flake ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')"
155+
- name: nix build
156+
if: ${{ matrix.attr != '' }}
157+
shell: bash
158+
run: nix build --accept-flake-config -L .#${{ matrix.attr }}
159+
160+
nix-build-checks-x86_64-linux:
161+
name: >-
162+
${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }}
163+
(x86_64-linux)
164+
needs: [nix-eval, nix-build-packages-x86_64-linux]
165+
runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }}
166+
if: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).x86_64_linux != null }}
167+
strategy:
168+
fail-fast: false
169+
max-parallel: 5
170+
matrix: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).x86_64_linux }}
171+
steps:
172+
- name: Checkout Repo
173+
if: ${{ matrix.attr != '' }}
174+
uses: actions/checkout@v4
175+
- name: Install nix
176+
if: ${{ matrix.attr != '' }}
177+
uses: ./.github/actions/nix-install-ephemeral
178+
with:
179+
push-to-cache: 'true'
75180
env:
76-
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
77-
AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
78-
AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
181+
DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
182+
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
183+
- name: nix build
184+
if: ${{ matrix.attr != '' }}
185+
shell: bash
186+
run: nix build --accept-flake-config -L .#${{ matrix.attr }}
79187

80188
run-testinfra:
81-
needs: build-run-image
82-
if: ${{ success() }}
189+
needs: [nix-eval, nix-build-packages-aarch64-linux, nix-build-checks-aarch64-linux, nix-build-packages-aarch64-darwin, nix-build-checks-aarch64-darwin, nix-build-packages-x86_64-linux, nix-build-checks-x86_64-linux]
190+
if: |
191+
!cancelled() &&
192+
needs.nix-eval.result == 'success' &&
193+
(needs.nix-build-packages-aarch64-linux.result == 'skipped' || needs.nix-build-packages-aarch64-linux.result == 'success') &&
194+
(needs.nix-build-checks-aarch64-linux.result == 'skipped' || needs.nix-build-checks-aarch64-linux.result == 'success') &&
195+
(needs.nix-build-packages-aarch64-darwin.result == 'skipped' || needs.nix-build-packages-aarch64-darwin.result == 'success') &&
196+
(needs.nix-build-checks-aarch64-darwin.result == 'skipped' || needs.nix-build-checks-aarch64-darwin.result == 'success') &&
197+
(needs.nix-build-packages-x86_64-linux.result == 'skipped' || needs.nix-build-packages-x86_64-linux.result == 'success') &&
198+
(needs.nix-build-checks-x86_64-linux.result == 'skipped' || needs.nix-build-checks-x86_64-linux.result == 'success')
83199
uses: ./.github/workflows/testinfra-ami-build.yml
84200
secrets:
85201
DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
86202

87203
run-tests:
88-
needs: build-run-image
89-
if: ${{ success() }}
204+
needs: [nix-eval, nix-build-packages-aarch64-linux, nix-build-checks-aarch64-linux, nix-build-packages-aarch64-darwin, nix-build-checks-aarch64-darwin, nix-build-packages-x86_64-linux, nix-build-checks-x86_64-linux]
205+
if: |
206+
!cancelled() &&
207+
needs.nix-eval.result == 'success' &&
208+
(needs.nix-build-packages-aarch64-linux.result == 'skipped' || needs.nix-build-packages-aarch64-linux.result == 'success') &&
209+
(needs.nix-build-checks-aarch64-linux.result == 'skipped' || needs.nix-build-checks-aarch64-linux.result == 'success') &&
210+
(needs.nix-build-packages-aarch64-darwin.result == 'skipped' || needs.nix-build-packages-aarch64-darwin.result == 'success') &&
211+
(needs.nix-build-checks-aarch64-darwin.result == 'skipped' || needs.nix-build-checks-aarch64-darwin.result == 'success') &&
212+
(needs.nix-build-packages-x86_64-linux.result == 'skipped' || needs.nix-build-packages-x86_64-linux.result == 'success') &&
213+
(needs.nix-build-checks-x86_64-linux.result == 'skipped' || needs.nix-build-checks-x86_64-linux.result == 'success')
90214
uses: ./.github/workflows/test.yml

‎.github/workflows/nix-eval.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Nix Eval
2+
3+
on:
4+
workflow_call:
5+
outputs:
6+
packages_matrix:
7+
description: 'Generated build matrix for packages'
8+
value: ${{ jobs.eval.outputs.packages_matrix }}
9+
checks_matrix:
10+
description: 'Generated build matrix for checks'
11+
value: ${{ jobs.eval.outputs.checks_matrix }}
12+
secrets:
13+
DEV_AWS_ROLE:
14+
required: false
15+
NIX_SIGN_SECRET_KEY:
16+
required: false
17+
18+
jobs:
19+
eval:
20+
runs-on: blacksmith-32vcpu-ubuntu-2404
21+
outputs:
22+
packages_matrix: ${{ steps.set-matrix.outputs.packages_matrix }}
23+
checks_matrix: ${{ steps.set-matrix.outputs.checks_matrix }}
24+
steps:
25+
- name: Checkout Repo
26+
uses: actions/checkout@v4
27+
- name: Install nix
28+
uses: ./.github/actions/nix-install-ephemeral
29+
with:
30+
push-to-cache: 'true'
31+
env:
32+
DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
33+
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
34+
- name: Restart Nix Daemon
35+
run: |
36+
sudo mv /nix/var/nix/daemon-socket/socket /tmp
37+
- name: Mount Nix cache disk
38+
uses: useblacksmith/stickydisk@v1
39+
with:
40+
key: ${{ github.repository }}-nix-cache-eval-${{ runner.os }}
41+
path: /nix
42+
- name: Restart Nix Daemon
43+
run: |
44+
sudo systemctl restart nix-daemon.service nix-daemon.socket
45+
- id: set-matrix
46+
name: Generate Nix Matrix
47+
run: |
48+
set -Eeu -o pipefail
49+
nix run --accept-flake-config .\#github-matrix -- checks legacyPackages
50+
51+
sudo systemctl stop nix-daemon.socket || true
52+
sudo systemctl stop nix-daemon.service || true
53+
sudo pkill -9 nix-daemon || true
54+
sleep 2

‎ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@ BEGIN
1414
SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow
1515
WHERE usename = p_usename;
1616
END;
17-
$$ LANGUAGE plpgsql SECURITY DEFINER;
17+
$$ LANGUAGE plpgsql
18+
SET search_path = ''
19+
SECURITY DEFINER;
1820

1921
REVOKE ALL ON FUNCTION pgbouncer.get_auth(p_usename TEXT) FROM PUBLIC;
2022
GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(p_usename TEXT) TO pgbouncer;

‎ansible/files/postgresql_config/conf.d/read-replica.conf‎ renamed to ‎ansible/files/postgresql_config/conf.d/read_replica.conf‎

File renamed without changes.

‎ansible/files/postgresql_config/postgresql.conf.j2‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -169,9 +169,6 @@ jit_provider = 'llvmjit' # JIT library to use
169169
# WAL-G specific configurations
170170
#include = '/etc/postgresql-custom/wal-g.conf'
171171

172-
# read replica specific configurations
173-
include = '/etc/postgresql-custom/read-replica.conf'
174-
175172
# supautils specific configurations
176173
#include = '/etc/postgresql-custom/supautils.conf'
177174

‎ansible/tasks/setup-postgres.yml‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -167,18 +167,13 @@
167167
loop_control:
168168
loop_var: 'pg_config_item'
169169

170-
- name: Allow adminapi to write custom config
171-
ansible.builtin.file:
172-
group: postgres
173-
mode: 0775
174-
owner: postgres
175-
path: '{{ item }}'
176-
recurse: yes
177-
state: directory
178-
with_items:
179-
- '/etc/postgresql'
180-
- '/etc/postgresql-custom'
181-
- '/etc/postgresql-custom/conf.d'
170+
- name: Move read-replica.conf file to /etc/postgresql-custom/conf.d/read-replica.conf
171+
ansible.builtin.copy:
172+
dest: '/etc/postgresql-custom/conf.d/read_replica.conf'
173+
mode: '0664'
174+
owner: 'postgres'
175+
group: 'postgres'
176+
src: 'files/postgresql_config/conf.d/read_replica.conf'
182177

183178
- name: create placeholder config files
184179
ansible.builtin.file:

0 commit comments

Comments
 (0)