Skip to content

Commit d4e5d33

Browse files
hunleydhunleyd
committed
Merge branch 'develop' of github.com:supabase/postgres into INDATA-152
* 'develop' of github.com:supabase/postgres: fix: don't run a nix installer on qemu image build (#1916) fix: restore ami release to us east 1 (#1913) feat: upgrade admin-mgr to 0.32.3 (#1915) fix: Prepare job (lines 23-25): Moved checkout before nix-install-ephemeral (#1914) feat: support multiple versions of the wal2json extension (#1875)
2 parents 519f7b0 + 595495d commit d4e5d33

File tree

12 files changed

+126
-42
lines changed

12 files changed

+126
-42
lines changed

‎.github/workflows/ami-release-nix.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ jobs:
9696
GIT_SHA=${{github.sha}}
9797
nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
9898
# why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined?
99-
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
99+
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' amazon-arm64-nix.pkr.hcl
100100
101101
- name: Build AMI stage 2
102102
env:
@@ -105,7 +105,7 @@ jobs:
105105
GIT_SHA=${{github.sha}}
106106
nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl
107107
POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
108-
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
108+
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' stage2-nix-psql.pkr.hcl
109109
110110
- name: Grab release version
111111
id: process_release_version
@@ -184,9 +184,9 @@ jobs:
184184
- name: Cleanup resources after build
185185
if: ${{ always() }}
186186
run: |
187-
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
187+
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
188188
189189
- name: Cleanup resources on build cancellation
190190
if: ${{ cancelled() }}
191191
run: |
192-
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
192+
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids

‎.github/workflows/dockerhub-release-matrix.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@ jobs:
2020
outputs:
2121
matrix_config: ${{ steps.set-matrix.outputs.matrix_config }}
2222
steps:
23-
- uses: ./.github/actions/nix-install-ephemeral
2423
- name: Checkout Repo
2524
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
25+
- uses: ./.github/actions/nix-install-ephemeral
2626
- name: Generate build matrix
2727
id: set-matrix
2828
run: |

‎.github/workflows/qemu-image-build.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,6 @@ jobs:
4848
- name: Checkout Repo
4949
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
5050

51-
- uses: ./.github/actions/nix-install-ephemeral
52-
5351
- name: Run checks if triggered manually
5452
if: ${{ github.event_name == 'workflow_dispatch' }}
5553
run: |

‎ansible/vars.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ postgres_major:
1010

1111
# Full version strings for each major version
1212
postgres_release:
13-
postgresorioledb-17: "17.5.1.065-orioledb-INDATA152-1"
14-
postgres17: "17.6.1.044-INDATA152-1"
15-
postgres15: "15.14.1.044-INDATA152-1"
13+
postgresorioledb-17: "17.5.1.068-orioledb-INDATA152-1"
14+
postgres17: "17.6.1.047-INDATA152-1"
15+
postgres15: "15.14.1.047-INDATA152-1"
1616

1717
# Non Postgres Extensions
1818
pgbouncer_release: 1.19.0
@@ -54,7 +54,7 @@ postgres_exporter_release_checksum:
5454
amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84
5555

5656
adminapi_release: "0.93.0"
57-
adminmgr_release: "0.32.1"
57+
adminmgr_release: "0.32.3"
5858
supabase_admin_agent_release: 1.4.38
5959
supabase_admin_agent_splay: 30s
6060

‎nix/ext/tests/default.nix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ builtins.listToAttrs (
224224
"pgaudit"
225225
"pg_tle"
226226
"vector"
227+
"wal2json"
227228
"wrappers"
228229
]
229230
)

‎nix/ext/versions.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -757,5 +757,14 @@
757757
],
758758
"hash": "sha256-crxj5R9jblIv0h8lpqddAoYe2UqgUlnvbOajKTzVces="
759759
}
760+
},
761+
"wal2json": {
762+
"2.6": {
763+
"postgresql": [
764+
"15",
765+
"17"
766+
],
767+
"hash": "sha256-+QoACPCKiFfuT2lJfSUmgfzC5MXf75KpSoc2PzPxKyM="
768+
}
760769
}
761770
}

‎nix/ext/wal2json.nix‎

Lines changed: 90 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,106 @@
11
{
2+
pkgs,
23
lib,
34
stdenv,
45
fetchFromGitHub,
56
postgresql,
7+
makeWrapper,
68
}:
79

8-
stdenv.mkDerivation rec {
10+
let
911
pname = "wal2json";
10-
version = "2_6";
12+
build =
13+
version: rev: hash:
14+
stdenv.mkDerivation rec {
15+
inherit version pname;
1116

12-
src = fetchFromGitHub {
13-
owner = "eulerto";
14-
repo = "wal2json";
15-
rev = "wal2json_${builtins.replaceStrings [ "." ] [ "_" ] version}";
16-
hash = "sha256-+QoACPCKiFfuT2lJfSUmgfzC5MXf75KpSoc2PzPxKyM=";
17-
};
17+
src = fetchFromGitHub {
18+
owner = "eulerto";
19+
repo = "wal2json";
20+
rev = "wal2json_${builtins.replaceStrings [ "." ] [ "_" ] version}";
21+
inherit hash;
22+
};
23+
24+
buildInputs = [ postgresql ];
25+
26+
makeFlags = [ "USE_PGXS=1" ];
27+
28+
installPhase = ''
29+
runHook preInstall
30+
31+
mkdir -p $out/share/postgresql/extension
32+
33+
# Install versioned library
34+
install -Dm755 ${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix}
35+
if [[ "${version}" == "${latestVersion}" ]]; then
36+
cp sql/*.sql $out/share/postgresql/extension/
37+
fi
38+
39+
touch $out/share/postgresql/extension/${pname}--${version}.control
40+
touch $out/share/postgresql/extension/${pname}--${version}.sql
41+
42+
runHook postInstall
43+
'';
44+
45+
meta = with lib; {
46+
description = "PostgreSQL JSON output plugin for changeset extraction";
47+
homepage = "https://github.com/eulerto/wal2json";
48+
changelog = "https://github.com/eulerto/wal2json/releases/";
49+
platforms = postgresql.meta.platforms;
50+
license = licenses.bsd3;
51+
};
52+
};
53+
allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).wal2json;
54+
supportedVersions = lib.filterAttrs (
55+
_: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
56+
) allVersions;
57+
versions = lib.naturalSort (lib.attrNames supportedVersions);
58+
latestVersion = lib.last versions;
59+
numberOfVersions = builtins.length versions;
60+
packages = builtins.attrValues (
61+
lib.mapAttrs (name: value: build name value.rev value.hash) supportedVersions
62+
);
63+
in
64+
pkgs.buildEnv {
65+
name = pname;
66+
paths = packages;
67+
nativeBuildInputs = [ makeWrapper ];
68+
pathsToLink = [
69+
"/lib"
70+
"/share/postgresql/extension"
71+
];
72+
postBuild = ''
73+
ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
1874
19-
buildInputs = [ postgresql ];
75+
{
76+
echo "default_version = '${latestVersion}'"
77+
} > $out/share/postgresql/extension/${pname}.control
2078
21-
makeFlags = [ "USE_PGXS=1" ];
79+
# Create empty upgrade files between consecutive versions
80+
# plpgsql_check ships without upgrade scripts - extensions are backward-compatible
81+
previous_version=""
82+
for ver in ${lib.concatStringsSep " " versions}; do
83+
if [[ -n "$previous_version" ]]; then
84+
touch $out/share/postgresql/extension/${pname}--''${previous_version}--''${ver}.sql
85+
fi
86+
previous_version=$ver
87+
done
2288
23-
installPhase = ''
24-
install -D -t $out/lib *${postgresql.dlSuffix}
25-
install -D -t $out/share/postgresql/extension sql/*.sql
89+
# checks
90+
(set -x
91+
test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${
92+
toString (numberOfVersions + 1)
93+
}"
94+
)
2695
'';
2796

28-
meta = with lib; {
29-
description = "PostgreSQL JSON output plugin for changeset extraction";
30-
homepage = "https://github.com/eulerto/wal2json";
31-
changelog = "https://github.com/eulerto/wal2json/releases/tag/wal2json_${version}";
32-
platforms = postgresql.meta.platforms;
33-
license = licenses.bsd3;
97+
passthru = {
98+
inherit versions numberOfVersions;
99+
pname = "${pname}-all";
100+
version =
101+
"multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
102+
defaultSettings = {
103+
wal_level = "logical";
104+
};
34105
};
35106
}

‎nix/tests/expected/wal2json.out‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,17 @@ from
1919
'reg_test',
2020
null,
2121
null,
22-
'include-pk', '1',
22+
'include-pk', 'true',
2323
'include-transaction', 'false',
2424
'include-timestamp', 'false',
25-
'include-type-oids', 'false',
25+
'include-type-oids', 'true',
2626
'format-version', '2',
27-
'actions', 'insert,update,delete'
27+
'actions', 'insert,update,delete',
28+
'add-tables', 'v.foo'
2829
) x;
29-
data
30-
--------------------------------------------------------------------------------------------------------------------------------------
31-
{"action":"I","schema":"v","table":"foo","columns":[{"name":"id","type":"integer","value":1}],"pk":[{"name":"id","type":"integer"}]}
30+
data
31+
----------------------------------------------------------------------------------------------------------------------------------------------------------------
32+
{"action":"I","schema":"v","table":"foo","columns":[{"name":"id","type":"integer","typeoid":23,"value":1}],"pk":[{"name":"id","type":"integer","typeoid":23}]}
3233
(1 row)
3334

3435
select

‎nix/tests/expected/z_15_ext_interface.out‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,8 @@ order by
121121
-----------------
122122
pg_cron
123123
tsm_system_time
124-
(2 rows)
124+
wal2json
125+
(3 rows)
125126

126127
/*
127128

‎nix/tests/expected/z_17_ext_interface.out‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,8 @@ order by
106106
pgjwt
107107
postgis_tiger_geocoder
108108
tsm_system_time
109-
(4 rows)
109+
wal2json
110+
(5 rows)
110111

111112
/*
112113

0 commit comments

Comments
 (0)