From 7e2735f0f361e880d97c867f2cc7d4317f87c6c6 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 1 Oct 2024 15:34:55 +0300 Subject: [PATCH 01/21] fix (15.6): wrappers for pg_upgrade --- nix/ext/wrappers/default.nix | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 98d6a7cbc..914a038fa 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -79,6 +79,22 @@ buildPgrxExtension_0_11_3 rec { else echo "Warning: $sql_file not found" fi + + echo "Creating wrappers.so symlinks to support pg_upgrade..." + lib_file="$out/lib/wrappers-$current_version.so" + + if [ -f "$lib_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/lib/wrappers-$previous_version.so" + echo "Creating $new_file" + ln -s "$lib_file" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $lib_file not found" + fi + rm git_tags.txt ''; From 102258543b4062e9b9054473536a7b1a8450f98c Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Thu, 26 Sep 2024 13:53:44 +0100 Subject: [PATCH 02/21] fix: account for `public` grantee --- ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index e978b2a54..4c776844b 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -356,6 +356,7 @@ begin end , case when rec.grantee = 'postgres'::regrole then 'supabase_admin' when rec.grantee = 'supabase_admin'::regrole then 'postgres' + when rec.grantee = 0 then 'public' else rec.grantee::regrole end )); @@ -382,7 +383,7 @@ begin when obj->>'objtype' = 'T' then 'types' when obj->>'objtype' = 'n' then 'schemas' end - , rec.grantee::regrole + , case when rec.grantee = 0 then 'public' else rec.grantee::regrole end , case when rec.is_grantable then 'with grant option' else '' end )); end if; From b13bea9bb582319e5a6ee9fb13dc3faf1176a4cf Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:19:38 +0100 Subject: [PATCH 03/21] fix(ci): respect postgresVersion input (#1237) --- .github/workflows/publish-nix-pgupgrade-scripts.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 5d373ad8f..eb5f7a755 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -72,6 +72,9 @@ jobs: id: process_release_version run: | VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + if [[ "${{ inputs.postgresVersion }}" != "" ]]; then + VERSION=${{ inputs.postgresVersion }} + fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Create a tarball containing pg_upgrade scripts From c31f5a327d3bc23fe11cafd950ef29dc1234d95b Mon Sep 17 00:00:00 2001 From: Kang Ming Date: Sat, 28 Sep 2024 06:33:03 -0700 Subject: [PATCH 04/21] feat: bump gotrue version to v2.162.0 (#1241) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- common.vars.pkr.hcl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index e76e99cb0..471b073c6 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.161.0 -gotrue_release_checksum: sha1:8e45f3511fee8f99a0b1567c73673991a0a5986c +gotrue_release: 2.162.0 +gotrue_release_checksum: sha1:855b23bd002577290c7d42d7042ac0f5316984b1 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index bacdd2c63..b7089a21b 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.124" +postgres-version = "15.6.1.125" diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 3c999dccd..fa1387690 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.1.93" +postgres-version = "15.1.1.94" From 853ab55a983a3505dcb13af5f13f4288a1677fc5 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:32:13 +0100 Subject: [PATCH 05/21] fix: only grant pg_read_all_data if it exists (#1242) * fix: only grant pg_read_all_data if it exists * fix: prevent `public` from being casted into `regrole` --- .../admin_api_scripts/pg_upgrade_scripts/common.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index 4c776844b..a11f8ff32 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -357,7 +357,7 @@ begin , case when rec.grantee = 'postgres'::regrole then 'supabase_admin' when rec.grantee = 'supabase_admin'::regrole then 'postgres' when rec.grantee = 0 then 'public' - else rec.grantee::regrole + else rec.grantee::regrole::text end )); end if; @@ -383,7 +383,7 @@ begin when obj->>'objtype' = 'T' then 'types' when obj->>'objtype' = 'n' then 'schemas' end - , case when rec.grantee = 0 then 'public' else rec.grantee::regrole end + , case when rec.grantee = 0 then 'public' else rec.grantee::regrole::text end , case when rec.is_grantable then 'with grant option' else '' end )); end if; @@ -530,7 +530,14 @@ $$; alter database postgres connection limit -1; -- #incident-2024-09-12-project-upgrades-are-temporarily-disabled -grant pg_read_all_data, pg_signal_backend to postgres; +do $$ +begin + if exists (select from pg_authid where rolname = 'pg_read_all_data') then + execute('grant pg_read_all_data to postgres'); + end if; +end +$$; +grant pg_signal_backend to postgres; set session authorization supabase_admin; drop role supabase_tmp; From 23f5c7ed129c3e54609e27a88d05f2ced9b4b26a Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 1 Oct 2024 23:25:19 +0300 Subject: [PATCH 06/21] chore: workflow fix --- .github/workflows/publish-nix-pgupgrade-bin-flake-version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml index 5b985f4be..13618f721 100644 --- a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml +++ b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml @@ -57,7 +57,7 @@ jobs: publish-prod: runs-on: ubuntu-latest - if: github.ref_name == 'develop' || contains( github.ref, 'release' ) + if: github.ref_name == 'develop' || contains( github.ref, 'release' ) || contains( github.ref, 'pcnc/wrappers-15.6' ) steps: - name: Checkout Repo From 5e986bbd1a6a3d597a76a486ee830f5e61dfa987 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 2 Oct 2024 19:57:50 +0300 Subject: [PATCH 07/21] fix: collision when patching wrappers lib locations for upgrades --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 31a7cd5b3..235b4f4c1 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -211,7 +211,7 @@ function patch_wrappers { WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH") if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" + cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" || true fi fi done @@ -225,7 +225,7 @@ function patch_wrappers { LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" + cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" || true fi fi fi From be24e6ac996b9f638afe8aa66b839b8c630951b5 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Fri, 4 Oct 2024 18:16:55 +0300 Subject: [PATCH 08/21] chore: timescaledb 2.9.1 --- flake.nix | 1 + nix/ext/timescaledb-2.9.1.nix | 51 +++++++++++++++++++ .../expected/extensions_sql_interface.out | 7 +-- 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 nix/ext/timescaledb-2.9.1.nix diff --git a/flake.nix b/flake.nix index 1f9d5f64d..512d32d31 100644 --- a/flake.nix +++ b/flake.nix @@ -107,6 +107,7 @@ ourExtensions = [ ./nix/ext/rum.nix ./nix/ext/timescaledb.nix + ./nix/ext/timescaledb-2.9.1.nix ./nix/ext/pgroonga.nix ./nix/ext/index_advisor.nix ./nix/ext/wal2json.nix diff --git a/nix/ext/timescaledb-2.9.1.nix b/nix/ext/timescaledb-2.9.1.nix new file mode 100644 index 000000000..643256182 --- /dev/null +++ b/nix/ext/timescaledb-2.9.1.nix @@ -0,0 +1,51 @@ +{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }: + +stdenv.mkDerivation rec { + pname = "timescaledb-apache"; + version = "2.9.1"; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ postgresql openssl libkrb5 ]; + + src = fetchFromGitHub { + owner = "timescale"; + repo = "timescaledb"; + rev = version; + hash = "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="; + }; + + cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] + ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; + + # Fix the install phase which tries to install into the pgsql extension dir, + # and cannot be manually overridden. This is rather fragile but works OK. + postPatch = '' + for x in CMakeLists.txt sql/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" + done + + for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" + done + ''; + + + # timescaledb-2.9.1.so already exists in the lib directory + # we have no need for the timescaledb.so or control file + postInstall = '' + rm $out/lib/timescaledb.so + mv $out/share/postgresql/extension/timescaledb.control $out/share/postgresql/extension/timescaledb-2.9.1.control + ''; + + meta = with lib; { + description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; + homepage = "https://www.timescale.com/"; + changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.asl20; + broken = versionOlder postgresql.version "13"; + }; +} diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index ab43f54e6..e240345a0 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -20,10 +20,11 @@ where installed_version is null order by name asc; - name ---------- + name +------------------- pg_cron -(1 row) + timescaledb-2.9.1 +(2 rows) /* From c73ab37e9d1a9e9e9eb321023b67c059df6a3099 Mon Sep 17 00:00:00 2001 From: Stojan Dimitrovski Date: Thu, 3 Oct 2024 11:57:23 +0200 Subject: [PATCH 09/21] feat: bump auth 2.162.1 on 15.6 (#1256) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 3c5ef2172..ed746051d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.162.0 -gotrue_release_checksum: sha1:855b23bd002577290c7d42d7042ac0f5316984b1 +gotrue_release: 2.162.1 +gotrue_release_checksum: sha1:a8b248521f000e027feea2b44e8d9dfb6b054b2e aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 3a55dd70e..89abf35c5 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.126" +postgres-version = "15.6.1.127" From 32f076de733b09a5dcd1a654b3341435d22e05c2 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Fri, 4 Oct 2024 20:05:59 +0300 Subject: [PATCH 10/21] fix(15.6): disable pg_stat_monitor (#1260) * fix: disable pg_stat_monitor * chore: bump version --- common-nix.vars.pkr.hcl | 2 +- nix/tests/postgresql.conf.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 89abf35c5..d988536f3 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.127" +postgres-version = "15.6.1.128" diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index d2d667b9a..ef860afcb 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -718,7 +718,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use From a46c06d312b418901d28828d9389682e6f78a27b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 16:36:51 -0400 Subject: [PATCH 11/21] fix: rm version num from plv8 lib --- nix/ext/plv8.nix | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 5cb4fb67e..c2591eeba 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -103,16 +103,19 @@ stdenv.mkDerivation (finalAttrs: { # so changing them does not cause issues. mv "$out/nix/store"/*/* "$out" rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" - + mv "$out/lib/plv8-${finalAttrs.version}.so" "$out/lib/plv8.so" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plv8.control" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plcoffee.control" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plls.control" ${lib.optionalString stdenv.isDarwin '' - install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}.so + install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8.so + install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8.so + install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so + install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8.so ''} ${lib.optionalString (!stdenv.isDarwin) '' - ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so ''} ''; From 17f9019598668d665e4cb6fe6911bbb484885b7d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 4 Oct 2024 18:03:55 -0400 Subject: [PATCH 12/21] feat: post build rename .so file and edit sql to point to new name --- nix/ext/wrappers/default.nix | 79 ++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 914a038fa..80df2425f 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -63,40 +63,57 @@ buildPgrxExtension_0_11_3 rec { echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt ''; - postInstall = '' - echo "Creating SQL files for previous versions..." - current_version="${version}" - sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" - - if [ -f "$sql_file" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" - echo "Creating $new_file" - cp "$sql_file" "$new_file" - fi - done < git_tags.txt - else - echo "Warning: $sql_file not found" - fi +postInstall = '' + echo "Modifying main SQL file to use unversioned library name..." + current_version="${version}" + main_sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" + if [ -f "$main_sql_file" ]; then + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$main_sql_file" + echo "Modified $main_sql_file" + else + echo "Warning: $main_sql_file not found" + fi - echo "Creating wrappers.so symlinks to support pg_upgrade..." - lib_file="$out/lib/wrappers-$current_version.so" + echo "Creating and modifying SQL files for previous versions..." + + if [ -f "$main_sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$main_sql_file" "$new_file" + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$new_file" + echo "Modified $new_file" + fi + done < git_tags.txt + else + echo "Warning: $main_sql_file not found" + fi - if [ -f "$lib_file" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/lib/wrappers-$previous_version.so" - echo "Creating $new_file" - ln -s "$lib_file" "$new_file" - fi - done < git_tags.txt - else - echo "Warning: $lib_file not found" - fi + echo "Creating wrappers.so symlinks to support pg_upgrade..." + lib_file="$out/lib/wrappers-$current_version.so" - rm git_tags.txt - ''; + if [ -f "$lib_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/lib/wrappers-$previous_version.so" + echo "Creating $new_file" + ln -s "$lib_file" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $lib_file not found" + fi + + rm git_tags.txt + mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so + + echo "Contents of updated wrappers.control:" + cat "$out/share/postgresql/extension/wrappers.control" + + echo "List of generated SQL files:" + ls -l $out/share/postgresql/extension/wrappers--*.sql +''; meta = with lib; { description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; From 94b89a379fec89a398122584c9d61044c86df2b8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 4 Oct 2024 13:57:04 -0400 Subject: [PATCH 13/21] chore: wrappers lib rename --- nix/ext/wrappers/default.nix | 84 +++++++++++++++--------------------- 1 file changed, 35 insertions(+), 49 deletions(-) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 80df2425f..cc516d655 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -63,57 +63,43 @@ buildPgrxExtension_0_11_3 rec { echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt ''; -postInstall = '' - echo "Modifying main SQL file to use unversioned library name..." - current_version="${version}" - main_sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" - if [ -f "$main_sql_file" ]; then - sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$main_sql_file" - echo "Modified $main_sql_file" - else - echo "Warning: $main_sql_file not found" - fi + postInstall = '' + echo "Creating SQL files for previous versions..." + current_version="${version}" + sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" + + if [ -f "$sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$sql_file" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $sql_file not found" + fi - echo "Creating and modifying SQL files for previous versions..." - - if [ -f "$main_sql_file" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" - echo "Creating $new_file" - cp "$main_sql_file" "$new_file" - sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$new_file" - echo "Modified $new_file" - fi - done < git_tags.txt - else - echo "Warning: $main_sql_file not found" - fi + echo "Creating wrappers.so symlinks to support pg_upgrade..." + lib_file="$out/lib/wrappers-$current_version.so" - echo "Creating wrappers.so symlinks to support pg_upgrade..." - lib_file="$out/lib/wrappers-$current_version.so" - - if [ -f "$lib_file" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/lib/wrappers-$previous_version.so" - echo "Creating $new_file" - ln -s "$lib_file" "$new_file" - fi - done < git_tags.txt - else - echo "Warning: $lib_file not found" - fi - - rm git_tags.txt - mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so - - echo "Contents of updated wrappers.control:" - cat "$out/share/postgresql/extension/wrappers.control" - - echo "List of generated SQL files:" - ls -l $out/share/postgresql/extension/wrappers--*.sql -''; + if [ -f "$lib_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/lib/wrappers-$previous_version.so" + echo "Creating $new_file" + ln -s "$lib_file" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $lib_file not found" + fi + + rm git_tags.txt + mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so + echo "Uncommenting module_pathname in wrappers.control..." + sed -i 's/^#module_pathname/module_pathname/' $out/share/postgresql/extension/wrappers.control + ''; meta = with lib; { description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; From fa488e9cb6e8511b75c17ff5b016c95634b26f3d Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 14 Oct 2024 15:43:48 +0300 Subject: [PATCH 14/21] chore: bump pg version --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index dd5d407f5..b34ad7792 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.129" +postgres-version = "15.6.1.129-rc1" From 697fc6db3f27f745bf6ea5272cd407b02a9acf2d Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 14 Oct 2024 15:48:54 +0300 Subject: [PATCH 15/21] fix: paths --- nix/ext/wrappers/default.nix | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index cc516d655..306300dc1 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -79,24 +79,22 @@ buildPgrxExtension_0_11_3 rec { else echo "Warning: $sql_file not found" fi + mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so echo "Creating wrappers.so symlinks to support pg_upgrade..." - lib_file="$out/lib/wrappers-$current_version.so" - - if [ -f "$lib_file" ]; then + if [ -f "$out/lib/wrappers.so" ]; then while read -r previous_version; do if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then new_file="$out/lib/wrappers-$previous_version.so" echo "Creating $new_file" - ln -s "$lib_file" "$new_file" + ln -s "$out/lib/wrappers.so" "$new_file" fi done < git_tags.txt else - echo "Warning: $lib_file not found" + echo "Warning: $out/lib/wrappers.so not found" fi rm git_tags.txt - mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so echo "Uncommenting module_pathname in wrappers.control..." sed -i 's/^#module_pathname/module_pathname/' $out/share/postgresql/extension/wrappers.control ''; From ed36aa1d002a337a3582a443af51beee0b2b5eb9 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 14 Oct 2024 17:00:22 +0300 Subject: [PATCH 16/21] fix: wrappers; plv8; backwards compat --- nix/ext/plv8.nix | 1 + nix/ext/wrappers/default.nix | 79 ++++++++++++++++++++---------------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index c2591eeba..338bba62e 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -104,6 +104,7 @@ stdenv.mkDerivation (finalAttrs: { mv "$out/nix/store"/*/* "$out" rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" mv "$out/lib/plv8-${finalAttrs.version}.so" "$out/lib/plv8.so" + ln -s "$out/lib/plv8.so" "$out/lib/plv8-${finalAttrs.version}.so" sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plv8.control" sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plcoffee.control" sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plls.control" diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 306300dc1..808946d3a 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -63,41 +63,52 @@ buildPgrxExtension_0_11_3 rec { echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt ''; - postInstall = '' - echo "Creating SQL files for previous versions..." - current_version="${version}" - sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" - - if [ -f "$sql_file" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" - echo "Creating $new_file" - cp "$sql_file" "$new_file" - fi - done < git_tags.txt - else - echo "Warning: $sql_file not found" - fi - mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so + postInstall = '' + echo "Modifying main SQL file to use unversioned library name..." + current_version="${version}" + main_sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" + if [ -f "$main_sql_file" ]; then + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$main_sql_file" + echo "Modified $main_sql_file" + else + echo "Warning: $main_sql_file not found" + fi + echo "Creating and modifying SQL files for previous versions..." + + if [ -f "$main_sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$main_sql_file" "$new_file" + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$new_file" + echo "Modified $new_file" + fi + done < git_tags.txt + else + echo "Warning: $main_sql_file not found" + fi + mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so + + echo "Creating wrappers.so symlinks to support pg_upgrade..." + if [ -f "$out/lib/wrappers.so" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/lib/wrappers-$previous_version.so" + echo "Creating $new_file" + ln -s "$out/lib/wrappers.so" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $out/lib/wrappers.so not found" + fi - echo "Creating wrappers.so symlinks to support pg_upgrade..." - if [ -f "$out/lib/wrappers.so" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/lib/wrappers-$previous_version.so" - echo "Creating $new_file" - ln -s "$out/lib/wrappers.so" "$new_file" - fi - done < git_tags.txt - else - echo "Warning: $out/lib/wrappers.so not found" - fi - - rm git_tags.txt - echo "Uncommenting module_pathname in wrappers.control..." - sed -i 's/^#module_pathname/module_pathname/' $out/share/postgresql/extension/wrappers.control - ''; + rm git_tags.txt + echo "Contents of updated wrappers.control:" + cat "$out/share/postgresql/extension/wrappers.control" + echo "List of generated SQL files:" + ls -l $out/share/postgresql/extension/wrappers--*.sql + ''; meta = with lib; { description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; From 7cb3e6eac892f0db74d68de2b0b4fe595ba6be9d Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 14 Oct 2024 18:46:58 +0300 Subject: [PATCH 17/21] chore: stop patching wrappers --- .../pg_upgrade_scripts/initiate.sh | 54 ------------------- 1 file changed, 54 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 235b4f4c1..1db4aaa0f 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -181,58 +181,6 @@ EOF done } -function patch_wrappers { - local IS_NIX_UPGRADE=$1 - - WRAPPERS_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'wrappers');") - if [ "$WRAPPERS_ENABLED" = "f" ]; then - echo "Wrappers extension not enabled. Skipping." - return - fi - - # This is a workaround for older versions of wrappers which don't have the expected - # naming scheme, containing the version in their library's file name - # e.g. wrappers-0.1.16.so, rather than wrappers.so - # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's - # library directory, so we're making sure the new version has the expected (old version's) - # file name. - # After the upgrade completes, the new version's library file is used. - # i.e. - # - old version: wrappers-0.1.16.so - # - new version: wrappers-0.1.18.so - # - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so - if [ "$IS_NIX_UPGRADE" = "true" ]; then - if [ -d "$PGLIBOLD" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - OLD_LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - - find /nix/store/ -name "wrappers*so" -print0 | while read -r -d $'\0' WRAPPERS_LIB_PATH; do - if [ -f "$WRAPPERS_LIB_PATH" ]; then - WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then - echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" || true - fi - fi - done - fi - else - if [ -d "$PGLIBOLD" ]; then - WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit) - if [ -f "$WRAPPERS_LIB_PATH" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then - LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then - echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" || true - fi - fi - fi - fi - fi -} - function initiate_upgrade { mkdir -p "$MOUNT_POINT" SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/") @@ -409,8 +357,6 @@ function initiate_upgrade { export LD_LIBRARY_PATH="${PGLIBNEW}" fi - patch_wrappers "$IS_NIX_UPGRADE" - echo "9. Creating new data directory, initializing database" chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" From 55397c3c0786ac2f380513374e4040139e413657 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 14 Oct 2024 19:31:08 +0300 Subject: [PATCH 18/21] chore: bump pg version --- .github/workflows/publish-nix-pgupgrade-bin-flake-version.yml | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml index 13618f721..5b985f4be 100644 --- a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml +++ b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml @@ -57,7 +57,7 @@ jobs: publish-prod: runs-on: ubuntu-latest - if: github.ref_name == 'develop' || contains( github.ref, 'release' ) || contains( github.ref, 'pcnc/wrappers-15.6' ) + if: github.ref_name == 'develop' || contains( github.ref, 'release' ) steps: - name: Checkout Repo diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index b34ad7792..e5e2f4712 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.129-rc1" +postgres-version = "15.6.1.130" From 7fa1ba415059a63c92479b01561869d6127ac610 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Mon, 14 Oct 2024 19:33:22 +0300 Subject: [PATCH 19/21] chore: remove unneeded variable --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 1db4aaa0f..ed9a7ed9b 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -47,7 +47,6 @@ SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" PGBINOLD="/usr/lib/postgresql/bin" -PGLIBOLD="/usr/lib/postgresql/lib" PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" NIX_INSTALLER_PATH="/tmp/persistent/nix-installer" From 2537aa00804e14d4b9b9d08eef1167fc3d92ea22 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 15 Oct 2024 14:55:41 +0300 Subject: [PATCH 20/21] fix: remove timescaledb 2.9.1 control file --- nix/ext/timescaledb-2.9.1.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/ext/timescaledb-2.9.1.nix b/nix/ext/timescaledb-2.9.1.nix index 643256182..ad955e86f 100644 --- a/nix/ext/timescaledb-2.9.1.nix +++ b/nix/ext/timescaledb-2.9.1.nix @@ -36,7 +36,7 @@ stdenv.mkDerivation rec { # we have no need for the timescaledb.so or control file postInstall = '' rm $out/lib/timescaledb.so - mv $out/share/postgresql/extension/timescaledb.control $out/share/postgresql/extension/timescaledb-2.9.1.control + rm $out/share/postgresql/extension/timescaledb.control ''; meta = with lib; { From 1fac4f2252f80bc1667281d4ba9ce3c06e1342f6 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 15 Oct 2024 18:43:02 +0300 Subject: [PATCH 21/21] chore: fix interface --- nix/tests/expected/extensions_sql_interface.out | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index e240345a0..ab43f54e6 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -20,11 +20,10 @@ where installed_version is null order by name asc; - name -------------------- + name +--------- pg_cron - timescaledb-2.9.1 -(2 rows) +(1 row) /*