Skip to content

Commit d9a8841

Browse files
committed
Merge remote-tracking branch 'origin' into INDATA-152
* origin: feat: Ensure postgres conf.d directory is set up (#1961) Multi version ext/pg partman (#1854)
2 parents c198de4 + 269f137 commit d9a8841

20 files changed

+672
-73
lines changed

Dockerfile-15

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ COPY . /nixpg
113113

114114
WORKDIR /nixpg
115115

116-
RUN nix profile install .#psql_15/bin
116+
RUN nix profile add path:.#psql_15/bin
117117

118118
RUN nix store gc
119119

@@ -167,7 +167,7 @@ FROM base as groonga
167167

168168
WORKDIR /nixpg
169169

170-
RUN nix profile install .#supabase-groonga && \
170+
RUN nix profile add path:.#supabase-groonga && \
171171
mkdir -p /tmp/groonga-plugins && \
172172
cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/
173173

@@ -209,7 +209,7 @@ COPY --from=groonga /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins
209209
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf
210210
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf
211211
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf
212-
COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/
212+
COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/conf.d
213213
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf
214214
COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
215215
COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
@@ -227,7 +227,6 @@ RUN \
227227
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
228228
echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
229229
usermod -aG postgres wal-g && \
230-
mkdir -p /etc/postgresql-custom/conf.d && \
231230
chown -R postgres:postgres /etc/postgresql-custom
232231

233232
# # Include schema migrations

Dockerfile-17

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ COPY . /nixpg
116116

117117
WORKDIR /nixpg
118118

119-
RUN nix profile install .#psql_17/bin
119+
RUN nix profile add path:.#psql_17/bin
120120

121121
RUN nix store gc
122122

@@ -171,7 +171,7 @@ FROM base as groonga
171171

172172
WORKDIR /nixpg
173173

174-
RUN nix profile install .#supabase-groonga && \
174+
RUN nix profile add path:.#supabase-groonga && \
175175
mkdir -p /tmp/groonga-plugins && \
176176
cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/
177177

@@ -213,7 +213,7 @@ COPY --from=groonga /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins
213213
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf
214214
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf
215215
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf
216-
COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/
216+
COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/conf.d
217217
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf
218218
COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
219219
COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
@@ -231,7 +231,6 @@ RUN \
231231
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
232232
echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
233233
usermod -aG postgres wal-g && \
234-
mkdir -p /etc/postgresql-custom/conf.d && \
235234
chown -R postgres:postgres /etc/postgresql-custom
236235

237236
# Remove items from postgresql.conf

Dockerfile-orioledb-17

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ COPY . /nixpg
116116

117117
WORKDIR /nixpg
118118

119-
RUN nix profile install .#psql_orioledb-17/bin
119+
RUN nix profile add path:.#psql_orioledb-17/bin
120120

121121
RUN nix store gc
122122

@@ -171,7 +171,7 @@ FROM base as groonga
171171

172172
WORKDIR /nixpg
173173

174-
RUN nix profile install .#supabase-groonga && \
174+
RUN nix profile add path:.#supabase-groonga && \
175175
mkdir -p /tmp/groonga-plugins && \
176176
cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/
177177

@@ -213,7 +213,7 @@ COPY --from=groonga /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins
213213
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf
214214
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf
215215
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf
216-
COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/
216+
COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/conf.d
217217
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf
218218
COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
219219
COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
@@ -231,7 +231,6 @@ RUN \
231231
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
232232
echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
233233
usermod -aG postgres wal-g && \
234-
mkdir -p /etc/postgresql-custom/conf.d && \
235234
chown -R postgres:postgres /etc/postgresql-custom
236235

237236
# Remove items from postgresql.conf

ansible/vars.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ postgres_major:
1010

1111
# Full version strings for each major version
1212
postgres_release:
13-
postgresorioledb-17: "17.6.0.017-orioledb"
14-
postgres17: "17.6.1.060"
15-
postgres15: "15.14.1.060"
13+
postgresorioledb-17: "17.6.0.018-orioledb"
14+
postgres17: "17.6.1.061"
15+
postgres15: "15.14.1.061"
1616

1717
# Non Postgres Extensions
1818
pgbouncer_release: 1.19.0
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
BEGIN;
2+
create schema if not exists "partman";
3+
create extension if not exists pg_partman with schema "partman";
4+
ROLLBACK;

migrations/tests/extensions/test.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,4 @@
2828
\ir 27-pg_repack.sql
2929
\ir 28-pgvector.sql
3030
\ir 29-pg_tle.sql
31+
\ir 30-pg_partman.sql

nix/ext/pg_partman.nix

Lines changed: 97 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,113 @@
11
{
2+
pkgs,
23
lib,
34
stdenv,
45
fetchFromGitHub,
56
postgresql,
7+
makeWrapper,
8+
switch-ext-version,
69
}:
710

8-
stdenv.mkDerivation rec {
11+
let
912
pname = "pg_partman";
10-
version = "5.1.0";
13+
libName = "pg_partman_bgw";
14+
build =
15+
version: hash:
16+
stdenv.mkDerivation rec {
17+
inherit pname version;
1118

12-
buildInputs = [ postgresql ];
19+
buildInputs = [ postgresql ];
20+
21+
src = fetchFromGitHub {
22+
owner = "pgpartman";
23+
repo = pname;
24+
rev = "refs/tags/v${version}";
25+
inherit hash;
26+
};
27+
28+
installPhase = ''
29+
mkdir -p $out/{lib,share/postgresql/extension}
30+
31+
# Install versioned library
32+
install -Dm755 src/${libName}${postgresql.dlSuffix} $out/lib/${libName}-${version}${postgresql.dlSuffix}
33+
34+
# Only install SQL files for the latest version
35+
if [[ "${version}" == "${latestVersion}" ]]; then
36+
# Install all SQL files from sql/ directory
37+
cp -r sql/* $out/share/postgresql/extension/
38+
39+
# Install upgrade scripts
40+
cp updates/* $out/share/postgresql/extension/
41+
fi
42+
43+
# Create versioned control file with modified module path
44+
sed -e "/^default_version =/d" \
45+
-e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}'|" \
46+
${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control
47+
'';
48+
49+
meta = with lib; {
50+
description = "Partition management extension for PostgreSQL";
51+
homepage = "https://github.com/pgpartman/pg_partman";
52+
changelog = "https://github.com/pgpartman/pg_partman/blob/v${version}/CHANGELOG.md";
53+
platforms = postgresql.meta.platforms;
54+
license = licenses.postgresql;
55+
};
56+
};
57+
allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).pg_partman;
58+
supportedVersions = lib.filterAttrs (
59+
_: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
60+
) allVersions;
61+
versions = lib.naturalSort (lib.attrNames supportedVersions);
62+
latestVersion = lib.last versions;
63+
numberOfVersions = builtins.length versions;
64+
packages = builtins.attrValues (
65+
lib.mapAttrs (name: value: build name value.hash) supportedVersions
66+
);
67+
in
68+
pkgs.buildEnv {
69+
name = pname;
70+
paths = packages;
71+
nativeBuildInputs = [ makeWrapper ];
72+
73+
pathsToLink = [
74+
"/lib"
75+
"/share/postgresql/extension"
76+
];
77+
78+
postBuild = ''
79+
{
80+
echo "default_version = '${latestVersion}'"
81+
cat $out/share/postgresql/extension/${pname}--${latestVersion}.control
82+
} > $out/share/postgresql/extension/${pname}.control
83+
ln -sfn ${libName}-${latestVersion}${postgresql.dlSuffix} $out/lib/${libName}${postgresql.dlSuffix}
1384
14-
src = fetchFromGitHub {
15-
owner = "pgpartman";
16-
repo = pname;
17-
rev = "refs/tags/v${version}";
18-
sha256 = "sha256-GrVOJ5ywZMyqyDroYDLdKkXDdIJSDGhDfveO/ZvrmYs=";
19-
};
2085
21-
installPhase = ''
22-
mkdir -p $out/{lib,share/postgresql/extension}
86+
# checks
87+
(set -x
88+
test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${
89+
toString (numberOfVersions + 1)
90+
}"
91+
)
2392
24-
cp src/*${postgresql.dlSuffix} $out/lib
25-
cp updates/* $out/share/postgresql/extension
26-
cp -r sql/* $out/share/postgresql/extension
27-
cp *.control $out/share/postgresql/extension
93+
makeWrapper ${lib.getExe switch-ext-version} $out/bin/switch_pg_partman_version \
94+
--prefix EXT_WRAPPER : "$out" --prefix EXT_NAME : "${pname}" --prefix LIB_NAME : "${libName}"
2895
'';
2996

30-
meta = with lib; {
31-
description = "Partition management extension for PostgreSQL";
32-
homepage = "https://github.com/pgpartman/pg_partman";
33-
changelog = "https://github.com/pgpartman/pg_partman/blob/v${version}/CHANGELOG.md";
34-
platforms = postgresql.meta.platforms;
35-
license = licenses.postgresql;
36-
broken = versionOlder postgresql.version "14";
97+
passthru = {
98+
inherit
99+
versions
100+
numberOfVersions
101+
switch-ext-version
102+
libName
103+
;
104+
pname = "${pname}-all";
105+
hasBackgroundWorker = true;
106+
defaultSchema = "partman";
107+
defaultSettings = {
108+
shared_preload_libraries = [ libName ];
109+
};
110+
version =
111+
"multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
37112
};
38113
}

nix/ext/tests/default.nix

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,16 @@ let
146146
"17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}],
147147
}
148148
extension_name = "${pname}"
149+
support_upgrade = True
149150
pg17_configuration = "${pg17-configuration}"
150151
ext_has_background_worker = ${
151152
if (installedExtension "15") ? hasBackgroundWorker then "True" else "False"
152153
}
153154
sql_test_directory = Path("${../../tests}")
154155
pg_regress_test_name = "${(installedExtension "15").pgRegressTestName or pname}"
156+
ext_schema = "${(installedExtension "15").defaultSchema or "public"}"
157+
lib_name = "${(installedExtension "15").libName or pname}"
158+
print(f"Running tests for extension: {lib_name}")
155159
156160
${builtins.readFile ./lib.py}
157161
@@ -160,7 +164,8 @@ let
160164
server.wait_for_unit("multi-user.target")
161165
server.wait_for_unit("postgresql.service")
162166
163-
test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory)
167+
test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade, ext_schema)
168+
test.create_schema()
164169
165170
with subtest("Check upgrade path with postgresql 15"):
166171
test.check_upgrade_path("15")
@@ -174,7 +179,7 @@ let
174179
175180
if ext_has_background_worker:
176181
with subtest("Test switch_${pname}_version"):
177-
test.check_switch_extension_with_background_worker(Path("${psql_15}/lib/${pname}.so"), "15")
182+
test.check_switch_extension_with_background_worker(Path(f"${psql_15}/lib/{lib_name}.so"), "15")
178183
179184
with subtest("Check pg_regress with postgresql 15 after installing the last version"):
180185
test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name)
@@ -224,6 +229,7 @@ builtins.listToAttrs (
224229
"pg_stat_monitor"
225230
"pg_tle"
226231
"pgaudit"
232+
"pg_partman"
227233
"vector"
228234
"wal2json"
229235
"wrappers"

nix/ext/tests/lib.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ def __init__(
2020
versions: Versions,
2121
sql_test_dir: Path,
2222
support_upgrade: bool = True,
23+
schema: str = "public",
2324
):
2425
"""Initialize the PostgreSQL extension test framework.
2526
@@ -35,6 +36,10 @@ def __init__(
3536
self.versions = versions
3637
self.support_upgrade = support_upgrade
3738
self.sql_test_dir = sql_test_dir
39+
self.schema = schema
40+
41+
def create_schema(self):
42+
self.run_sql(f"CREATE SCHEMA IF NOT EXISTS {self.schema};")
3843

3944
def run_sql(self, query: str) -> str:
4045
return self.vm.succeed(
@@ -50,8 +55,12 @@ def drop_extension(self):
5055
self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name};")
5156

5257
def install_extension(self, version: str):
58+
if self.schema != "public":
59+
ext_schema = f"SCHEMA {self.schema} "
60+
else:
61+
ext_schema = ""
5362
self.run_sql(
54-
f"""CREATE EXTENSION {self.extension_name} WITH VERSION '{version}' CASCADE;"""
63+
f"""CREATE EXTENSION {self.extension_name} WITH {ext_schema}VERSION '{version}' CASCADE;"""
5564
)
5665
# Verify version was installed correctly
5766
self.assert_version_matches(version)

nix/ext/versions.json

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -353,21 +353,31 @@
353353
"hash": "sha256-9St/ESPiFq2NiPKqbwHLwkIyATKUkOGxFcUrWgT+Iqo="
354354
}
355355
},
356-
"pgmq": {
357-
"1.4.4": {
358-
"postgresql": [
359-
"15"
360-
],
361-
"hash": "sha256-z+8/BqIlHwlMnuIzMz6eylmYbSmhtsNt7TJf/CxbdVw="
362-
},
363-
"1.5.1": {
356+
"pg_partman": {
357+
"5.3.1": {
364358
"postgresql": [
365359
"15",
366-
"17"
360+
"17",
361+
"orioledb-17"
367362
],
368-
"hash": "sha256-IU+i6ONPwtgsFKdzya6E+222ualR66gkbb0lDr+7Rb8="
363+
"hash": "sha256-LK+z/0puUuNLz2ah/SDaxxyLS/On/cmCBhKNwTLa7Pk="
369364
}
370365
},
366+
"pgmq": {
367+
"1.4.4": {
368+
"postgresql": [
369+
"15"
370+
],
371+
"hash": "sha256-z+8/BqIlHwlMnuIzMz6eylmYbSmhtsNt7TJf/CxbdVw="
372+
},
373+
"1.5.1": {
374+
"postgresql": [
375+
"15",
376+
"17"
377+
],
378+
"hash": "sha256-IU+i6ONPwtgsFKdzya6E+222ualR66gkbb0lDr+7Rb8="
379+
}
380+
},
371381
"pgroonga": {
372382
"3.0.7": {
373383
"postgresql": [

0 commit comments

Comments
 (0)