feat: multiple versions for the timescaledb-apache extension

ansible/vars.yml

@@ -9,9 +9,9 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.5.1.026-orioledb"
-  postgres17: "17.6.1.005"
-  postgres15: "15.14.1.005"
+  postgresorioledb-17: "17.5.1.027-orioledb"
+  postgres17: "17.6.1.006"
+  postgres15: "15.14.1.006"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"

nix/ext/tests/lib.py

@@ -147,7 +147,7 @@ def check_switch_extension_with_background_worker(
             )
         last_version = available_versions[-1]
         assert ext_version.endswith(
-            f"{self.extension_name}-{last_version}.so"
+            f"{last_version}.so"
         ), f"Expected {self.extension_name} version {last_version}, but found {ext_version}"
 
         # Switch to the first version
@@ -157,13 +157,13 @@ def check_switch_extension_with_background_worker(
         # Check that we are using the first version now
         ext_version = self.vm.succeed(f"readlink -f {extension_lib_path}").strip()
         assert ext_version.endswith(
-            f"{self.extension_name}-{first_version}.so"
+            f"{first_version}.so"
         ), f"Expected {self.extension_name} version {first_version}, but found {ext_version}"
 
         # Switch to the first version
         self.vm.succeed(f"switch_{self.extension_name}_version {last_version}")
         # Check that we are using the last version now
         ext_version = self.vm.succeed(f"readlink -f {extension_lib_path}").strip()
         assert ext_version.endswith(
-            f"{self.extension_name}-{last_version}.so"
+            f"{last_version}.so"
         ), f"Expected {self.extension_name} version {last_version}, but found {ext_version}"

nix/ext/tests/timescaledb.nix

@@ -0,0 +1,78 @@
+{ self, pkgs }:
+let
+  pname = "timescaledb";
+  inherit (pkgs) lib;
+  installedExtension =
+    postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all";
+  versions = (installedExtension "15").versions;
+  postgresqlWithExtension =
+    postgresql:
+    let
+      majorVersion = lib.versions.major postgresql.version;
+      pkg = pkgs.buildEnv {
+        name = "postgresql-${majorVersion}-${pname}";
+        paths = [
+          postgresql
+          postgresql.lib
+          (installedExtension majorVersion)
+        ];
+        passthru = {
+          inherit (postgresql) version psqlSchema;
+          lib = pkg;
+          withPackages = _: pkg;
+        };
+        nativeBuildInputs = [ pkgs.makeWrapper ];
+        pathsToLink = [
+          "/"
+          "/bin"
+          "/lib"
+        ];
+        postBuild = ''
+          wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib
+          wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib
+          wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib
+        '';
+      };
+    in
+    pkg;
+  psql_15 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15;
+in
+self.inputs.nixpkgs.lib.nixos.runTest {
+  name = "timescaledb";
+  hostPkgs = pkgs;
+  nodes.server =
+    { ... }:
+    {
+      services.postgresql = {
+        enable = true;
+        package = (postgresqlWithExtension psql_15);
+        settings = {
+          shared_preload_libraries = "timescaledb";
+        };
+      };
+    };
+  testScript =
+    { ... }:
+    ''
+      ${builtins.readFile ./lib.py}
+
+      start_all()
+
+      server.wait_for_unit("multi-user.target")
+      server.wait_for_unit("postgresql.service")
+
+      versions = {
+        "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') versions)}],
+      }
+      extension_name = "${pname}"
+      support_upgrade = True
+
+      test = PostgresExtensionTest(server, extension_name, versions, support_upgrade)
+
+      with subtest("Check upgrade path with postgresql 15"):
+        test.check_upgrade_path("15")
+
+      with subtest("Test switch_${pname}_version"):
+        test.check_switch_extension_with_background_worker(Path("${psql_15}/lib/${pname}.so"), "15")
+    '';
+}

nix/ext/timescaledb.nix

@@ -6,53 +6,147 @@
   postgresql,
   openssl,
   libkrb5,
+  buildEnv,
+  makeWrapper,
+  switch-ext-version,
+  coreutils,
+  writeShellApplication,
 }:
 
-stdenv.mkDerivation rec {
-  pname = "timescaledb-apache";
-  version = "2.16.1";
+let
+  pname = "timescaledb";
+  build =
+    version: hash: _revision:
+    stdenv.mkDerivation rec {
+      inherit pname version;
 
-  nativeBuildInputs = [ cmake ];
-  buildInputs = [
-    postgresql
-    openssl
-    libkrb5
-  ];
+      nativeBuildInputs = [ cmake ];
+      buildInputs = [
+        postgresql
+        openssl
+        libkrb5
+      ];
+
+      src = fetchFromGitHub {
+        owner = "timescale";
+        repo = "timescaledb";
+        rev = version;
+        inherit hash;
+      };
+
+      cmakeFlags = [
+        "-DSEND_TELEMETRY_DEFAULT=OFF"
+        "-DREGRESS_CHECKS=OFF"
+        "-DTAP_CHECKS=OFF"
+        "-DAPACHE_ONLY=1"
+      ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ];
+
+      postPatch = ''
+        for x in CMakeLists.txt sql/CMakeLists.txt; do
+          if [ -f "$x" ]; then
+            substituteInPlace "$x" \
+              --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\""
+          fi
+        done
+
+        for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do
+          if [ -f "$x" ]; then
+            substituteInPlace "$x" \
+              --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\""
+          fi
+        done
+      '';
+
+      installPhase = ''
+        # Run cmake install first
+        cmake --install . --prefix=$out
+
+        # TimescaleDB creates two libraries:
+        # 1. timescaledb.so (loader) -> rename to timescaledb-<version>-loader.so
+        # 2. timescaledb-<version>.so (actual extension) -> keep as is
+
+        # Rename the loader library to be version-specific
+        if [ -f $out/lib/timescaledb${postgresql.dlSuffix} ]; then
+          mv $out/lib/timescaledb${postgresql.dlSuffix} $out/lib/timescaledb-loader-${version}${postgresql.dlSuffix}
+        fi
 
-  src = fetchFromGitHub {
-    owner = "timescale";
-    repo = "timescaledb";
-    rev = version;
-    hash = "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w=";
+        # The versioned library (timescaledb-VERSION.so) is already correctly named
+
+        # Create versioned control file with default_version removed and module_pathname pointing to symlink
+        if [ -f $out/share/postgresql/extension/timescaledb.control ]; then
+          sed -e "/^default_version =/d" \
+              -e "s|^module_pathname = .*|module_pathname = '\$libdir/timescaledb'|" \
+            $out/share/postgresql/extension/timescaledb.control > $out/share/postgresql/extension/timescaledb--${version}.control
+          rm $out/share/postgresql/extension/timescaledb.control
+        fi
+      '';
+
+      meta = with lib; {
+        description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space";
+        homepage = "https://www.timescale.com/";
+        changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md";
+        license = licenses.asl20;
+        inherit (postgresql.meta) platforms;
+      };
+    };
+
+  allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).timescaledb;
+  supportedVersions = lib.filterAttrs (
+    _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
+  ) allVersions;
+  versions = lib.naturalSort (lib.attrNames supportedVersions);
+  latestVersion = lib.last versions;
+  numberOfVersions = builtins.length versions;
+  packages = builtins.attrValues (
+    lib.mapAttrs (name: value: build name value.hash (value.revision or name)) supportedVersions
+  );
+  switch-timescaledb-loader = writeShellApplication {
+    name = "switch_timescaledb_loader";
+    runtimeInputs = [ coreutils ];
+    text = ''
+      EXT_LOADER_TO_USE="$EXT_WRAPPER_LIB/$EXT_NAME-loader-$VERSION${postgresql.dlSuffix}"
+      if [ -f "$EXT_LOADER_TO_USE" ]; then
+        ln -sfnv "$EXT_LOADER_TO_USE" "$EXT_WRAPPER_LIB/$EXT_NAME${postgresql.dlSuffix}"
+      fi
+    '';
   };
+in
+buildEnv {
+  name = pname;
+  paths = packages;
+  nativeBuildInputs = [ makeWrapper ];
+  postBuild = ''
+    {
+      echo "default_version = '${latestVersion}'"
+      cat $out/share/postgresql/extension/${pname}--${latestVersion}.control
+    } > $out/share/postgresql/extension/${pname}.control
+
+    # Create symlink from the latest versioned loader to timescaledb.so
+    ln -sfn ${pname}-loader-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
 
-  cmakeFlags = [
-    "-DSEND_TELEMETRY_DEFAULT=OFF"
-    "-DREGRESS_CHECKS=OFF"
-    "-DTAP_CHECKS=OFF"
-    "-DAPACHE_ONLY=1"
-  ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ];
-
-  # Fix the install phase which tries to install into the pgsql extension dir,
-  # and cannot be manually overridden. This is rather fragile but works OK.
-  postPatch = ''
-    for x in CMakeLists.txt sql/CMakeLists.txt; do
-      substituteInPlace "$x" \
-        --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\""
-    done
-
-    for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do
-      substituteInPlace "$x" \
-        --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\""
-    done
+    # The versioned extension libraries (timescaledb-VERSION.so) are already in place
+
+    # checks - we should have loader files and versioned extension files
+    (set -x
+       test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" -gt 0
+    )
+    makeWrapper ${lib.getExe switch-ext-version} $out/bin/switch_timescaledb_version \
+      --prefix EXT_WRAPPER : "$out" --prefix EXT_NAME : "${pname}" --prefix EXTRA_STEPS : ${lib.getExe switch-timescaledb-loader}
   '';
 
-  meta = with lib; {
-    description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space";
-    homepage = "https://www.timescale.com/";
-    changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md";
-    platforms = postgresql.meta.platforms;
-    license = licenses.asl20;
-    broken = versionOlder postgresql.version "13";
+  pathsToLink = [
+    "/lib"
+    "/share/postgresql/extension"
+  ];
+
+  passthru = {
+    inherit versions numberOfVersions switch-ext-version;
+    pname = "${pname}-all";
+    hasBackgroundWorker = true;
+    defaultSettings = {
+      shared_preload_libraries = [ "timescaledb" ];
+    };
+    version =
+      "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
   };
 }

nix/ext/versions.json

@@ -130,6 +130,20 @@
       ],
       "hash": "sha256-Cpi2iASi1QJoED0Qs1dANqg/BNZTsz5S+pw8iYyW03Y="
     }
+  },
+    "timescaledb": {
+    "2.9.1": {
+      "postgresql": [
+        "15"
+      ],
+      "hash": "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="
+    },
+    "2.16.1": {
+      "postgresql": [
+        "15"
+      ],
+      "hash": "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w="
+    }
   },
   "vector": {
     "0.4.0": {

nix/packages/postgres.nix

@@ -19,7 +19,6 @@
       ourExtensions = [
         ../ext/rum.nix
         ../ext/timescaledb.nix
-        ../ext/timescaledb-2.9.1.nix
         ../ext/pgroonga.nix
         ../ext/index_advisor.nix
         ../ext/wal2json.nix