ci(postgresql): stop installing system PG before tests so issue #112 path is exercised

.github/workflows/integration_tests.yml

@@ -520,22 +520,32 @@ jobs:
           rm -rf "$HOME/sandboxes"/*
           echo '{}' > "$HOME/.dbdeployer/sandboxes.json"
 
-      - name: Install PostgreSQL for ts tests
+      - name: Download and unpack PostgreSQL debs for ts tests (no system install)
         run: |
           # Add PostgreSQL apt repo (PGDG) — required for postgresql-16 on ubuntu-22.04
           sudo apt-get install -y curl ca-certificates
           sudo install -d /usr/share/postgresql-common/pgdg
           sudo curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
           echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list
           sudo apt-get update
-          sudo apt-get install -y postgresql-16 postgresql-client-16
-          sudo systemctl stop postgresql || true
+
+          # Install libpq5 only — psql links against it dynamically. We do
+          # NOT install postgresql-16 itself, since that would put
+          # /usr/share/postgresql/16/ on the runner and mask issue #112.
+          sudo apt-get install -y libpq5
+
+          # Documented user flow: download the debs, then unpack via dbdeployer.
           export HOME="$GITHUB_WORKSPACE/home"
-          PG_FULL=$(dpkg -s postgresql-16 | grep '^Version:' | sed 's/Version: //' | cut -d'-' -f1)
-          mkdir -p "$HOME/opt/postgresql/${PG_FULL}/"{bin,lib,share}
-          cp -a /usr/lib/postgresql/16/bin/. "$HOME/opt/postgresql/${PG_FULL}/bin/"
-          cp -a /usr/lib/postgresql/16/lib/. "$HOME/opt/postgresql/${PG_FULL}/lib/"
-          cp -a /usr/share/postgresql/16/. "$HOME/opt/postgresql/${PG_FULL}/share/"
+          mkdir -p "$HOME"
+          apt-get download postgresql-16 postgresql-client-16
+          ./dbdeployer unpack --provider=postgresql \
+            postgresql-16_*.deb postgresql-client-16_*.deb
+
+          # Sanity: confirm no system PG install snuck in via dependencies.
+          if [ -d "/usr/share/postgresql/16" ]; then
+            echo "FAIL: /usr/share/postgresql/16/ exists — system PG install is masking issue #112"
+            exit 1
+          fi
 
       - name: Run ts PostgreSQL tests
         env:
@@ -586,20 +596,32 @@ jobs:
       - name: Build dbdeployer
         run: go build -o dbdeployer .
 
-      - name: Install PostgreSQL and set up binaries
-        run: |
-          # Install PostgreSQL to get properly configured binaries
-          sudo apt-get install -y postgresql-${PG_VERSION} postgresql-client-${PG_VERSION}
-          # Stop the system service — we'll manage our own instances
-          sudo systemctl stop postgresql || true
-          # Get full version and copy binaries into dbdeployer's expected layout
-          PG_FULL=$(dpkg -s postgresql-${PG_VERSION} | grep '^Version:' | sed 's/Version: //' | cut -d'-' -f1)
-          echo "PostgreSQL version: ${PG_FULL}"
-          mkdir -p ~/opt/postgresql/${PG_FULL}/{bin,lib,share}
-          cp -a /usr/lib/postgresql/${PG_VERSION}/bin/. ~/opt/postgresql/${PG_FULL}/bin/
-          cp -a /usr/lib/postgresql/${PG_VERSION}/lib/. ~/opt/postgresql/${PG_FULL}/lib/
-          cp -a /usr/share/postgresql/${PG_VERSION}/. ~/opt/postgresql/${PG_FULL}/share/
-          ls ~/opt/postgresql/${PG_FULL}/bin/
+      - name: Download and unpack PostgreSQL debs (no system install)
+        run: |
+          # Install libpq5 only — psql links against it dynamically at runtime.
+          # Deliberately DO NOT install postgresql-${PG_VERSION} or
+          # postgresql-client-${PG_VERSION}: that would create
+          # /usr/share/postgresql/${PG_VERSION}/ on the runner, which masks
+          # issue #112 by letting PG fall back to the absolute compiled-in
+          # SHAREDIR rather than exercising dbdeployer's relocation layout.
+          sudo apt-get install -y libpq5
+
+          # Documented user flow from README:
+          #   apt-get download postgresql-NN postgresql-client-NN
+          #   dbdeployer unpack --provider=postgresql ...
+          apt-get download postgresql-${PG_VERSION} postgresql-client-${PG_VERSION}
+          ./dbdeployer unpack --provider=postgresql \
+            postgresql-${PG_VERSION}_*.deb \
+            postgresql-client-${PG_VERSION}_*.deb
+          ls ~/opt/postgresql/
+
+          # Sanity: confirm we did NOT install postgresql-server side-effects.
+          # If /usr/share/postgresql/${PG_VERSION}/ exists, the test is no
+          # longer exercising the relocation path that issue #112 was about.
+          if [ -d "/usr/share/postgresql/${PG_VERSION}" ]; then
+            echo "FAIL: /usr/share/postgresql/${PG_VERSION}/ exists — system PG install is masking issue #112"
+            exit 1
+          fi
 
       - name: Test init --provider=postgresql
         run: |

.github/workflows/proxysql_integration_tests.yml

@@ -402,16 +402,24 @@ jobs:
       - name: Build dbdeployer
         run: go build -o dbdeployer .
 
-      - name: Install PostgreSQL and set up binaries
+      - name: Download and unpack PostgreSQL debs (no system install)
         run: |
-          sudo apt-get install -y postgresql-16 postgresql-client-16
-          sudo systemctl stop postgresql || true
-          PG_FULL=$(dpkg -s postgresql-16 | grep '^Version:' | sed 's/Version: //' | cut -d'-' -f1)
-          echo "PostgreSQL version: ${PG_FULL}"
-          mkdir -p ~/opt/postgresql/${PG_FULL}/{bin,lib,share}
-          cp -a /usr/lib/postgresql/16/bin/. ~/opt/postgresql/${PG_FULL}/bin/
-          cp -a /usr/lib/postgresql/16/lib/. ~/opt/postgresql/${PG_FULL}/lib/
-          cp -a /usr/share/postgresql/16/. ~/opt/postgresql/${PG_FULL}/share/
+          # Install libpq5 only — psql links against it dynamically at runtime.
+          # We do NOT install postgresql-16 itself, since that would create
+          # /usr/share/postgresql/16/ on the runner and mask issue #112.
+          sudo apt-get install -y libpq5
+
+          # Documented user flow: download the debs, then unpack via dbdeployer.
+          apt-get download postgresql-16 postgresql-client-16
+          ./dbdeployer unpack --provider=postgresql \
+            postgresql-16_*.deb postgresql-client-16_*.deb
+          ls ~/opt/postgresql/
+
+          # Sanity: confirm no system PG install snuck in via dependencies.
+          if [ -d "/usr/share/postgresql/16" ]; then
+            echo "FAIL: /usr/share/postgresql/16/ exists — system PG install is masking issue #112"
+            exit 1
+          fi
 
       - name: Test replication with --with-proxysql
         run: |

providers/postgresql/sandbox.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 
 	"github.com/ProxySQL/dbdeployer/providers"
 )
@@ -20,6 +21,10 @@ func (p *PostgreSQLProvider) CreateSandbox(config providers.SandboxConfig) (*pro
 	logDir := filepath.Join(dataDir, "log")
 	logFile := filepath.Join(config.Dir, "postgresql.log")
 
+	if err := checkLinuxLayout(basedir, binDir); err != nil {
+		return nil, err
+	}
+
 	replication := config.Options["replication"] == "true"
 
 	// Run initdb (data dir must not exist or must be empty)
@@ -94,3 +99,35 @@ func (p *PostgreSQLProvider) resolveBasedir(config providers.SandboxConfig) (str
 	}
 	return basedirFromVersion(config.Version)
 }
+
+// checkLinuxLayout detects PostgreSQL extractions produced by older
+// versions of dbdeployer (before issue #112 was fixed). In the old layout,
+// <basedir>/bin/<binary> were regular files; PG's make_relative_path() then
+// failed to relocate the compiled-in SHAREDIR (`/usr/share/postgresql/<major>`),
+// so initdb died with "could not open directory /usr/share/postgresql/<major>/timezonesets".
+//
+// New extractions place the real binaries under
+// <basedir>/lib/postgresql/<major>/bin/ and expose them via symlinks at
+// <basedir>/bin/. If we find a regular file there on Linux, the user
+// needs to re-run unpack.
+func checkLinuxLayout(basedir, binDir string) error {
+	if runtime.GOOS != "linux" {
+		return nil
+	}
+	fi, err := os.Lstat(filepath.Join(binDir, "postgres"))
+	if err != nil {
+		// Missing binary will be reported by the initdb step below with a
+		// clearer error than we could produce here.
+		return nil
+	}
+	if fi.Mode()&os.ModeSymlink != 0 {
+		return nil
+	}
+	return fmt.Errorf(
+		"PostgreSQL binaries at %s were unpacked by an older dbdeployer using\n"+
+			"a layout incompatible with deb-packaged PostgreSQL — initdb would fail\n"+
+			"to find share files (see issue #112). Re-run unpack to fix:\n"+
+			"    dbdeployer unpack --provider=postgresql <server.deb> <client.deb>",
+		basedir,
+	)
+}

providers/postgresql/unpack.go

@@ -63,23 +63,50 @@ func UnpackDebs(serverDeb, clientDeb, targetDir string) error {
 	major := strings.Split(version, ".")[0]
 
 	srcBin := filepath.Join(tmpDir, "usr", "lib", "postgresql", major, "bin")
-	srcLib := filepath.Join(tmpDir, "usr", "lib", "postgresql", major, "lib")
+	srcPgLib := filepath.Join(tmpDir, "usr", "lib", "postgresql", major, "lib")
 	srcShare := filepath.Join(tmpDir, "usr", "share", "postgresql", major)
 
-	dstBin := filepath.Join(targetDir, "bin")
-	dstLib := filepath.Join(targetDir, "lib")
-	dstShare := filepath.Join(targetDir, "share")
+	// Destination layout mirrors Debian's compile-time prefix so that PG's
+	// make_relative_path() (src/port/path.c) succeeds at runtime. Debian
+	// builds PG with BINDIR=/usr/lib/postgresql/<major>/bin and
+	// SHAREDIR=/usr/share/postgresql/<major>; the common prefix is just
+	// /usr/, so for PG to relocate SHAREDIR to <basedir>/share/postgresql/<major>
+	// the running binary's parent directory must end in
+	// "lib/postgresql/<major>/bin". Same logic applies to PKGLIBDIR.
+	//
+	// We therefore place the real binaries and extension libs under
+	// <basedir>/lib/postgresql/<major>/, and expose them via symlinks at
+	// <basedir>/bin/ so existing callers (sandbox.go, scripts.go) keep
+	// working. On Linux, PG's find_my_exec() reads /proc/self/exe which
+	// resolves symlinks to the real file, so launching via the symlink
+	// still gives PG the relocatable path it needs.
+	dstPgBin := filepath.Join(targetDir, "lib", "postgresql", major, "bin")
+	dstPgLib := filepath.Join(targetDir, "lib", "postgresql", major, "lib")
+	dstShare := filepath.Join(targetDir, "share")                       // flat, for `initdb -L`
+	dstPgShare := filepath.Join(targetDir, "share", "postgresql", major) // nested, for postgres runtime
+	dstBin := filepath.Join(targetDir, "bin")                           // symlinks into dstPgBin
 
-	for _, dir := range []string{dstBin, dstLib, dstShare} {
+	// Make UnpackDebs idempotent: remove any prior extraction artifacts
+	// in the subtrees we own so re-running unpack picks up the latest
+	// layout cleanly (in particular when migrating from a pre-fix layout
+	// where <basedir>/bin/<binary> were real files instead of symlinks).
+	for _, dir := range []string{dstBin, filepath.Join(targetDir, "lib"), dstShare} {
+		if err := os.RemoveAll(dir); err != nil {
+			return fmt.Errorf("removing stale %s: %w", dir, err)
+		}
+	}
+
+	for _, dir := range []string{dstPgBin, dstPgLib, dstShare, dstPgShare, dstBin} {
 		if err := os.MkdirAll(dir, 0755); err != nil {
 			return fmt.Errorf("creating directory %s: %w", dir, err)
 		}
 	}
 
 	copies := []struct{ src, dst string }{
-		{srcBin, dstBin},
-		{srcLib, dstLib},
-		{srcShare, dstShare},
+		{srcBin, dstPgBin},
+		{srcPgLib, dstPgLib},
+		{srcShare, dstShare},   // flat copy for `initdb -L`
+		{srcShare, dstPgShare}, // nested copy for postgres server runtime
 	}
 	for _, c := range copies {
 		if _, err := os.Stat(c.src); os.IsNotExist(err) {
@@ -91,17 +118,21 @@ func UnpackDebs(serverDeb, clientDeb, targetDir string) error {
 		}
 	}
 
-	// The postgres binary resolves share data relative to its own binary as
-	// ../share/postgresql/<major>/ (compiled-in prefix from deb packaging).
-	// Copy share files there too so both initdb (-L share/) and the postgres
-	// server binary can find timezonesets and other share data.
-	pgShareCompat := filepath.Join(dstShare, "postgresql", major)
-	if err := os.MkdirAll(pgShareCompat, 0755); err != nil {
-		return fmt.Errorf("creating compat share dir: %w", err)
+	// Expose every server-side binary as <basedir>/bin/<name>, symlinked
+	// into the nested lib/postgresql/<major>/bin/ directory.
+	entries, err := os.ReadDir(dstPgBin)
+	if err != nil {
+		return fmt.Errorf("reading %s: %w", dstPgBin, err)
 	}
-	compatCmd := exec.Command("cp", "-a", srcShare+"/.", pgShareCompat+"/") //nolint:gosec // paths are from controlled deb extraction
-	if output, err := compatCmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("copying share to compat path: %s: %w", string(output), err)
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		linkTarget := filepath.Join("..", "lib", "postgresql", major, "bin", entry.Name())
+		linkPath := filepath.Join(dstBin, entry.Name())
+		if err := os.Symlink(linkTarget, linkPath); err != nil {
+			return fmt.Errorf("symlinking %s -> %s: %w", linkPath, linkTarget, err)
+		}
 	}
 
 	for _, bin := range RequiredBinaries() {