diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 35584dd..5ce9df9 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -520,7 +520,7 @@ jobs: rm -rf "$HOME/sandboxes"/* echo '{}' > "$HOME/.dbdeployer/sandboxes.json" - - name: Install PostgreSQL for ts tests + - name: Download and unpack PostgreSQL debs for ts tests (no system install) run: | # Add PostgreSQL apt repo (PGDG) — required for postgresql-16 on ubuntu-22.04 sudo apt-get install -y curl ca-certificates @@ -528,14 +528,24 @@ jobs: sudo curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update - sudo apt-get install -y postgresql-16 postgresql-client-16 - sudo systemctl stop postgresql || true + + # Install libpq5 only — psql links against it dynamically. We do + # NOT install postgresql-16 itself, since that would put + # /usr/share/postgresql/16/ on the runner and mask issue #112. + sudo apt-get install -y libpq5 + + # Documented user flow: download the debs, then unpack via dbdeployer. export HOME="$GITHUB_WORKSPACE/home" - PG_FULL=$(dpkg -s postgresql-16 | grep '^Version:' | sed 's/Version: //' | cut -d'-' -f1) - mkdir -p "$HOME/opt/postgresql/${PG_FULL}/"{bin,lib,share} - cp -a /usr/lib/postgresql/16/bin/. "$HOME/opt/postgresql/${PG_FULL}/bin/" - cp -a /usr/lib/postgresql/16/lib/. "$HOME/opt/postgresql/${PG_FULL}/lib/" - cp -a /usr/share/postgresql/16/. "$HOME/opt/postgresql/${PG_FULL}/share/" + mkdir -p "$HOME" + apt-get download postgresql-16 postgresql-client-16 + ./dbdeployer unpack --provider=postgresql \ + postgresql-16_*.deb postgresql-client-16_*.deb + + # Sanity: confirm no system PG install snuck in via dependencies. + if [ -d "/usr/share/postgresql/16" ]; then + echo "FAIL: /usr/share/postgresql/16/ exists — system PG install is masking issue #112" + exit 1 + fi - name: Run ts PostgreSQL tests env: @@ -586,20 +596,32 @@ jobs: - name: Build dbdeployer run: go build -o dbdeployer . - - name: Install PostgreSQL and set up binaries - run: | - # Install PostgreSQL to get properly configured binaries - sudo apt-get install -y postgresql-${PG_VERSION} postgresql-client-${PG_VERSION} - # Stop the system service — we'll manage our own instances - sudo systemctl stop postgresql || true - # Get full version and copy binaries into dbdeployer's expected layout - PG_FULL=$(dpkg -s postgresql-${PG_VERSION} | grep '^Version:' | sed 's/Version: //' | cut -d'-' -f1) - echo "PostgreSQL version: ${PG_FULL}" - mkdir -p ~/opt/postgresql/${PG_FULL}/{bin,lib,share} - cp -a /usr/lib/postgresql/${PG_VERSION}/bin/. ~/opt/postgresql/${PG_FULL}/bin/ - cp -a /usr/lib/postgresql/${PG_VERSION}/lib/. ~/opt/postgresql/${PG_FULL}/lib/ - cp -a /usr/share/postgresql/${PG_VERSION}/. ~/opt/postgresql/${PG_FULL}/share/ - ls ~/opt/postgresql/${PG_FULL}/bin/ + - name: Download and unpack PostgreSQL debs (no system install) + run: | + # Install libpq5 only — psql links against it dynamically at runtime. + # Deliberately DO NOT install postgresql-${PG_VERSION} or + # postgresql-client-${PG_VERSION}: that would create + # /usr/share/postgresql/${PG_VERSION}/ on the runner, which masks + # issue #112 by letting PG fall back to the absolute compiled-in + # SHAREDIR rather than exercising dbdeployer's relocation layout. + sudo apt-get install -y libpq5 + + # Documented user flow from README: + # apt-get download postgresql-NN postgresql-client-NN + # dbdeployer unpack --provider=postgresql ... + apt-get download postgresql-${PG_VERSION} postgresql-client-${PG_VERSION} + ./dbdeployer unpack --provider=postgresql \ + postgresql-${PG_VERSION}_*.deb \ + postgresql-client-${PG_VERSION}_*.deb + ls ~/opt/postgresql/ + + # Sanity: confirm we did NOT install postgresql-server side-effects. + # If /usr/share/postgresql/${PG_VERSION}/ exists, the test is no + # longer exercising the relocation path that issue #112 was about. + if [ -d "/usr/share/postgresql/${PG_VERSION}" ]; then + echo "FAIL: /usr/share/postgresql/${PG_VERSION}/ exists — system PG install is masking issue #112" + exit 1 + fi - name: Test init --provider=postgresql run: | diff --git a/.github/workflows/proxysql_integration_tests.yml b/.github/workflows/proxysql_integration_tests.yml index c364eea..500c531 100644 --- a/.github/workflows/proxysql_integration_tests.yml +++ b/.github/workflows/proxysql_integration_tests.yml @@ -402,16 +402,24 @@ jobs: - name: Build dbdeployer run: go build -o dbdeployer . - - name: Install PostgreSQL and set up binaries + - name: Download and unpack PostgreSQL debs (no system install) run: | - sudo apt-get install -y postgresql-16 postgresql-client-16 - sudo systemctl stop postgresql || true - PG_FULL=$(dpkg -s postgresql-16 | grep '^Version:' | sed 's/Version: //' | cut -d'-' -f1) - echo "PostgreSQL version: ${PG_FULL}" - mkdir -p ~/opt/postgresql/${PG_FULL}/{bin,lib,share} - cp -a /usr/lib/postgresql/16/bin/. ~/opt/postgresql/${PG_FULL}/bin/ - cp -a /usr/lib/postgresql/16/lib/. ~/opt/postgresql/${PG_FULL}/lib/ - cp -a /usr/share/postgresql/16/. ~/opt/postgresql/${PG_FULL}/share/ + # Install libpq5 only — psql links against it dynamically at runtime. + # We do NOT install postgresql-16 itself, since that would create + # /usr/share/postgresql/16/ on the runner and mask issue #112. + sudo apt-get install -y libpq5 + + # Documented user flow: download the debs, then unpack via dbdeployer. + apt-get download postgresql-16 postgresql-client-16 + ./dbdeployer unpack --provider=postgresql \ + postgresql-16_*.deb postgresql-client-16_*.deb + ls ~/opt/postgresql/ + + # Sanity: confirm no system PG install snuck in via dependencies. + if [ -d "/usr/share/postgresql/16" ]; then + echo "FAIL: /usr/share/postgresql/16/ exists — system PG install is masking issue #112" + exit 1 + fi - name: Test replication with --with-proxysql run: | diff --git a/providers/postgresql/sandbox.go b/providers/postgresql/sandbox.go index f5bfd12..5fb65c4 100644 --- a/providers/postgresql/sandbox.go +++ b/providers/postgresql/sandbox.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "github.com/ProxySQL/dbdeployer/providers" ) @@ -20,6 +21,10 @@ func (p *PostgreSQLProvider) CreateSandbox(config providers.SandboxConfig) (*pro logDir := filepath.Join(dataDir, "log") logFile := filepath.Join(config.Dir, "postgresql.log") + if err := checkLinuxLayout(basedir, binDir); err != nil { + return nil, err + } + replication := config.Options["replication"] == "true" // Run initdb (data dir must not exist or must be empty) @@ -94,3 +99,35 @@ func (p *PostgreSQLProvider) resolveBasedir(config providers.SandboxConfig) (str } return basedirFromVersion(config.Version) } + +// checkLinuxLayout detects PostgreSQL extractions produced by older +// versions of dbdeployer (before issue #112 was fixed). In the old layout, +// /bin/ were regular files; PG's make_relative_path() then +// failed to relocate the compiled-in SHAREDIR (`/usr/share/postgresql/`), +// so initdb died with "could not open directory /usr/share/postgresql//timezonesets". +// +// New extractions place the real binaries under +// /lib/postgresql//bin/ and expose them via symlinks at +// /bin/. If we find a regular file there on Linux, the user +// needs to re-run unpack. +func checkLinuxLayout(basedir, binDir string) error { + if runtime.GOOS != "linux" { + return nil + } + fi, err := os.Lstat(filepath.Join(binDir, "postgres")) + if err != nil { + // Missing binary will be reported by the initdb step below with a + // clearer error than we could produce here. + return nil + } + if fi.Mode()&os.ModeSymlink != 0 { + return nil + } + return fmt.Errorf( + "PostgreSQL binaries at %s were unpacked by an older dbdeployer using\n"+ + "a layout incompatible with deb-packaged PostgreSQL — initdb would fail\n"+ + "to find share files (see issue #112). Re-run unpack to fix:\n"+ + " dbdeployer unpack --provider=postgresql ", + basedir, + ) +} diff --git a/providers/postgresql/unpack.go b/providers/postgresql/unpack.go index 3384b3b..76ad764 100644 --- a/providers/postgresql/unpack.go +++ b/providers/postgresql/unpack.go @@ -63,23 +63,50 @@ func UnpackDebs(serverDeb, clientDeb, targetDir string) error { major := strings.Split(version, ".")[0] srcBin := filepath.Join(tmpDir, "usr", "lib", "postgresql", major, "bin") - srcLib := filepath.Join(tmpDir, "usr", "lib", "postgresql", major, "lib") + srcPgLib := filepath.Join(tmpDir, "usr", "lib", "postgresql", major, "lib") srcShare := filepath.Join(tmpDir, "usr", "share", "postgresql", major) - dstBin := filepath.Join(targetDir, "bin") - dstLib := filepath.Join(targetDir, "lib") - dstShare := filepath.Join(targetDir, "share") + // Destination layout mirrors Debian's compile-time prefix so that PG's + // make_relative_path() (src/port/path.c) succeeds at runtime. Debian + // builds PG with BINDIR=/usr/lib/postgresql//bin and + // SHAREDIR=/usr/share/postgresql/; the common prefix is just + // /usr/, so for PG to relocate SHAREDIR to /share/postgresql/ + // the running binary's parent directory must end in + // "lib/postgresql//bin". Same logic applies to PKGLIBDIR. + // + // We therefore place the real binaries and extension libs under + // /lib/postgresql//, and expose them via symlinks at + // /bin/ so existing callers (sandbox.go, scripts.go) keep + // working. On Linux, PG's find_my_exec() reads /proc/self/exe which + // resolves symlinks to the real file, so launching via the symlink + // still gives PG the relocatable path it needs. + dstPgBin := filepath.Join(targetDir, "lib", "postgresql", major, "bin") + dstPgLib := filepath.Join(targetDir, "lib", "postgresql", major, "lib") + dstShare := filepath.Join(targetDir, "share") // flat, for `initdb -L` + dstPgShare := filepath.Join(targetDir, "share", "postgresql", major) // nested, for postgres runtime + dstBin := filepath.Join(targetDir, "bin") // symlinks into dstPgBin - for _, dir := range []string{dstBin, dstLib, dstShare} { + // Make UnpackDebs idempotent: remove any prior extraction artifacts + // in the subtrees we own so re-running unpack picks up the latest + // layout cleanly (in particular when migrating from a pre-fix layout + // where /bin/ were real files instead of symlinks). + for _, dir := range []string{dstBin, filepath.Join(targetDir, "lib"), dstShare} { + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("removing stale %s: %w", dir, err) + } + } + + for _, dir := range []string{dstPgBin, dstPgLib, dstShare, dstPgShare, dstBin} { if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("creating directory %s: %w", dir, err) } } copies := []struct{ src, dst string }{ - {srcBin, dstBin}, - {srcLib, dstLib}, - {srcShare, dstShare}, + {srcBin, dstPgBin}, + {srcPgLib, dstPgLib}, + {srcShare, dstShare}, // flat copy for `initdb -L` + {srcShare, dstPgShare}, // nested copy for postgres server runtime } for _, c := range copies { if _, err := os.Stat(c.src); os.IsNotExist(err) { @@ -91,17 +118,21 @@ func UnpackDebs(serverDeb, clientDeb, targetDir string) error { } } - // The postgres binary resolves share data relative to its own binary as - // ../share/postgresql// (compiled-in prefix from deb packaging). - // Copy share files there too so both initdb (-L share/) and the postgres - // server binary can find timezonesets and other share data. - pgShareCompat := filepath.Join(dstShare, "postgresql", major) - if err := os.MkdirAll(pgShareCompat, 0755); err != nil { - return fmt.Errorf("creating compat share dir: %w", err) + // Expose every server-side binary as /bin/, symlinked + // into the nested lib/postgresql//bin/ directory. + entries, err := os.ReadDir(dstPgBin) + if err != nil { + return fmt.Errorf("reading %s: %w", dstPgBin, err) } - compatCmd := exec.Command("cp", "-a", srcShare+"/.", pgShareCompat+"/") //nolint:gosec // paths are from controlled deb extraction - if output, err := compatCmd.CombinedOutput(); err != nil { - return fmt.Errorf("copying share to compat path: %s: %w", string(output), err) + for _, entry := range entries { + if entry.IsDir() { + continue + } + linkTarget := filepath.Join("..", "lib", "postgresql", major, "bin", entry.Name()) + linkPath := filepath.Join(dstBin, entry.Name()) + if err := os.Symlink(linkTarget, linkPath); err != nil { + return fmt.Errorf("symlinking %s -> %s: %w", linkPath, linkTarget, err) + } } for _, bin := range RequiredBinaries() {