Add pgrx test and reorganize PGXS tests

Move the PGXS tests to a subdirectory separate testing PGXS from testing zip vs. Git bundling in `cicd.yml`. Then add a pgrx "hello world" project in `test/pgrx` and test it, too, including `pg_regress` tests. Update `/etc/sudoers` and the `PATH` environment variable to allow otherwise unprivileged users to do what they need to do. Also remove `nobody`. Copy-edit the README a bit and explicitly set the PATH environment variable in the `Dockerfile`, because `/etc/profile.d` scripts apparently don't run in GitHub workflows.
pgxn · Apr 23, 2024 · ca4c927 · ca4c927
1 parent bf83f61
commit ca4c927
Show file tree

Hide file tree

Showing 23 changed files with 222 additions and 52 deletions.
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
@@ -5,41 +5,65 @@ on:
   schedule:
     - cron: '0 14 3 * *' # Monthly at 2pm on the 3rd
 jobs:
-  test:
-    name: 🐘 PostgreSQL ${{ matrix.pg }}
+  pgxs:
+    name: 🐘 PGXS on Postgres ${{ matrix.pg }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        pg: [16, 15, 14, 13, 12, 11, 10, 9.6, 9.5, 9.4, 9.3, 9.2, 9.1, '9.0', 8.4, 8.3, 8.2]
+        pg: [17, 16, 15, 14, 13, 12, 11, 10, 9.6, 9.5, 9.4, 9.3, 9.2, 9.1, '9.0', 8.4, 8.3, 8.2]
     steps:
       - uses: actions/checkout@v4
       - name: Build Image
         run: docker build -t pgxn-tools-test .
-      # Test with Git repo
-      - name: Test Git as root
-        run: "docker run -w /repo --rm --volume \"$(pwd):/repo\" pgxn-tools-test ./test/runtest.sh ${{ matrix.pg }} git"
-      - name: Test Git as non-root
-        run: "docker run -w /repo --rm --volume \"$(pwd):/repo\" -e AS_USER=worker pgxn-tools-test ./test/runtest.sh ${{ matrix.pg }} git"
-      - name: Test Git with extra file
-        run: "docker run -w /repo --rm --volume \"$(pwd):/repo\" pgxn-tools-test ./test/runtest.sh ${{ matrix.pg }} git yes"
-      - name: Test Git with archive-all
+      # First test system UID, otherwise it can't use files touched by root.
+      - name: Test with system UID
+        run: "docker run -w /repo --rm --volume \"$(pwd):/repo\" -e AS_USER=pgxn_worker -e LOCAL_UID=$(id -u) pgxn-tools-test ./test/pgxs/runtest.sh ${{ matrix.pg }}"
+      - name: Test as root
+        run: "docker run -w /repo --rm --volume \"$(pwd):/repo\" pgxn-tools-test ./test/pgxs/runtest.sh ${{ matrix.pg }}"
+
+  pgrx:
+    name: 🦀 pgrx on Postgres ${{ matrix.pg }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        pg: [16, 15, 14, 13, 12, 11]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Image
+        run: docker build -t pgxn-tools-test .
+      # First test system UID, otherwise it can't use files touched by root.
+      - name: Test with system UID
+        run: "docker run -w /repo --rm --volume \"$(pwd):/repo\" -e AS_USER=pgxn_worker -e LOCAL_UID=$(id -u) pgxn-tools-test ./test/pgrx/runtest.sh ${{ matrix.pg }}"
+      - name: Test as root
+        run: "docker run -w /repo --rm --volume \"$(pwd):/repo\" pgxn-tools-test ./test/pgrx/runtest.sh ${{ matrix.pg }}"
+
+  bundle:
+    name: ${{ matrix.util.icon }} Bundle with ${{ matrix.util.name }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        util:
+          - { icon: 🏷️, name: git, dir: "", run: ./test/pgxs }
+          - { icon: 🤐, name: zip, dir: /test, run: ./pgxs }
+    env: { PG_VERSION: 16 }
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Image
+        run: docker build -t pgxn-tools-test .
+      - name: Test ${{ matrix.util.name }}
+        run: "docker run -w /repo --rm --volume \"$(pwd)${{ matrix.util.dir }}:/repo\" pgxn-tools-test ${{ matrix.util.run }}/runtest.sh ${{ env.PG_VERSION }} ${{ matrix.util[1] }}"
+      - name: Test ${{ matrix.util.name }} with extra file
+        run: "docker run -w /repo --rm --volume \"$(pwd)${{ matrix.util.dir }}:/repo\" pgxn-tools-test ${{ matrix.util.run }}/runtest.sh ${{ env.PG_VERSION }} ${{ matrix.util[1] }} yes"
+      - name: Test ${{ matrix.util.name }} with archive-all
         env: { GIT_ARCHIVE_CMD: archive-all }
-        run: "docker run -w /repo --rm --volume \"$(pwd):/repo\" pgxn-tools-test ./test/runtest.sh ${{ matrix.pg }} git"
-      # Test without Git repo
-      - name: Test Zip as root
-        run: "docker run -w /repo --rm --volume \"$(pwd)/test:/repo\" pgxn-tools-test ./runtest.sh ${{ matrix.pg }} zip"
-      - name: Test Zip as non-root
-        run: "docker run -w /repo --rm --volume \"$(pwd)/test:/repo\" -e AS_USER=worker pgxn-tools-test ./runtest.sh ${{ matrix.pg }} zip"
-      - name: Test Zip with zip excluded file
-        run: "docker run -w /repo --rm --volume \"$(pwd)/test:/repo\" pgxn-tools-test ./runtest.sh ${{ matrix.pg }} zip yes"
-      # Test NO_CLUSTER
+        run: "docker run -w /repo --rm --volume \"$(pwd)${{ matrix.util.dir }}:/repo\" pgxn-tools-test ${{ matrix.util.run }}/runtest.sh ${{ env.PG_VERSION }} ${{ matrix.util[1] }}"
       - name: Test NO_CLUSTER
-        run: "docker run -w /repo -e NO_CLUSTER=1 --rm --volume \"$(pwd):/repo\" pgxn-tools-test ./test/no_cluster_test.sh ${{ matrix.pg }}"
+        run: "docker run -w /repo -e NO_CLUSTER=1 --rm --volume \"$(pwd)${{ matrix.util.dir }}:/repo\" pgxn-tools-test ${{ matrix.util.run }}/no_cluster_test.sh ${{ env.PG_VERSION }}"
 
   publish:
     # Publish for a tag starting with v.
     name: Push to Docker Hub
-    needs: test
+    needs: [pgxs, pgrx, bundle]
     if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     steps:

diff --git a/Dockerfile b/Dockerfile
@@ -3,21 +3,26 @@ FROM debian:bookworm-slim AS pgxn-config
 ADD https://salsa.debian.org/postgresql/postgresql-common/-/raw/master/pgdg/apt.postgresql.org.sh /usr/local/bin/
 
 RUN chmod +x /usr/local/bin/apt.postgresql.org.sh \
+    # Install apt dependencies
     && apt-get update \
     && apt-get install -y --no-install-recommends \
         build-essential clang llvm llvm-dev llvm-runtime cmake libtoml-parser-perl \
         pgxnclient libtap-parser-sourcehandler-pgtap-perl libipc-run-perl libtest-simple-perl sudo gosu \
         ca-certificates gnupg2 zip unzip libarchive-tools curl git libicu-dev libxml2 locales ssl-cert \
+    # Clean out unwanted stuff
     && apt-get -y purge postgresql-client-common \
     && apt-get clean \
     && rm -rf /var/cache/apt/* /var/lib/apt/lists/* \
+    # Install CPAN dependencies
     && curl -L https://cpanmin.us/ -o cpanm && chmod +x cpanm \
     && ./cpanm --notest PGXN::Meta::Validator \
     && rm -r cpanm ~/.cpanm \
+    # Configure sudoers to allow otherwise unprivileged users to do everything.
     && echo Defaults	lecture = never >> /etc/sudoers \
+    && echo 'Defaults env_keep += "CARGO_* RUSTUP_* PGRX_*"' >> /etc/sudoers \
     && perl -i -pe 's/\bALL$/NOPASSWD:ALL/g' /etc/sudoers \
     && echo 'postgres	ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers \
-    && echo 'nobody	ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers \
+    && perl -i -pe 's/^(Defaults\s+secure_path)/# $1/' /etc/sudoers \
     # Ensure Git can do stuff in the working directory (issue #5).
     && git config --system --add safe.directory '*' \
     # Install git-archive-all
@@ -26,11 +31,11 @@ RUN chmod +x /usr/local/bin/apt.postgresql.org.sh \
     && install -m 0755 git_archive_all.py "$(git --exec-path)/git-archive-all" \
     && rm git_archive_all.py \
     # Install the Rust toolchain
-    && curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf | env CARGO_HOME=/usr/share/cargo RUSTUP_HOME=/usr/share/rustup bash -s -- -y \
-    && echo "PATH=\"${PATH}:/usr/share/cargo/bin\"" > /etc/profile.d/cargo.sh
+    && curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf | env CARGO_HOME=/usr/share/cargo RUSTUP_HOME=/usr/share/rustup bash -s -- -y --profile minimal --component rustfmt --component clippy \
+    && chmod 0777 /usr/share/cargo /usr/share/cargo/bin
 
 COPY bin/* /usr/local/bin/
 
-ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 CARGO_HOME=/usr/share/cargo RUSTUP_HOME=/usr/share/rustup
+ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 CARGO_HOME=/usr/share/cargo PGRX_HOME=/tmp/.pgrx RUSTUP_HOME=/usr/share/rustup PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/share/cargo/bin
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["/bin/bash"]
diff --git a/README.md b/README.md
@@ -9,8 +9,8 @@ releases to PGXN. The image contains these utilities:
 
 *   [`pgxn`][cli]: The PGXN command-line client
 *   [`pg_prove`]: Runs and harnessing pgTAP tests
-*   [`pg-start`] Pass a PostgreSQL major version to install and starts a PostgreSQL cluster
-*   [`pg-build-test`]: Builds and tests an extension in the current directory
+*   [`pg-start`] Installs a major version of PostgreSQL and starts a cluster
+*   [`pg-build-test`]: Builds and tests a [PGXS] extension in the current directory
 *   [`pgrx-build-test`]: Builds and tests a [pgrx] extension in the current directory
 *   [`pgxn-bundle`]: Validates the PGXN META.json file and bundles up a release
 *   [`pgxn-release`]: Release to PGXN
@@ -35,10 +35,13 @@ directory.
 
 ### Unprivileged User
 
+**NOTE:** GitHub workflow container jobs must be run as root. The options for
+unprivileged users in this section will not work in that context. See
+[Unprivileged User Workflow](#unprivileged-user-workflow) for details.
+
 By default the container runs as `root`. To run as an unprivileged user, pass
 the `AS_USER` environment variable and a user with that name will be created
-with `sudo` privileges (already used by `pg-start`, `pg-build-test`, and
-`pgrx-build-test`):
+with `sudo` privileges (already used by `pg-start` and `pg-build-test`):
 
 ``` sh
 docker run -it --rm -w /repo -e AS_USER=worker \
@@ -56,7 +59,7 @@ docker run -it --rm -w /repo -e AS_USER=worker -e LOCAL_UID=$(id -u) \
     sh -c 'sudo pg-start 14 && pg-build-test'
 ```
 
-### Sudo-Enabled Users
+#### Included Users
 
 The `nobody` user, included in the image, and the `postgres` user, created by
 `pg-start`, also have full permission to use `sudo` without a password prompt.
@@ -74,7 +77,7 @@ jobs:
   test:
     strategy:
       matrix:
-        pg: [16, 15, 14, 13, 12, 11, 10, 9.6, 9.5, 9.4, 9.3, 9.2, 9.1, 9.0, 8.4, 8.3, 8.2]
+        pg: [17, 16, 15, 14, 13, 12, 11, 10, 9.6, 9.5, 9.4, 9.3, 9.2, 9.1, 9.0, 8.4, 8.3, 8.2]
     name: 🐘 PostgreSQL ${{ matrix.pg }}
     runs-on: ubuntu-latest
     container: pgxn/pgxn-tools
@@ -84,7 +87,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v4
       - name: Test on PostgreSQL ${{ matrix.pg }}
-        run: pg-build-test # or pgrx-build-test for a pgrx extension
+        run: pg-build-test # or pgrx-build-test
 ```
 
 This example demonstrates automatic publishing of a release whenever a tag is
@@ -139,10 +142,10 @@ jobs:
 
 ### Unprivileged User Workflow
 
-GitHub workflows [require the root user] to work with the workspace. To perform
-tasks as an unprivileged user, first set things up as the root user, then use
-[gosu] to execute a command as the `postgres` (can still run `sudo`) or `nobody`
-(no privileges at all) user. For example:
+GitHub workflows [require the root user] to work with the workspace. To
+perform tasks as an unprivileged user, first set things up as the root user,
+then use [gosu] to execute a command as the `postgres` (can run `sudo`) or
+`nobody` (no privileges at all) user. For example:
 
 ``` yaml
     container: pgxn/pgxn-tools
@@ -158,6 +161,21 @@ with the current directory's files all owned the newly-created `postgres` user,
 the last `run` commands executes `pg-build-test` as `postgres`, with the
 necessary permissions to write files to the workspace directory.
 
+Alternatively, run the Docker image directly and set `AS_USER` to the desired
+username and pass the GitHub worker UID via the `LOCAL_UID` variable:
+
+``` yaml
+    steps:
+      - uses: actions/checkout@v4
+      - run: >-
+        docker run -w /repo --rm --volume "$(pwd):/repo"
+               -e AS_USER=pgxn_worker -e LOCAL_UID=$(id -u)
+               pgxn/pgxn-tools pg-start && pg-build-test
+```
+
+This allows the user in the container to run a as the same UID as the host
+user and therefore have access to all the mounted files owned by that user.
+
 Tools
 -----
 
@@ -229,8 +247,8 @@ pg_createcluster --start 16 my16 -p 5416 -- -A trust
 pg-build-test
 ```
 
-Simply builds, installs, and tests a PostgreSQL extension or other code in the
-current directory. Effectively the equivalent of:
+Simply builds, installs, and tests a [PGXS] PostgreSQL extension or other code
+in the current directory. Effectively the equivalent of:
 
 ``` sh
 make
@@ -256,9 +274,9 @@ pg-build-test
 pgrx-build-test
 ```
 
-Build, install, and test a PostgreSQL [pgrx] extension. It reads the required
-version of [pgrx] from the `Cargo.toml` file, which must be v0.11.4 or higher.
-Effectively the equivalent of:
+Builds, installs, and tests a PostgreSQL [pgrx] extension. It reads the
+required version of [pgrx] from the `Cargo.toml` file, which must be v0.11.4
+or higher. Effectively the equivalent of:
 
 ``` sh
 cargo install --locked cargo-pgrx --version ${PGRX_VERSION}
@@ -273,11 +291,6 @@ all output. It will also run `make installcheck` if it finds a `Makefile` that
 appears to define the `installcheck` target, and emit the contents of the
 `regression.diffs` file if it fails.
 
-**Note:** Since `pgrx` uses `sudo` to start the cluster as the `postgres`
-user, so some environment variables may not be present while tests run. If
-your Rust code reads environment variables it should guard against
-`NotPresent` errors to handle unexpectedly missing environment variables.
-
 ### [`pgxn-bundle`]
 
 ``` sh
@@ -439,6 +452,15 @@ The image includes these packages; pass additional packages to
 *   [ssl-cert](https://packages.debian.org/bookworm/ssl-cert)
 *   [git-archive-all](https://github.com/Kentzo/git-archive-all) (run `git archive-all`)
 
+### Rust Components
+
+The image includes the following Rust [components]; use `rustup component add`
+to add additional components at runtime.
+
+*   [rust minimal]: https://rust-lang.github.io/rustup/concepts/profiles.html
+*   [rustfmt]: https://github.com/rust-lang/rustfmt
+*   [clippy]: https://github.com/rust-lang/rust-clippy
+
 Author
 ------
 
@@ -474,3 +496,5 @@ Copyright (c) 2020-2024 The PGXN Maintainers. Distributed under the
   [PostgreSQL TAP]: https://www.postgresql.org/docs/current/regress-tap.html
   [TAP]: https://testanything.org "Test Anything Protocol"
   [pgrx]: https://github.com/pgcentralfoundation/pgrx
+  [PGXS]: https://www.postgresql.org/docs/current/extend-pgxs.html
+  [components]: https://rust-lang.github.io/rustup/concepts/components.html
diff --git a/bin/pgrx-build-test b/bin/pgrx-build-test
@@ -55,7 +55,7 @@ run [qw(cargo pgrx package --test --pg-config), $pg_config] or exit $? >> 8;
 # Install the extension.
 # (Must come before test: https://github.com/pgcentralfoundation/pgrx/issues/1670)
 say "### Installing $cfg->{package}{name}";
-run [qw(cargo pgrx install --test --pg-config), $pg_config] or exit $? >> 8;
+run [qw(sudo cargo pgrx install --test --pg-config), $pg_config] or exit $? >> 8;
 
 # Run the tests as the postgres user.
 say "### Testing $cfg->{package}{name}";

diff --git a/test/.gitattributes b/test/.gitattributes
@@ -1,4 +1,4 @@
 .gitignore export-ignore
 .gitattributes export-ignore
 .github export-ignore
-/runtest.sh export-ignore
+runtest.sh export-ignore
diff --git a/test/pgrx/.cargo/config.toml b/test/pgrx/.cargo/config.toml
@@ -0,0 +1,3 @@
+[target.'cfg(target_os="macos")']
+# Postgres symbols won't be available until runtime
+rustflags = ["-Clink-arg=-Wl,-undefined,dynamic_lookup"]
diff --git a/test/pgrx/.gitignore b/test/pgrx/.gitignore
@@ -0,0 +1,8 @@
+.DS_Store
+.idea/
+/target
+*.iml
+**/*.rs.bk
+Cargo.lock
+regression.*
+results/
diff --git a/test/pgrx/Cargo.toml b/test/pgrx/Cargo.toml
@@ -0,0 +1,32 @@
+[package]
+name = "hello"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[features]
+default = ["pg13"]
+pg11 = ["pgrx/pg11", "pgrx-tests/pg11" ]
+pg12 = ["pgrx/pg12", "pgrx-tests/pg12" ]
+pg13 = ["pgrx/pg13", "pgrx-tests/pg13" ]
+pg14 = ["pgrx/pg14", "pgrx-tests/pg14" ]
+pg15 = ["pgrx/pg15", "pgrx-tests/pg15" ]
+pg16 = ["pgrx/pg16", "pgrx-tests/pg16" ]
+pg_test = []
+
+[dependencies]
+pgrx = "=0.11.4"
+
+[dev-dependencies]
+pgrx-tests = "=0.11.4"
+
+[profile.dev]
+panic = "unwind"
+
+[profile.release]
+panic = "unwind"
+opt-level = 3
+lto = "fat"
+codegen-units = 1
diff --git a/test/pgrx/Makefile b/test/pgrx/Makefile
@@ -0,0 +1,9 @@
+EXTENSION    = hello
+EXTVERSION   = 0.1.0
+TESTS        = test/sql/base.sql
+REGRESS      = base
+REGRESS_OPTS = --inputdir=test --load-extension=$(EXTENSION)
+PG_CONFIG    = pg_config
+
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
diff --git a/test/pgrx/hello.control b/test/pgrx/hello.control
@@ -0,0 +1,5 @@
+comment = 'hello:  Created by pgrx'
+default_version = '@CARGO_VERSION@'
+module_pathname = '$libdir/hello'
+relocatable = false
+superuser = true
diff --git a/test/pgrx/results/base.out b/test/pgrx/results/base.out
@@ -0,0 +1,6 @@
+SELECT hello_hello();
+ hello_hello  
+--------------
+ Hello, hello
+(1 row)
+
diff --git a/test/pgrx/runtest.sh b/test/pgrx/runtest.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -eu
+
+pgversion=$1
+cd "$(dirname "$0")"
+pg-start "$pgversion"
+pgrx-build-test