diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2991cf3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,132 @@ +# pgColumnar continuous integration. +# +# This mirrors the cadence the project already runs locally rather than +# inventing a second one: a build preflight across every supported major, which +# is what catches an API break, and the suite run on one major, which is what +# catches a behaviour break. The full five-major suite matrix stays a local gate +# (test/run_all_versions.sh) because PostgreSQL 19 is beta and not packaged in +# PGDG stable, so CI cannot reproduce it honestly. +# +# Warnings are failures here for the same reason they are in the local matrix. + +name: build and test + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Build against every supported major. Fast, and it is what an API change + # between majors trips first. + build: + name: build (PG ${{ matrix.pg }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + pg: ['15', '16', '17', '18'] + steps: + - uses: actions/checkout@v4 + + - name: Install PostgreSQL ${{ matrix.pg }} and codec headers + run: | + set -euo pipefail + sudo install -d /usr/share/postgresql-common/pgdg + sudo curl -fsSL -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ + https://www.postgresql.org/media/keys/ACCC4CF8.asc + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ + https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ + | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + liblz4-dev libzstd-dev zlib1g-dev + + - name: Build, treating warnings as failures + run: | + set -euo pipefail + PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg }}/bin/pg_config + "$PG_CONFIG" --version + make PG_CONFIG="$PG_CONFIG" 2> build.err || { cat build.err; exit 1; } + if grep -q 'warning:' build.err; then + echo "::error::compiler warnings are failures in this project" + grep 'warning:' build.err + exit 1 + fi + cat build.err || true + + - name: Verify the module's symbols resolve against the server + run: | + set -euo pipefail + BIN=$(/usr/lib/postgresql/${{ matrix.pg }}/bin/pg_config --bindir) + missing=$(nm -D --undefined-only pgcolumnar.so \ + | awk '{print $2}' | sort -u \ + | while read -r s; do + nm -D --defined-only "$BIN/postgres" 2>/dev/null \ + | grep -q " $s\$" || echo "$s" + done | grep -vE '^(_|__|GLIBC)' | head -20 || true) + # Informational: the local gate does this strictly, and the packaged + # server exports a different set than a source build, so a difference + # here is reported rather than failed on. + [ -z "$missing" ] && echo "all resolve" || { echo "unresolved (informational):"; echo "$missing"; } + + # Run the suites on one major. This is the behaviour gate; the local + # five-major matrix remains the release gate. + suites: + name: suites (PG 17) + runs-on: ubuntu-latest + needs: build + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Install PostgreSQL 17, codec headers, and pyarrow + run: | + set -euo pipefail + sudo install -d /usr/share/postgresql-common/pgdg + sudo curl -fsSL -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ + https://www.postgresql.org/media/keys/ACCC4CF8.asc + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ + https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ + | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + postgresql-17 postgresql-server-dev-17 \ + liblz4-dev libzstd-dev zlib1g-dev python3-pip + # The Arrow and Parquet suites use pyarrow as an independent reader and + # skip themselves without it, which would be a silent loss of coverage. + pip3 install --break-system-packages --quiet pyarrow || pip3 install --quiet pyarrow + + - name: Stop the packaged cluster + # The suites start their own throwaway clusters on private ports; the + # packaged one is unused and only competes for shared memory. + run: sudo systemctl stop postgresql || true + + - name: Run the suite matrix on PG 17 + run: | + set -euo pipefail + # PGC_SKIP_TIMING drops the three wall-clock suites: a shared runner + # cannot hold a ratio still, and a gate that reds for reasons unrelated + # to the change is worse than one that does not run. They stay in the + # local matrix, which is where those numbers mean anything. + # PGC_JOBS is sized for a 4-core runner rather than the local 8. + sudo -E env "PATH=$PATH" PGC_SKIP_TIMING=1 PGC_JOBS=4 \ + bash test/run_all_versions.sh /usr/lib/postgresql/17/bin/pg_config + + - name: Collect logs on failure + if: failure() + run: | + for f in /tmp/pgcolumnar-matrix-*/*.log; do + [ -e "$f" ] || continue + echo "===== $f =====" + tail -60 "$f" + done diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index c7361e2..e898075 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -228,10 +228,22 @@ for pgc in "${CONFIGS[@]}"; do wait # Then the timing-sensitive suites, one at a time, with nothing else running. + # + # PGC_SKIP_TIMING=1 drops them entirely. That exists for shared CI hardware, + # where the wall-clock ratios these three assert cannot be trusted: a runner + # is noisy by construction and a gate that goes red for reasons unrelated to + # the change teaches its readers to discount red, which is worse than not + # running it. They stay in every local run, which is where the numbers mean + # something. for s in "${SUITES[@]}"; do if ! is_timing_suite "$s"; then continue fi + if [ "${PGC_SKIP_TIMING:-0}" = 1 ]; then + echo " SKIP $s (PGC_SKIP_TIMING)" + echo 0 >"$builddir/${s}.rc" + continue + fi port=$((BASE_PORT++)) PGC_SKIP_BUILD=1 PGC_PORT="$port" \ bash "$builddir/test/${s}.sh" "$pgc" >"$builddir/${s}.log" 2>&1 diff --git a/test/wal_envelope.sh b/test/wal_envelope.sh index d779767..8bd82ca 100755 --- a/test/wal_envelope.sh +++ b/test/wal_envelope.sh @@ -66,19 +66,29 @@ end="$(awk -v s="$start" 'NR > s && /^[A-Za-z_][A-Za-z0-9_]*\(/ {print NR; exit} check "ColumnarTruncateMainFork was found" "$([ -n "$start" ] && echo yes || echo no)" "yes" -# line number of the first match of a pattern inside the function, or 0 +# Line number of the first line inside the function that CONTAINS the literal +# string, or empty. +# +# Deliberately a substring test rather than a regex match. Every pattern below is +# literal C text, and passing one through awk's -v made the backslashes a string +# escape before the regex ever saw them: awk turns "\(" into "(", so +# 'XLogInsert\(RM_SMGR_ID' reached the matcher as 'XLogInsert(RM_SMGR_ID' with an +# unterminated group, and never matched. Whether that happened at all depended on +# the awk build, so this suite passed locally and failed on a runner with a +# different mawk, reporting three steps of the envelope as missing when the code +# was fine. index() has no escaping question to get wrong. at() { awk -v s="$start" -v e="$end" -v pat="$1" \ - 'NR >= s && NR <= e && $0 ~ pat { print NR; exit }' "$FN" + 'NR >= s && NR <= e && index($0, pat) { print NR; exit }' "$FN" } -delay_set="$(at 'delayChkptFlags \|= DELAY_CHKPT_COMPLETE')" -crit_in="$(at 'START_CRIT_SECTION\(\)')" -insert="$(at 'XLogInsert\(RM_SMGR_ID')" -flush="$(at 'XLogFlush\(')" -trunc="$(at 'COLUMNAR_SMGRTRUNCATE\(')" +delay_set="$(at 'delayChkptFlags |= DELAY_CHKPT_COMPLETE')" +crit_in="$(at 'START_CRIT_SECTION()')" +insert="$(at 'XLogInsert(RM_SMGR_ID')" +flush="$(at 'XLogFlush(')" +trunc="$(at 'COLUMNAR_SMGRTRUNCATE(')" delay_clear="$(at 'delayChkptFlags &= ~DELAY_CHKPT_COMPLETE')" -crit_out="$(at 'END_CRIT_SECTION\(\)')" +crit_out="$(at 'END_CRIT_SECTION()')" for v in delay_set crit_in insert flush trunc delay_clear crit_out; do eval "val=\$$v"