use cargo nextest to parallelize CI

.config/nextest.toml

@@ -0,0 +1,32 @@
+# cargo-nextest runs one process per test and schedules across test
+# binaries, so the suite is no longer serialized behind the slowest
+# binary. Ports come from `tests/common/ports.rs`, which reserves them
+# across processes.
+
+[profile.default]
+# Unit tests are cheap and numerous; the cap that matters is on the
+# cluster group below.
+test-threads = "num-cpus"
+# Warn while a test drags, kill only what is clearly wedged. The
+# longest drill (kill_restart, 5 spawn/kill/restart cycles) runs ~100s
+# unloaded.
+slow-timeout = { period = "45s", terminate-after = 8 }
+failure-output = "immediate-final"
+final-status-level = "slow"
+
+# Every integration test stages one or two PostgreSQL clusters plus a
+# ClickHouse server, so their concurrency is bounded by RAM and disk,
+# not by cores.
+[test-groups.clusters]
+max-threads = 4
+
+[[profile.default.overrides]]
+filter = 'kind(test)'
+test-group = 'clusters'
+
+# Longest pole by far (~90s: 5 seeded spawn/kill/restart cycles each).
+# Started last it sets the wall clock on its own; started first it hides
+# under everything else.
+[[profile.default.overrides]]
+filter = 'binary(kill_restart)'
+priority = 50

.github/workflows/ci.yml

@@ -30,18 +30,6 @@ jobs:
         pg-major: [16, 17, 18]
 
     steps:
-      # ubuntu-latest ships ~14GB free on /; `cargo --all-targets` links
-      # every integration test as its own static binary and can exhaust it
-      # mid-link. ENOSPC surfaces as an lld SIGBUS + a spurious "report to
-      # LLVM" banner. Fast `rm -rf` of the big preinstalled dirs
-      # (android/dotnet/haskell, ~21GB) buys enough room; skip large-packages
-      # (slow apt remove, minutes) and swap-storage (frees /mnt, not /).
-      - name: Free disk space
-        uses: jlumbroso/free-disk-space@v1.3.1
-        with:
-          large-packages: false
-          swap-storage: false
-
       - name: Checkout walshadow
         uses: actions/checkout@v7
         with:
@@ -53,6 +41,13 @@ jobs:
         with:
           components: clippy, rustfmt
 
+      # nextest runs one process per test and schedules across test
+      # binaries; `cargo test` runs binaries one at a time.
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
+
       - name: Cargo cache
         uses: Swatinem/rust-cache@v2
         with:
@@ -164,17 +159,18 @@ jobs:
       # + wal_stream_chunk_boundary (use the freshly regenerated fixtures),
       # clickhouse-c-rs clickhouse_local + readme_quickstarts (need
       # clickhouse binary + server on :9000).
-      - name: Test (workspace, single-threaded — each test owns a PG cluster)
+      #
+      # Concurrency comes from `.config/nextest.toml`: unit tests run
+      # `num-cpus` wide, cluster drills are capped by the `clusters`
+      # test group. No `--no-capture`, it would force one thread.
+      - name: Test (workspace)
         working-directory: walshadow
         env:
-          # Tests pick non-overlapping ports but PG cluster startup is heavy;
-          # avoid thrashing the runner by serializing.
-          RUST_TEST_THREADS: "1"
           # Opts tests with `fx::dump_artifacts` into copying their tempdir
           # here on failure. Empty on success; uploaded as a job artifact
           # by the next step when present.
           WALSHADOW_ARTIFACT_DIR: ${{ github.workspace }}/test-artifacts
-        run: cargo test --workspace --all-targets --locked -- --nocapture
+        run: cargo nextest run --workspace --all-targets --locked --no-fail-fast
 
       - name: Upload test artifacts on failure
         if: failure()
@@ -201,6 +197,12 @@ jobs:
     timeout-minutes: 60
 
     steps:
+      # Coverage-only: instrumented binaries carry a counter table per
+      # test target and every test process drops its own `.profraw`, so
+      # this job's disk draw dwarfs the matrix jobs' (which run without
+      # any cleanup). Frees android/dotnet/haskell (~20GB); skip
+      # large-packages (slow apt remove, minutes) and swap-storage
+      # (frees /mnt, not /).
       - name: Free disk space
         uses: jlumbroso/free-disk-space@v1.3.1
         with:
@@ -220,10 +222,10 @@ jobs:
           # this component.
           components: llvm-tools-preview
 
-      - name: Install cargo-llvm-cov
+      - name: Install cargo-llvm-cov + cargo-nextest
         uses: taiki-e/install-action@v2
         with:
-          tool: cargo-llvm-cov
+          tool: cargo-llvm-cov,cargo-nextest
 
       - name: Cargo cache
         uses: Swatinem/rust-cache@v2
@@ -311,12 +313,11 @@ jobs:
       - name: Run instrumented test suite (cargo llvm-cov)
         working-directory: walshadow
         env:
-          RUST_TEST_THREADS: "1"
           WALSHADOW_ARTIFACT_DIR: ${{ github.workspace }}/test-artifacts
         run: |
           cargo llvm-cov clean --workspace
-          cargo llvm-cov --workspace --all-targets --locked --no-report \
-            -- --nocapture
+          cargo llvm-cov nextest --workspace --all-targets --locked \
+            --no-report --no-fail-fast
 
       - name: Upload test artifacts on failure
         if: failure()

README.md

@@ -132,7 +132,7 @@ params stay boot-only
 
 ```
 make -C pgext
-cargo test
+cargo nextest run --workspace --all-targets
 cargo clippy --all-targets -- -D warnings
 ```
 
@@ -143,6 +143,13 @@ without it. Walshadow-side timeouts are
 seconds-scale by design — long timeouts mask stalls rather than
 surface them
 
+CI runs the suite through [cargo-nextest](https://nexte.st), which
+schedules across test binaries; `cargo test` still works but runs
+binaries one at a time. Concurrency limits live in
+`.config/nextest.toml`. Tests reserve TCP ports through
+`tests/common/ports.rs`, so parallel runs — including several at once on
+one machine — do not collide
+
 ## Repository layout
 
 ```

plans/ops.md

@@ -15,8 +15,8 @@ is to make long-running daemon survivable, observable, resumable
   debug window, drop older ones to bound disk
 - Persist resume state (six LSNs + resolved floor + source identity)
   across `kill -9` so daemon restart hands source's slot byte-identical
-  write/flush/apply triple, and `cargo test --test kill_restart` proves
-  end-state parity over 15 seeded kill/restart cycles
+  write/flush/apply triple, and `cargo nextest run -E 'test(kill_restart)'`
+  proves end-state parity over 15 seeded kill/restart cycles
 
 ## Preflight validators
 
@@ -390,9 +390,10 @@ persist between kill and restart
 ## Kill-restart drill
 
 [`tests/kill_restart.rs`](../tests/kill_restart.rs). Three cutoff
-strategies × five seeded windows = 15 daemon spawn/kill/restart cycles
-per CI invocation. Source PG + CH server + basebackup-cloned shadow
-stand up once, daemon cycles inside
+strategies, one test each, five seeded windows per test = 15 daemon
+spawn/kill/restart cycles per CI invocation. Each test stands up its own
+source PG + CH server + basebackup-cloned shadow, daemon cycles inside,
+so the three run concurrently
 
 Strategies:
 
@@ -418,15 +419,16 @@ assert CH `count + sum(id) + md5(string_agg(name, ',' ORDER BY id))`
 matches source
 
 `WALSHADOW_KILL_SEED` env (default `0xC11AC11A`) seeds inline
-splitmix-style LCG so CI is reproducible. Per-(strategy, run) seed
-derivative shifts 250-750 ms kill window within each strategy. Nightly
-rotation across seeds surfaces rare-window bugs
+splitmix-style LCG so CI is reproducible. Strategy is folded into the
+base seed and the per-run derivative shifts the 250-750 ms kill window,
+so the three strategies walk different windows. Nightly rotation across
+seeds surfaces rare-window bugs
 
 Test is NOT `#[ignore]`. Uses runtime skip-gates checking
 `fx::pg_available()` / `fx::pg_basebackup_available()` /
 `fx::clickhouse_available()` — silently `return` when binaries are
 absent, panics on actual failure when present (switched away from
-`#[ignore]` so default `cargo test` exercises drill on any dev box with
+`#[ignore]` so a default test run exercises drill on any dev box with
 PG + CH on PATH)
 
 Source pins `wal_keep_size = '128MB'` so 250-750 ms of WAL stays

tests/add_column_default.rs

@@ -28,12 +28,6 @@ use walshadow::mapping::ColumnMapping;
 use walshadow::mapping::TableTarget;
 use walshadow::schema::RelName;
 
-const SOURCE_PORT: u16 = 17401;
-const SHADOW_PORT: u16 = 17402;
-const CH_TCP_PORT: u16 = 17409;
-const CH_HTTP_PORT: u16 = 17410;
-const WALSENDER_PORT: u16 = 17450;
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn add_column_default_replicates_pre_alter_default() {
     if !fx::pg_available() {
@@ -49,6 +43,7 @@ async fn add_column_default_replicates_pre_alter_default() {
         return;
     }
 
+    let slot = fx::Ports::alloc();
     let tmp = tempfile::tempdir().unwrap();
     let (
         fx::BootstrappedClusters {
@@ -62,16 +57,16 @@ async fn add_column_default_replicates_pre_alter_default() {
         "CREATE SCHEMA s14;\n\
          CREATE TABLE s14.t (id bigint PRIMARY KEY, payload text);\n\
          ALTER TABLE s14.t REPLICA IDENTITY FULL;\n",
-        SOURCE_PORT,
-        SHADOW_PORT,
-        WALSENDER_PORT,
+        slot.source,
+        slot.shadow,
+        slot.walsender,
     )
     .await;
     let _src_stop = fx::StopOnDrop { sh: &source };
     let _shd_stop = fx::StopOnDrop { sh: &shadow };
 
     let ch_tmp = tempfile::tempdir().unwrap();
-    let ch = fx::ChServer::spawn(ch_tmp, CH_TCP_PORT, CH_HTTP_PORT).expect("spawn ch");
+    let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch");
     ch.query("CREATE DATABASE IF NOT EXISTS walshadow_test")
         .expect("create db");
     ch.query(
@@ -115,7 +110,7 @@ async fn add_column_default_replicates_pre_alter_default() {
         shadow_filter_dir: &shadow_filter_dir,
         shadow_stream_state,
         ch_database: "walshadow_test",
-        ch_tcp_port: CH_TCP_PORT,
+        ch_tcp_port: slot.ch_tcp,
         mappings,
         app_name: "walshadow-add-col-default",
         ddl: None,