fix: keep the web token out of the systemd journal, and verify the unit for real

.github/workflows/ci.yml

@@ -114,10 +114,13 @@ jobs:
       - name: shellcheck (pinned)
         run: |
           docker run --rm -v "$PWD":/mnt:ro koalaman/shellcheck:v0.11.0 --version
-          docker run --rm -v "$PWD":/mnt:ro koalaman/shellcheck:v0.11.0 /mnt/scripts/install.sh
+          docker run --rm -v "$PWD":/mnt:ro koalaman/shellcheck:v0.11.0 \
+            /mnt/scripts/install.sh /mnt/scripts/verify-systemd-unit.sh
 
       - name: bash -n
-        run: bash -n scripts/install.sh
+        run: |
+          bash -n scripts/install.sh
+          bash -n scripts/verify-systemd-unit.sh
 
   installer:
     name: Installer on ${{ matrix.image }}
@@ -219,3 +222,75 @@ jobs:
             echo "::error::refusal installed $pkg_delta package(s); it must leave the host untouched"
             exit 1
           fi
+
+  systemd-unit:
+    name: systemd unit
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # `systemd-analyze verify` only parses the unit file. It cannot tell you that
+      # ProtectHome/ProtectSystem still leave the token writable, that
+      # Restart=on-failure recovers the service, that the token stays out of the
+      # journal, or that the thing serves at all. Those need systemd to really run
+      # it, so this boots a container with systemd as PID 1.
+      #
+      # Reproduce locally with the same three steps: build the image, run it
+      # privileged with the cgroup mount, then exec the install and the script.
+      - name: Build a systemd-capable image
+        run: |
+          set -euo pipefail
+          cat > /tmp/Dockerfile <<'DOCKERFILE'
+          FROM debian:12
+          RUN apt-get update && apt-get install -y --no-install-recommends \
+                  systemd systemd-sysv python3 python3-venv python3-pip ca-certificates \
+              && rm -rf /var/lib/apt/lists/*
+          STOPSIGNAL SIGRTMIN+3
+          CMD ["/sbin/init"]
+          DOCKERFILE
+          docker build -t replicant-systemd -f /tmp/Dockerfile /tmp
+
+      - name: Boot systemd and install Replicant
+        run: |
+          set -euo pipefail
+          docker run -d --name rsd --privileged \
+            --tmpfs /run --tmpfs /run/lock \
+            -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host \
+            replicant-systemd /sbin/init
+          timeout 120 bash -c \
+            'until docker exec rsd systemctl is-system-running --wait 2>/dev/null \
+               | grep -qE "running|degraded"; do sleep 2; done'
+
+          # The unit's WorkingDirectory is a repository checkout and the install is
+          # editable, because the catalog and the frontend live outside the package.
+          tar --exclude=./.git --exclude=./webui/node_modules --exclude=./.venv -cf - . \
+            | docker exec -i rsd bash -c 'mkdir -p /opt/replicant && tar -xf - -C /opt/replicant'
+
+          # No npm build here: these assertions are about systemd running the unit,
+          # not about the frontend, and every endpoint they touch is served either way.
+          docker exec rsd bash -c '
+            set -e
+            useradd --system --home-dir /opt/replicant --shell /usr/sbin/nologin replicant
+            cd /opt/replicant
+            python3 -m venv .venv
+            ./.venv/bin/pip install -q -e ".[web]"
+            chown -R replicant:replicant /opt/replicant'
+
+      - name: Enable the unit and assert on it
+        run: |
+          set -euo pipefail
+          docker exec rsd bash -c '
+            set -e
+            cp /opt/replicant/scripts/replicant-web.service /etc/systemd/system/
+            systemctl daemon-reload
+            systemctl enable --now replicant-web'
+          sleep 5
+          docker exec rsd bash /opt/replicant/scripts/verify-systemd-unit.sh
+
+      - name: Unit journal on failure
+        if: failure()
+        run: docker exec rsd journalctl -u replicant-web --no-pager -l || true
+
+      - name: Tear down
+        if: always()
+        run: docker rm -f rsd || true

CHANGELOG.md

@@ -38,9 +38,26 @@ every restart and reaching it from another machine meant an SSH tunnel. Driven b
 - **The terminal tab is off by default on a non-loopback bind**, restored with
   `--enable-terminal`, and reported to the frontend so the tab is hidden rather
   than offered and broken.
-- **`scripts/replicant-web.service`**, a systemd unit template. [Unverified]: not
-  yet started by a real systemd. Verify with
-  `systemd-analyze verify scripts/replicant-web.service` on a Linux host.
+- **`scripts/replicant-web.service`**, a systemd unit template. **Verified** against
+  a real systemd (Debian 12, systemd 252, PID 1 in a container): the unit starts,
+  runs as a non-root service user, serves, refuses an unauthenticated request,
+  writes its token 0600 under `ProtectHome=read-only`, disables the terminal tab on
+  its `0.0.0.0` bind, and recovers from `SIGKILL` via `Restart=on-failure` with the
+  token intact. `scripts/verify-systemd-unit.sh` is those assertions, and CI runs
+  them on every push (job `systemd-unit`), so the unit cannot rot silently.
+
+### Security: the token no longer reaches the systemd journal
+
+Running the unit for real found a defect that no amount of reading it would have.
+The startup banner printed the full URL including `?token=...`. Interactively that
+is the point. Under systemd, **stdout is the journal**, so every start wrote the
+token in cleartext into a file readable by root and the systemd-journal group,
+giving away precisely what the `0600` token file protects.
+
+The banner now prints the token only when stdout is a terminal. Otherwise it prints
+the URL without it and names the file to read it from, and drops the "stop: Ctrl-C"
+hint, since nobody is at a keyboard. Covered by three unit tests and by an assertion
+in `scripts/verify-systemd-unit.sh` that greps the journal for the live token.
 
 ### Security: what replaces the loopback bind
 

README.md

@@ -283,8 +283,11 @@ To run it as a service, `scripts/replicant-web.service` is a systemd unit templa
 sudo cp scripts/replicant-web.service /etc/systemd/system/
 sudo systemctl daemon-reload
 sudo systemctl enable --now replicant-web
+sudo cat /opt/replicant/.config/replicant/web-token   # the token to open the UI with
 ```
 
+The banner prints the token only when it is attached to a terminal. Under systemd, stdout is the journal, and `journalctl` is readable by root and the systemd-journal group, so the token is deliberately kept out of it and read from the file instead. CI runs the unit under a real systemd on every push and asserts exactly that, along with the restart behaviour and the token's permissions.
+
 > **Keeping it loopback-only.** That is still the default: plain `replicant web` binds 127.0.0.1 and nothing else. To reach a loopback-only instance from your workstation, tunnel it rather than rebinding: `ssh -N -L 9787:127.0.0.1:9787 operator@sensor`.
 
 A run streams live CEF while it emits, plots the delivered rate, and writes its manifest when it finishes. The rate here runs well above the 2000 eps cap shown because this run has no collector: the cap governs sending, so a dry run or a file-only run is not throttled.

replicant/web/server.py

@@ -65,6 +65,7 @@
     load_or_create_web_token,
     parse_anchor,
     stale_anchor_warning,
+    web_token_path,
 )
 from replicant.core.models import Catalog, CollectorProfile, Intensity, RunRequest, Transport
 from replicant.core.orchestrator import Orchestrator, effective_identity
@@ -733,19 +734,32 @@ def startup_lines(
     token: str | None,
     token_state: str,
     terminal: bool,
+    reveal_token: bool = True,
+    token_path: str | None = None,
 ) -> list[str]:
-    """The startup banner: URL, then token state, then terminal state."""
+    """The startup banner: URL, then token state, then terminal state.
+
+    ``reveal_token`` is False when stdout is not a terminal, which under systemd
+    means the journal. Printing the token there writes it in cleartext to a file
+    readable by root and the systemd-journal group, giving away precisely what the
+    0600 token file protects. In that case the banner names the file instead, and
+    drops the Ctrl-C hint, since nobody is at a keyboard.
+    """
     wildcard = _normalize_host(host) in _WILDCARD_BINDS
-    url = display_url(host, port, token)
+    url = display_url(host, port, token if reveal_token else None)
     bind = f"{host}:{port}" + (" (all interfaces)" if wildcard else "")
+    token_line = token_state
+    if token and not reveal_token:
+        token_line = f"{token_state}, read it from {token_path or web_token_path()}"
     lines = [
         "Replicant web UI",
         f"  URL      : {url}",
         f"  bind     : {bind}",
-        f"  token    : {token_state}",
+        f"  token    : {token_line}",
         f"  terminal : {'enabled' if terminal else 'disabled (--enable-terminal to allow)'}",
-        "  stop     : Ctrl-C",
     ]
+    if reveal_token:
+        lines.append("  stop     : Ctrl-C")
     if wildcard:
         lines.insert(
             2, f"  remote   : http://<this host's address>:{port}/ from the rest of the segment"
@@ -783,12 +797,17 @@ def serve(
     sock = bind_socket(host, port)
     bound_port = int(sock.getsockname()[1])
 
+    # A non-tty stdout means something is capturing this: under systemd, the
+    # journal. The token must not go there.
+    reveal = sys.stdout.isatty()
     for line in startup_lines(
         host,
         bound_port,
         token=token or None,
         token_state=token_state,
         terminal=policy.terminal_enabled,
+        reveal_token=reveal,
+        token_path=str(web_token_path()),
     ):
         print(line, flush=True)
 

scripts/replicant-web.service

@@ -7,7 +7,11 @@
 #   sudo systemctl enable --now replicant-web
 #   systemctl status replicant-web
 #
-# The token persists across restarts, so the URL stays valid. Print it with:
+# The token persists across restarts, so the URL stays valid. It is deliberately
+# NOT printed in the startup banner when stdout is not a terminal, which under
+# systemd means the journal: `journalctl` is readable by root and the
+# systemd-journal group, so printing it there would give away exactly what the
+# 0600 token file protects. Read it from the file instead:
 #
 #   sudo cat /opt/replicant/.config/replicant/web-token
 #

scripts/verify-systemd-unit.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+# Copyright 2026 Imran Hafeez (RZA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Assertions for scripts/replicant-web.service, run INSIDE a container that has a
+# real systemd as PID 1 and the unit already enabled.
+#
+# `systemd-analyze verify` only parses the unit. It cannot tell you that the
+# sandboxing directives still let the token be written, that Restart=on-failure
+# actually recovers the service, that the token stays out of the journal, or that
+# the thing serves once it is up. Those need systemd to really run it.
+#
+# Driven by .github/workflows/ci.yml (job: systemd-unit), which also documents how
+# to reproduce the container locally.
+#
+# Comparisons go through `expect` rather than `A && B || C`: shellcheck flags that
+# form (SC2015) because C also runs when A is false, and scripts/install.sh already
+# settled on spelling these out.
+set -uo pipefail
+
+FAILED=0
+
+pass() { printf 'PASS  %s\n' "$1"; }
+
+fail() {
+  printf 'FAIL  %s\n' "$1"
+  if [[ -n "${2:-}" ]]; then
+    printf '      %s\n' "$2"
+  fi
+  FAILED=1
+}
+
+# expect <label> <actual> <expected> [detail-on-failure]
+expect() {
+  if [[ "$2" == "$3" ]]; then
+    pass "$1"
+  else
+    fail "$1" "got '$2', expected '$3'${4:+ | $4}"
+  fi
+}
+
+PORT="${REPLICANT_WEB_PORT:-9787}"
+UNIT=replicant-web
+CFG=/opt/replicant/.config/replicant
+
+probe() {
+python3 - "$1" <<'PY'
+import sys, urllib.error, urllib.request
+try:
+    with urllib.request.urlopen(sys.argv[1], timeout=5) as response:
+        print(response.status)
+except urllib.error.HTTPError as exc:
+    print(exc.code)
+except Exception as exc:  # noqa: BLE001 - any transport failure is a non-answer
+    print("ERR %s" % exc)
+PY
+}
+
+# 1. systemd's own parser accepts it, with no warnings.
+if out="$(systemd-analyze verify "/etc/systemd/system/$UNIT.service" 2>&1)"; then
+  expect "systemd-analyze verify is silent" "${out:-}" ""
+else
+  fail "systemd-analyze verify rejected the unit" "$out"
+fi
+
+# 2. The service reached running.
+expect "unit is active" "$(systemctl is-active "$UNIT" 2>&1)" "active" \
+  "$(systemctl status "$UNIT" --no-pager -l 2>&1 | tail -5)"
+
+# 3. Not running as root. /proc rather than ps, which a minimal image lacks.
+main_pid="$(systemctl show -p MainPID --value "$UNIT")"
+if [[ "$main_pid" =~ ^[0-9]+$ ]] && (( main_pid > 0 )); then
+  expect "runs as the service user, not root" \
+    "$(stat -c '%U' "/proc/$main_pid" 2>/dev/null)" "replicant"
+else
+  fail "no MainPID" "unit did not reach a running state"
+fi
+
+# 4/5. It serves, and it refuses an unauthenticated call. A unit that brought up an
+# open server would otherwise look identical to a working one.
+expect "/api/health answers 200" "$(probe "http://127.0.0.1:$PORT/api/health")" "200"
+expect "/api/catalog refuses an unauthenticated request" \
+  "$(probe "http://127.0.0.1:$PORT/api/catalog")" "401"
+
+# 6. The token was written despite ProtectHome=read-only and ProtectSystem=full.
+# This is the pairing that breaks if the unit ever moves config back under the
+# service user's home directory.
+if [[ -f "$CFG/web-token" ]]; then
+  expect "token written 0600 under the sandbox" "$(stat -c '%a' "$CFG/web-token")" "600"
+  expect "config dir is 0700" "$(stat -c '%a' "$CFG")" "700"
+else
+  fail "no token file at $CFG/web-token" "$(systemctl status "$UNIT" --no-pager -l 2>&1 | tail -5)"
+fi
+
+# 7. The token is NOT in the journal. Under systemd, stdout IS the journal, so a
+# banner that prints the token writes it in cleartext to a file readable by root
+# and the systemd-journal group, giving away what the 0600 file protects.
+token="$(cat "$CFG/web-token" 2>/dev/null || echo)"
+if [[ -n "$token" ]]; then
+  if journalctl -u "$UNIT" --no-pager -o cat 2>/dev/null | grep -qF "$token"; then
+    fail "token appears in the journal" "the startup banner must not print it off a tty"
+  else
+    pass "token is absent from the journal"
+  fi
+fi
+
+# 8. Terminal tab off by default, because ExecStart binds 0.0.0.0.
+if [[ -n "$token" ]]; then
+  term="$(python3 - "$PORT" "$token" <<'PY'
+import json, sys, urllib.request
+request = urllib.request.Request("http://127.0.0.1:%s/api/config" % sys.argv[1])
+request.add_header("Authorization", "Bearer %s" % sys.argv[2])
+try:
+    with urllib.request.urlopen(request, timeout=5) as response:
+        print(json.load(response)["terminal_enabled"])
+except Exception as exc:  # noqa: BLE001
+    print("ERR %s" % exc)
+PY
+)"
+  expect "terminal tab disabled on the 0.0.0.0 bind" "$term" "False"
+fi
+
+# 9. Restart=on-failure really recovers it. Wait for the condition that matters,
+# serving again, not merely a changed MainPID: systemd sets that as soon as it
+# forks, and RestartSec=5s puts the socket several seconds behind it. Asserting on
+# the PID alone reported the unit broken when it was still starting.
+before="$(systemctl show -p MainPID --value "$UNIT")"
+kill -9 "$before" 2>/dev/null
+now=""
+recovered=0
+for _ in $(seq 1 60); do
+  sleep 0.5
+  now="$(systemctl show -p MainPID --value "$UNIT")"
+  if [[ ! "$now" =~ ^[0-9]+$ ]] || (( now == 0 )) || [[ "$now" == "$before" ]]; then
+    continue
+  fi
+  if [[ "$(probe "http://127.0.0.1:$PORT/api/health")" == "200" ]]; then
+    recovered=1
+    break
+  fi
+done
+if (( recovered )); then
+  pass "Restart=on-failure recovered it and it serves again ($before -> $now)"
+else
+  fail "did not recover after SIGKILL within 30s" \
+    "$(systemctl status "$UNIT" --no-pager -l 2>&1 | tail -10)"
+fi
+
+# 10. The token survives the restart, which is the point of persisting it.
+expect "token survived the restart" "$(cat "$CFG/web-token" 2>/dev/null || echo)" "$token"
+
+printf '\n'
+if (( FAILED )); then
+  printf 'RESULT: FAILURES PRESENT\n'
+  exit 1
+fi
+printf 'RESULT: ALL CHECKS PASSED\n'