test: an involuntary skip is a failure — tools/_skip_policy.py and the rig lane (#178a)

pytest.ini

@@ -50,7 +50,9 @@ testpaths =
     tools/test_package_verify.py
     tools/test_pytest_boundary.py
     tools/test_reserved_test_host.py
+    tools/test_rig_skip_contract.py
     tools/test_runner_coverage.py
+    tools/test_skip_policy.py
     tools/test_uci_data_acc.py
 
 norecursedirs = .git libs ip65 ip65-build build dist tests tools/uci

tests/rig_phase1_dhcp.py

@@ -9,10 +9,16 @@
 Run:
     PYTHONPATH=tools python3 tests/rig_phase1_dhcp.py
 
-Exit codes:
+Exit codes (tools/_skip_policy.py, issue #178):
     0 -- PASS
-    0 -- SKIP (clearly printed)
-    1 -- FAIL
+    1 -- FAIL (a check ran and failed)
+    0 -- NOT APPLICABLE: this rig is Linux-only, and on any other host
+         tests/rig_vice_https_macos.py owns the coverage.  A named verdict,
+         never a bare skip.
+    2 -- COULD NOT RUN (a prerequisite is missing ON LINUX, or the build is
+         broken -- nothing was verified).  Set C64_ALLOW_SKIP=1 to accept a
+         prerequisite-missing run as exit 0; a FAILED BUILD is never opted
+         out of.
 """
 
 from __future__ import annotations
@@ -27,6 +33,9 @@
 if _TOOLS not in sys.path:
     sys.path.insert(0, _TOOLS)
 
+# needs _TOOLS on sys.path, hence the placement below the block above
+from _skip_policy import cannot_run, not_applicable  # noqa: E402
+
 PRG_PATH = os.path.join(_REPO_ROOT, "build", "c64-https.prg")
 
 # Exact literal from src/boot.asm (menu_msg @ line 424-426).
@@ -38,9 +47,42 @@
 DHCP_TIMEOUT = 90.0
 
 
-def _skip(reason: str) -> int:
-    print(f"SKIP: {reason}")
-    return 0
+_CERTIFIES = "ip65 net_dhcp_acquire end-to-end in VICE"
+
+
+def _cannot_run(reason: str, *, opt_out: bool = True) -> int:
+    """An involuntary skip is a FAILURE -- nothing was verified (issue #178).
+
+    `opt_out=False` for a broken build: a failed `make` is never laundered
+    into a pass, not even by C64_ALLOW_SKIP.
+    """
+    return cannot_run(
+        reason,
+        executed=0,
+        total=1,
+        certifies=_CERTIFIES,
+        opt_out_env="C64_ALLOW_SKIP" if opt_out else None,
+    )
+
+
+_COUNTERPART = (
+    "tests/rig_vice_https_macos.py owns this coverage on macOS -- its handshake begins with the same ip65 DHCP acquisition"
+)
+
+
+def _wrong_platform() -> int:
+    """A VOLUNTARY skip: this host can never run this rig (issue #178).
+
+    The load-bearing question is what the remedy is.  "install iproute2" is
+    an involuntary skip and stays exit 2 -- but on a non-Linux host there is
+    no remedy at all, and another rig owns the coverage, so nothing is lost
+    and exit 2 would be a red nobody can ever clear.
+    """
+    return not_applicable(
+        f"this rig is Linux-only (br-c64 bridge + netfilter + /proc/net/udp); "
+        f"this host is {sys.platform} -- {_COUNTERPART}",
+        certifies=_CERTIFIES,
+    )
 
 
 def _ensure_built() -> bool:
@@ -59,19 +101,28 @@ def main() -> int:
     from https_e2e import (
         BridgeEnv,
         check_prerequisites,
+        platform_supported,
         launch_vice_on_bridge,
         shutdown_vice,
         press_key,
         wait_for_screen_text,
         get_screen_text,
     )
 
+    # Platform FIRST: check_prerequisites() cannot answer this, because the
+    # same string ("ip not on PATH") means "installable" on Linux and "wrong
+    # OS" everywhere else (issue #178).
+    if not platform_supported():
+        return _wrong_platform()
+
     missing = check_prerequisites()
     if missing:
-        return _skip("missing prerequisites: " + "; ".join(missing))
+        return _cannot_run("missing prerequisites: " + "; ".join(missing))
 
     if not _ensure_built():
-        return _skip("c64-https.prg could not be built")
+        return _cannot_run("c64-https.prg could not be built -- `make` "
+                           "failed; this is a broken build, not a "
+                           "missing prerequisite", opt_out=False)
 
     # ---- Run the test ------------------------------------------------------
     handle = None

tests/rig_phase2_http.py

@@ -9,10 +9,21 @@
 Run:
     sudo PYTHONPATH=tools python3 tests/rig_phase2_http.py
 
-Exit codes:
+Exit codes (tools/_skip_policy.py, issue #178):
     0 -- PASS
-    0 -- SKIP (clearly printed)
-    1 -- FAIL
+    1 -- FAIL (a check ran and failed)
+    0 -- NOT APPLICABLE: this rig is Linux-only.  A named verdict, never a
+         bare skip -- and, uniquely among the four bridge rigs, a verdict
+         that DOES cost coverage: tests/rig_vice_https_macos.py is the
+         macOS counterpart for the emulated-RR-Net path, but it drives it
+         over TLS, so plaintext HTTP specifically has no macOS rig.  Exit 0
+         anyway because there is no remedy on this platform (see
+         _wrong_platform), not because nothing is lost.  Say so out loud
+         rather than let the exit code imply otherwise.
+    2 -- COULD NOT RUN (a prerequisite is missing ON LINUX, or the build is
+         broken -- nothing was verified).  Set C64_ALLOW_SKIP=1 to accept a
+         prerequisite-missing run as exit 0; a FAILED BUILD is never opted
+         out of.
 """
 
 from __future__ import annotations
@@ -27,6 +38,9 @@
 if _TOOLS not in sys.path:
     sys.path.insert(0, _TOOLS)
 
+# needs _TOOLS on sys.path, hence the placement below the block above
+from _skip_policy import cannot_run, not_applicable  # noqa: E402
+
 PRG_PATH = os.path.join(_REPO_ROOT, "build", "c64-https.prg")
 
 # Screen needles (from src/boot.asm string labels).
@@ -40,9 +54,48 @@
 HTTP_TIMEOUT = 120.0
 
 
-def _skip(reason: str) -> int:
-    print(f"SKIP: {reason}")
-    return 0
+_CERTIFIES = "the plaintext HTTP path over emulated RR-Net"
+
+
+def _cannot_run(reason: str, *, opt_out: bool = True) -> int:
+    """An involuntary skip is a FAILURE -- nothing was verified (issue #178).
+
+    `opt_out=False` for a broken build: a failed `make` is never laundered
+    into a pass, not even by C64_ALLOW_SKIP.
+    """
+    return cannot_run(
+        reason,
+        executed=0,
+        total=1,
+        certifies=_CERTIFIES,
+        opt_out_env="C64_ALLOW_SKIP" if opt_out else None,
+    )
+
+
+_COUNTERPART = (
+    "tests/rig_vice_https_macos.py is the macOS counterpart for the emulated-RR-Net path; note it drives it over TLS, so plaintext HTTP specifically has no macOS rig"
+)
+
+
+def _wrong_platform() -> int:
+    """A VOLUNTARY skip: this host can never run this rig (issue #178).
+
+    The load-bearing question is what the remedy is.  "install iproute2" is
+    an involuntary skip and stays exit 2 -- but on a non-Linux host there is
+    no remedy at all, so exit 2 would be a red nobody can ever clear.
+
+    Note what this rig does NOT get to say, and what its three siblings do:
+    that another rig owns the coverage.  For DHCP and for HTTPS the macOS
+    rig genuinely re-runs the same path; for PLAINTEXT HTTP over emulated
+    RR-Net it does not, so this verdict really does lose coverage on macOS.
+    That is disclosed here and in _COUNTERPART rather than smoothed over --
+    a voluntary skip may be quiet, but it may not misdescribe what it costs.
+    """
+    return not_applicable(
+        f"this rig is Linux-only (br-c64 bridge + netfilter + /proc/net/udp); "
+        f"this host is {sys.platform} -- {_COUNTERPART}",
+        certifies=_CERTIFIES,
+    )
 
 
 def _ensure_built() -> bool:
@@ -97,6 +150,7 @@ def main() -> int:
     from https_e2e import (
         BridgeEnv,
         check_prerequisites,
+        platform_supported,
         launch_vice_on_bridge,
         shutdown_vice,
         press_key,
@@ -106,12 +160,20 @@ def main() -> int:
         stop_http_listener,
     )
 
+    # Platform FIRST: check_prerequisites() cannot answer this, because the
+    # same string ("ip not on PATH") means "installable" on Linux and "wrong
+    # OS" everywhere else (issue #178).
+    if not platform_supported():
+        return _wrong_platform()
+
     missing = check_prerequisites()
     if missing:
-        return _skip("missing prerequisites: " + "; ".join(missing))
+        return _cannot_run("missing prerequisites: " + "; ".join(missing))
 
     if not _ensure_built():
-        return _skip("c64-https.prg could not be built")
+        return _cannot_run("c64-https.prg could not be built -- `make` "
+                           "failed; this is a broken build, not a "
+                           "missing prerequisite", opt_out=False)
 
     handle = None
     listener = None

tests/rig_phase3_https.py

@@ -14,10 +14,16 @@
 Run:
     sudo PYTHONPATH=tools python3 tests/rig_phase3_https.py
 
-Exit codes:
+Exit codes (tools/_skip_policy.py, issue #178):
     0 -- PASS
-    0 -- SKIP (clearly printed)
-    1 -- FAIL
+    1 -- FAIL (a check ran and failed)
+    0 -- NOT APPLICABLE: this rig is Linux-only, and on any other host
+         tests/rig_vice_https_macos.py owns the coverage.  A named verdict,
+         never a bare skip.
+    2 -- COULD NOT RUN (a prerequisite is missing ON LINUX, or the build is
+         broken -- nothing was verified).  Set C64_ALLOW_SKIP=1 to accept a
+         prerequisite-missing run as exit 0; a FAILED BUILD is never opted
+         out of.
 """
 
 from __future__ import annotations
@@ -32,6 +38,9 @@
 if _TOOLS not in sys.path:
     sys.path.insert(0, _TOOLS)
 
+# needs _TOOLS on sys.path, hence the placement below the block above
+from _skip_policy import cannot_run, not_applicable  # noqa: E402
+
 PRG_PATH = os.path.join(_REPO_ROOT, "build", "c64-https.prg")
 
 # Screen needles (from src/boot.asm string labels).
@@ -82,9 +91,42 @@
 HTTPS_TIMEOUT = 1800.0
 
 
-def _skip(reason: str) -> int:
-    print(f"SKIP: {reason}")
-    return 0
+_CERTIFIES = "the TLS 1.3 handshake + GET over emulated RR-Net"
+
+
+def _cannot_run(reason: str, *, opt_out: bool = True) -> int:
+    """An involuntary skip is a FAILURE -- nothing was verified (issue #178).
+
+    `opt_out=False` for a broken build: a failed `make` is never laundered
+    into a pass, not even by C64_ALLOW_SKIP.
+    """
+    return cannot_run(
+        reason,
+        executed=0,
+        total=1,
+        certifies=_CERTIFIES,
+        opt_out_env="C64_ALLOW_SKIP" if opt_out else None,
+    )
+
+
+_COUNTERPART = (
+    "tests/rig_vice_https_macos.py owns this coverage on macOS"
+)
+
+
+def _wrong_platform() -> int:
+    """A VOLUNTARY skip: this host can never run this rig (issue #178).
+
+    The load-bearing question is what the remedy is.  "install iproute2" is
+    an involuntary skip and stays exit 2 -- but on a non-Linux host there is
+    no remedy at all, and another rig owns the coverage, so nothing is lost
+    and exit 2 would be a red nobody can ever clear.
+    """
+    return not_applicable(
+        f"this rig is Linux-only (br-c64 bridge + netfilter + /proc/net/udp); "
+        f"this host is {sys.platform} -- {_COUNTERPART}",
+        certifies=_CERTIFIES,
+    )
 
 
 def _ensure_built() -> bool:
@@ -458,6 +500,7 @@ def main() -> int:
     from https_e2e import (
         BridgeEnv,
         check_prerequisites,
+        platform_supported,
         launch_vice_on_bridge,
         shutdown_vice,
         press_key,
@@ -467,12 +510,20 @@ def main() -> int:
         stop_https_listener,
     )
 
+    # Platform FIRST: check_prerequisites() cannot answer this, because the
+    # same string ("ip not on PATH") means "installable" on Linux and "wrong
+    # OS" everywhere else (issue #178).
+    if not platform_supported():
+        return _wrong_platform()
+
     missing = check_prerequisites()
     if missing:
-        return _skip("missing prerequisites: " + "; ".join(missing))
+        return _cannot_run("missing prerequisites: " + "; ".join(missing))
 
     if not _ensure_built():
-        return _skip("c64-https.prg could not be built")
+        return _cannot_run("c64-https.prg could not be built -- `make` "
+                           "failed; this is a broken build, not a "
+                           "missing prerequisite", opt_out=False)
 
     handle = None
     listener = None