test: six suites could exit 0 having tested nothing

tools/test_dns.py

@@ -108,7 +108,28 @@ def main():
     os.chdir(PROJECT_ROOT)
 
     if skip_if_no_network():
-        sys.exit(0)
+        # An involuntary skip is a failure: nothing above was configured, so
+        # not one of the four DNS assertions ran and exiting 0 would report
+        # "DNS resolution is fine" on a host that cannot even reach the
+        # emulated wire. Exit 2 ("could not run"), distinct from 1 ("tests
+        # failed"), so a caller can tell the two apart.
+        #
+        # An operator who knows this host has no TAP rig and wants the
+        # network suites out of the way makes that choice EXPLICIT — the
+        # other half of the audit rule (7497e48): an explicit skip is
+        # allowed, but must never be silent.
+        if os.environ.get("C64_NET_TESTS_OPTIONAL") == "1":
+            print("EXPLICIT SKIP (C64_NET_TESTS_OPTIONAL=1): test_dns.py did "
+                  "NOT run.\n  0 of 4 DNS assertions executed; this exit 0 "
+                  "certifies nothing about DNS.")
+            sys.exit(0)
+        print("CANNOT RUN: test_dns.py needs the TAP network rig "
+              "(tools/rig-up-macos.sh or the Linux tap-c64 setup).\n"
+              "  0 of 4 DNS assertions executed — this run certifies "
+              "nothing.\n"
+              "  Set C64_NET_TESTS_OPTIONAL=1 to make skipping it a "
+              "deliberate, exit-0 choice.", file=sys.stderr)
+        sys.exit(2)
 
     # Late imports -- only needed if prerequisites are met
     from c64_test_harness import (

tools/test_ecdsa_p384_kat.py

@@ -44,7 +44,9 @@
                 the dual-overlay swap dispatch + SHA-384 + splice path
                 works without paying for the verify (which on a busy
                 VICE warp host can take 5-30 min/vector).  Use this
-                first if --full hangs.
+                first if --full hangs.  It leaves every vector without a
+                verdict, so it always exits non-zero and can never report
+                OVERALL: PASS — read the per-step output, not the tally.
 
 Environment:
     C64_SKIP_BUILD=1            Reuse existing build artifacts (skip make).
@@ -544,20 +546,31 @@ def _step(label: str, addr: int, *, step_timeout: float) -> dict | None:
     if err: return err
 
     if sha_only:
-        # Diagnostic short-circuit: confirm dual-overlay flow + SHA + splice
-        # all work without paying for the (slow) ecdsa_verify_384 call.
-        # We've already verified the digest matched host hashlib above; the
-        # swap_to_curve completed; treat that as a "structural PASS" and
-        # synthesise a result dict.
+        # Diagnostic short-circuit: the dual-overlay swap, SHA-384 and the
+        # digest splice have all been confirmed above without paying for
+        # the (slow) ecdsa_verify_384 call.
+        #
+        # What must NOT happen here is synthesising `valid` from
+        # vec["expected_valid"]: that fabricates the answer the test exists
+        # to obtain, makes every vector compare equal to its own
+        # expectation, and prints OVERALL: PASS for a run in which
+        # ecdsa_verify_384 was never executed. The vector has no verdict,
+        # so it is reported as one that could not run — a failure, per the
+        # audit rule that an involuntary skip is a failure. --sha-only is a
+        # diagnostic; by construction it can never certify the verify path.
         if verbose:
-            print(f"      [--sha-only] dual-overlay structural PASS "
-                  f"(skipped ecdsa_verify_384)")
+            print(f"      [--sha-only] swap + SHA-384 + splice OK; "
+                  f"ecdsa_verify_384 NOT run — no verdict for this vector")
         return {
-            "valid": vec["expected_valid"],   # synthesised: assume PASS
+            "error": "--sha-only: ecdsa_verify_384 was never executed, so "
+                     "this vector has no verdict (counted as a failure). "
+                     "The swap/SHA-384/splice steps up to it did pass.",
+            "valid": None,
             "carry": 0xFE,                    # marker for "skipped"
             "seconds": 0.0,
             "overall_seconds": time.perf_counter() - t_overall,
             "sha_only": True,
+            "failed_step": "ecdsa_verify_384 (skipped by --sha-only)",
         }
 
     # Step 9: verify (the slow one — bench wall-clock ~75-90 s on real
@@ -850,6 +863,12 @@ def main() -> int:
     total_fail = v_fail + u_fail
     overall = "PASS" if total_fail == 0 else "FAIL"
     print(f"OVERALL: {overall}")
+    if sha_only:
+        print("NOTE: --sha-only skips ecdsa_verify_384, so no vector can "
+              "produce a verdict\n      and this mode can never report "
+              "OVERALL: PASS. It is a diagnostic for the\n      "
+              "swap + SHA-384 + splice path only; drop --sha-only to test "
+              "the verify.")
     return 0 if total_fail == 0 else 1
 
 

tools/test_http_integration.py

@@ -89,7 +89,27 @@ def main():
     os.chdir(PROJECT_ROOT)
 
     if not check_prerequisites():
-        sys.exit(0)
+        # An involuntary skip is a failure: none of the five end-to-end HTTP
+        # assertions ran, so exiting 0 would report a working HTTP path on a
+        # host that has no TAP rig at all. Exit 2 ("could not run") to keep
+        # that distinct from 1 ("tests failed").
+        #
+        # C64_NET_TESTS_OPTIONAL=1 is the explicit, deliberate skip — the
+        # other half of the audit rule (7497e48): explicit skips are allowed,
+        # but never silent.
+        if os.environ.get("C64_NET_TESTS_OPTIONAL") == "1":
+            print("EXPLICIT SKIP (C64_NET_TESTS_OPTIONAL=1): "
+                  "test_http_integration.py did NOT run.\n  0 of 5 HTTP "
+                  "assertions executed; this exit 0 certifies nothing about "
+                  "the HTTP path.")
+            sys.exit(0)
+        print("CANNOT RUN: test_http_integration.py needs the TAP network "
+              "rig (tap-c64 + x64sc + dnsmasq).\n"
+              "  0 of 5 HTTP assertions executed — this run certifies "
+              "nothing.\n"
+              "  Set C64_NET_TESTS_OPTIONAL=1 to make skipping it a "
+              "deliberate, exit-0 choice.", file=sys.stderr)
+        sys.exit(2)
 
     # Late imports -- only needed if prerequisites are met
     sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

tools/test_tls_handshake.py

@@ -1318,7 +1318,13 @@ def main():
     print(f"  Passed: {passed}/{total}")
     print(f"  Failed: {failed}/{total}")
     if total == 0:
-        print("\n  [?] No tests ran (routines not yet implemented?)")
+        # Zero assertions is not a pass — see the same guard in
+        # test_tls_record.py. An involuntary skip is a failure.
+        print("\n  [-] TLS HANDSHAKE: NO TESTS RAN — every group was "
+              "skipped\n      (routines missing or all groups errored). "
+              "This run certifies nothing.")
+        print(f"{'='*60}")
+        sys.exit(1)
     elif failed == 0:
         print(f"\n  [+] TLS HANDSHAKE: ALL {total} TESTS PASSED")
     else:

tools/test_tls_record.py

@@ -833,7 +833,15 @@ def main():
     print(f"  Passed: {passed}/{total}")
     print(f"  Failed: {failed}/{total}")
     if total == 0:
-        print("\n  [?] No tests ran (routines not yet implemented?)")
+        # Zero assertions is not a pass. Every group returning (0, 0) means
+        # the routines under test were never exercised — an unrequested,
+        # involuntary skip — and "0 failed" would otherwise exit 0 and read
+        # as a green record layer.
+        print("\n  [-] TLS RECORD LAYER: NO TESTS RAN — every group was "
+              "skipped\n      (routines missing or all groups errored). "
+              "This run certifies nothing.")
+        print(f"{'='*60}")
+        sys.exit(1)
     elif failed == 0:
         print(f"\n  [+] TLS RECORD LAYER: ALL {total} TESTS PASSED")
     else:

tools/test_x509_name.py

@@ -206,9 +206,17 @@ def main() -> int:
             "tls_hostname", "tls_hostname_len", "cert_buf_size"]
     for n in need:
         if labels.address(n) is None:
-            print(f"SKIP: label {n} missing — this is a BACKEND=uci-only feature "
-                  f"(see #135); nothing to test on this build.")
-            return 0
+            # An involuntary skip is a failure. This suite is the only
+            # coverage server-name validation has; on a build that does not
+            # export the routine (BACKEND=ip65, see #135) every one of the
+            # vectors below is silently dropped, and exiting 0 here would
+            # report "hostname checking is fine" having checked nothing.
+            # Run it against a BACKEND=uci build, or do not run it.
+            print(f"CANNOT RUN: label {n} missing — server name validation is "
+                  f"a BACKEND=uci-only feature (see #135). None of the "
+                  f"vectors executed; this run certifies nothing.",
+                  file=sys.stderr)
+            return 2
     cert_cap = labels.address("cert_buf_size")
 
     vectors = build_vectors()
@@ -230,9 +238,19 @@ def main() -> int:
             return 2
 
         passed = failed = 0
+        oversize = []
         for name, der, host, want, why in vectors:
             if len(der) > cert_cap:
-                print(f"  SKIP {name}: {len(der)} B exceeds cert_buf")
+                # Declared vector, no verdict: it leaves the denominator
+                # unless it is accounted for. Same rule as the KAT oracle's
+                # "every declared vector must produce a verdict" check, and
+                # this one is load-bearing — the vectors most likely to
+                # outgrow cert_buf are the real CA-issued leaves, i.e. the
+                # ones the client actually has to get right.
+                print(f"  [-] CANNOT RUN {name}: {len(der)} B exceeds "
+                      f"cert_buf ({cert_cap} B) — counted as a failure")
+                oversize.append((name, len(der)))
+                failed += 1
                 continue
             write_bytes(t, labels["cert_buf"], der)
             write_bytes(t, labels["cert_data_ptr"],
@@ -248,7 +266,12 @@ def main() -> int:
             gs = "hung" if got is None else f"C={got}"
             print(f"  [{verdict}] {name}: {gs} (want C={want}) — {why}")
 
-        print(f"\nRESULTS: {passed}/{passed+failed} passed")
+        if oversize:
+            print(f"\n  VECTORS THAT COULD NOT RUN (counted as failures)")
+            for name, n in oversize:
+                print(f"  [-] {name}: {n} B > cert_buf {cert_cap} B")
+        print(f"\nRESULTS: {passed}/{passed+failed} passed"
+              f"{f' ({len(oversize)} could not run)' if oversize else ''}")
         return 0 if failed == 0 else 1