Sleep: forbid wake → deep/REM transitions — the one part of #348 that survives a de-contaminated benchmark

Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStagerV2.swift

@@ -177,11 +177,32 @@ public enum SleepStagerV2 {
 
     /// Transition matrix (rows = from, cols = to). Self-transitions dominate; deep↔rem rare; wake mostly
     /// to/from light. A priori, not fit.
+    ///
+    /// The AWAKE row encodes sleep-onset physiology directly: a sleeper does not enter N3 or REM straight
+    /// out of wakefulness — descent runs through N1/N2 — so wake→deep and wake→rem are ZERO rather than the
+    /// small non-zero values they used to carry, and the freed mass goes to the wake self-loop, which makes
+    /// a WASO episode span several epochs instead of flickering back to sleep after one. This row is the one
+    /// part of PR #348's DREAMT re-tune that survives measurement on a de-contaminated reference set; the
+    /// rest of that PR (its base priors, motion-gate multipliers, deep gate, awake dead-zone, emission
+    /// coefficients and the deep/rem/light transition rows) was reverted by #437 and stays reverted, having
+    /// measured neutral-to-negative here. See the header note on `viterbi` for why a zero is safe — and note
+    /// that ZERO is a strong prior rather than a prohibition: the viterbi floor turns it into ≈ -20.7 against
+    /// wake→light's ≈ -2.3, an ~18.4 log-unit penalty a sufficiently strong emission can still cross, so a
+    /// genuine sleep-onset REM period stays representable instead of structurally impossible.
+    ///
+    /// Measured on one wearer's 36 recorded nights, against the strap's own band `sleep_state` (an
+    /// independent reference the recipe cannot contaminate — 21 nights, 15 554 epochs): sleep/wake kappa
+    /// 0.105 → 0.118 and wake sensitivity 16.0 % → 17.6 %, with the healthy-stratum wake fraction essentially
+    /// unmoved (9.43 % → 9.96 %, i.e. no repeat of the #437 blow-out). Confirmed afterwards against
+    /// human-scored PSG hypnograms (PhysioNet sleep-accel, 31 subjects / 26 773 epochs, #991): 4-class kappa
+    /// 0.356 → 0.363, REM F1 0.569 → 0.575, wake sensitivity 30.42 % → 30.84 %, and the #437 stage-fraction
+    /// guard holds against truth as well. n = 1 wearer for the band figures; see `Tools/SleepBench` and the
+    /// PR for the full ablation and its limits.
     static let transition: [String: [String: Double]] = [
         "deep":  ["deep": 0.86, "rem": 0.007, "light": 0.126, "awake": 0.007],
         "rem":   ["deep": 0.005, "rem": 0.88, "light": 0.10, "awake": 0.015],
         "light": ["deep": 0.06, "rem": 0.06, "light": 0.85, "awake": 0.03],
-        "awake": ["deep": 0.01, "rem": 0.02, "light": 0.27, "awake": 0.70]]
+        "awake": ["deep": 0.0, "rem": 0.0, "light": 0.10, "awake": 0.90]]
 
     /// One 30 s epoch's recipe features. Optionals are "no measurement"; the z-score / percentile treat a
     /// missing value as the neutral centre so a sparse channel never blocks a stage.
@@ -461,8 +482,11 @@ public enum SleepStagerV2 {
     /// uniform start. Ties resolve to the earlier stage in `stageNames`.
     static func viterbi(_ emSeq: [[String: Double]]) -> [String] {
         if emSeq.isEmpty { return [] }
-        // Floor before ln so a zeroed transition entry (a legal hand-edit) can never hit ln(0) = -Inf
-        // and poison the lattice. Inert for the current matrix (no zero entries). Kept from #348.
+        // Floor before ln so a zeroed transition entry can never hit ln(0) = -Inf and poison the lattice.
+        // LOAD-BEARING, not defensive: the awake row carries wake→deep = wake→rem = 0.0, so this floor is
+        // the only thing between those two entries and -Inf. Deleting it does not remove dead code, it
+        // breaks the stager. Floored, a zero costs ln(1e-9) ≈ -20.7 against wake→light's ≈ -2.3. The floor
+        // arrived with #348 and survived #437; the zeros it now carries arrived later.
         let logT = transition.mapValues { row in row.mapValues { log(max($0, 1e-9)) } }
         var V = emSeq[0]   // uniform start
         var back: [[String: String]] = []

Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerV2Tests.swift

@@ -327,4 +327,20 @@ final class SleepStagerV2Tests: XCTestCase {
             XCTAssertEqual(row.values.reduce(0, +), 1.0, accuracy: 1e-9, "transition row '\(from)' must sum to 1.0")
         }
     }
+
+    /// Pin the AWAKE transition row directly, for the same reason the deep row is pinned: the end-to-end
+    /// golden is only sensitive where its own input sits near a decision boundary, and it does NOT move when
+    /// this row changes — so without this assertion the row is effectively unguarded. Twin:
+    /// `SleepStagerV2Test.wakeRowForbidsDirectDescentIntoDeepOrRem`.
+    ///
+    /// The zeros are the physiological claim: sleep onset descends through N1/N2, so wake never transitions
+    /// straight into N3 or REM. They are also why `viterbi` floors the log at 1e-9 — asserted here so the two
+    /// can never drift apart and reintroduce ln(0) = -inf.
+    func testWakeRowForbidsDirectDescentIntoDeepOrRem() {
+        XCTAssertEqual(SleepStagerV2.transition["awake"]!,
+                       ["deep": 0.0, "rem": 0.0, "light": 0.10, "awake": 0.90])
+        // A zeroed entry must reach the lattice as a large finite penalty, never -inf.
+        let lp = log(max(SleepStagerV2.transition["awake"]!["deep"]!, 1e-9))
+        XCTAssertTrue(lp.isFinite, "a zeroed transition must not produce a non-finite log-weight")
+    }
 }

android/app/src/main/java/com/noop/analytics/SleepStagerV2.kt

@@ -197,12 +197,33 @@ object SleepStagerV2 {
     private const val respWeight = 0.6
 
     /** Transition matrix (rows = from, cols = to). Self-transitions dominate; deep↔rem rare; wake mostly
-     *  to/from light. A priori, not fit. */
+     *  to/from light. A priori, not fit.
+     *
+     *  The AWAKE row encodes sleep-onset physiology directly: a sleeper does not enter N3 or REM straight
+     *  out of wakefulness — descent runs through N1/N2 — so wake→deep and wake→rem are ZERO rather than the
+     *  small non-zero values they used to carry, and the freed mass goes to the wake self-loop, which makes
+     *  a WASO episode span several epochs instead of flickering back to sleep after one. This row is the one
+     *  part of PR #348's DREAMT re-tune that survives measurement on a de-contaminated reference set; the
+     *  rest of that PR (its base priors, motion-gate multipliers, deep gate, awake dead-zone, emission
+     *  coefficients and the deep/rem/light transition rows) was reverted by #437 and stays reverted, having
+     *  measured neutral-to-negative here. See the header note on [viterbi] for why a zero is safe — and note
+     *  that ZERO is a strong prior rather than a prohibition: the viterbi floor turns it into ≈ -20.7 against
+     *  wake→light's ≈ -2.3, an ~18.4 log-unit penalty a sufficiently strong emission can still cross, so a
+     *  genuine sleep-onset REM period stays representable instead of structurally impossible.
+     *
+     *  Measured on one wearer's 36 recorded nights, against the strap's own band `sleep_state` (an
+     *  independent reference the recipe cannot contaminate — 21 nights, 15 554 epochs): sleep/wake kappa
+     *  0.105 → 0.118 and wake sensitivity 16.0 % → 17.6 %, with the healthy-stratum wake fraction essentially
+     *  unmoved (9.43 % → 9.96 %, i.e. no repeat of the #437 blow-out). Confirmed afterwards against
+     *  human-scored PSG hypnograms (PhysioNet sleep-accel, 31 subjects / 26 773 epochs, #991): 4-class kappa
+     *  0.356 → 0.363, REM F1 0.569 → 0.575, wake sensitivity 30.42 % → 30.84 %, and the #437 stage-fraction
+     *  guard holds against truth as well. n = 1 wearer for the band figures; see `Tools/SleepBench` and the
+     *  PR for the full ablation and its limits. */
     internal val transition: Map<String, Map<String, Double>> = mapOf(
         "deep" to mapOf("deep" to 0.86, "rem" to 0.007, "light" to 0.126, "awake" to 0.007),
         "rem" to mapOf("deep" to 0.005, "rem" to 0.88, "light" to 0.10, "awake" to 0.015),
         "light" to mapOf("deep" to 0.06, "rem" to 0.06, "light" to 0.85, "awake" to 0.03),
-        "awake" to mapOf("deep" to 0.01, "rem" to 0.02, "light" to 0.27, "awake" to 0.70))
+        "awake" to mapOf("deep" to 0.0, "rem" to 0.0, "light" to 0.10, "awake" to 0.90))
 
     /** One 30 s epoch's recipe features. Nullable means "no measurement"; the z-score / percentile treat a
      *  missing value as the neutral centre so a sparse channel never blocks a stage. Internal (not private) so
@@ -497,8 +518,11 @@ object SleepStagerV2 {
      *  uniform start. Ties resolve to the earlier stage in [stageNames]. */
     private fun viterbi(emSeq: List<Map<String, Double>>): List<String> {
         if (emSeq.isEmpty()) return emptyList()
-        // Floor before ln so a zeroed transition entry (a legal hand-edit) can never hit ln(0) = -Inf
-        // and poison the lattice. Inert for the current matrix (no zero entries). Kept from #348.
+        // Floor before ln so a zeroed transition entry can never hit ln(0) = -Inf and poison the lattice.
+        // LOAD-BEARING, not defensive: the awake row carries wake→deep = wake→rem = 0.0, so this floor is
+        // the only thing between those two entries and -Inf. Deleting it does not remove dead code, it
+        // breaks the stager. Floored, a zero costs ln(1e-9) ≈ -20.7 against wake→light's ≈ -2.3. The floor
+        // arrived with #348 and survived #437; the zeros it now carries arrived later.
         val logT = transition.mapValues { (_, row) -> row.mapValues { (_, v) -> ln(maxOf(v, 1e-9)) } }
         var v = emSeq[0]   // uniform start
         val back = ArrayList<Map<String, String>>()

android/app/src/test/java/com/noop/analytics/SleepStagerV2Test.kt

@@ -9,6 +9,7 @@ import org.junit.Assert.assertNotNull
 import org.junit.Assert.assertTrue
 import org.junit.Test
 import kotlin.math.PI
+import kotlin.math.ln
 import kotlin.math.sin
 import kotlin.math.roundToInt
 
@@ -357,4 +358,24 @@ class SleepStagerV2Test {
             assertEquals("transition row '$from' must sum to 1.0", 1.0, row.values.sum(), 1e-9)
         }
     }
+
+    /**
+     * Pin the AWAKE transition row directly, for the same reason the deep row is pinned: the end-to-end golden
+     * is only sensitive where its own input sits near a decision boundary, and it does NOT move when this row
+     * changes — so without this assertion the row is effectively unguarded. Twin:
+     * `SleepStagerV2Tests.testWakeRowForbidsDirectDescentIntoDeepOrRem`.
+     *
+     * The zeros are the physiological claim: sleep onset descends through N1/N2, so wake never transitions
+     * straight into N3 or REM. They are also why [SleepStagerV2.viterbi] floors the log at 1e-9 — asserted here
+     * so the two can never drift apart and reintroduce ln(0) = -Inf.
+     */
+    @Test
+    fun wakeRowForbidsDirectDescentIntoDeepOrRem() {
+        assertEquals(
+            mapOf("deep" to 0.0, "rem" to 0.0, "light" to 0.10, "awake" to 0.90),
+            SleepStagerV2.transition["awake"])
+        // A zeroed entry must reach the lattice as a large finite penalty, never -Inf.
+        val lp = ln(maxOf(SleepStagerV2.transition["awake"]!!["deep"]!!, 1e-9))
+        assertTrue("a zeroed transition must not produce a non-finite log-weight", lp.isFinite())
+    }
 }