diff --git a/lib/ble/ble_engine.dart b/lib/ble/ble_engine.dart index 774cb45..dbd7fcb 100644 --- a/lib/ble/ble_engine.dart +++ b/lib/ble/ble_engine.dart @@ -84,6 +84,101 @@ typedef ArchiveSink = Future Function(ArchiveRecord archive); /// trigger now that listening is continuous and there's no discrete sync end. typedef DataStoredSink = void Function(); +/// Read v18 unix from the protocol shared header at inner[7:11]. +/// +/// Do **not** fall back to a misaligned offset-6 read: that overlaps the +/// record-index high byte and can invent a year-window-plausible timestamp +/// that is not monotonic with `recordIndex` (honesty violation — observed on +/// the fw 50.40.1.0 export where unix@7 was garbage because SET_CLOCK never +/// latched). Absent a plausible unix@7, abstain. +@visibleForTesting +int? gen5V18UnixFromInner(Uint8List inner, int wallNow) { + if (inner.length < 11) return null; + final view = inner.buffer.asByteData(inner.offsetInBytes, inner.lengthInBytes); + final at7 = view.getUint32(7, Endian.little); + return isPlausibleUnix(at7, wallNow) ? at7 : null; +} + +/// Hardware lenient v18 decode when [parseGen5Historical] returns null because +/// the protocol decoder's gravity/dynamic-accel gates reject real captures +/// (off-wrist / motion-heavy seconds still carry valid HR/RR). Never fabricates +/// gravity — ax/ay/az stay null when the vector fails the magnitude gate. +/// Never fabricates time — requires a plausible unix@7. +@visibleForTesting +Sample? sampleFromGen5V18Lenient(Uint8List inner, int wallNow) { + if (inner.length < kGen5V18MinInnerLen || inner[1] != 18) return null; + final unix = gen5V18UnixFromInner(inner, wallNow); + if (unix == null) return null; + final view = inner.buffer.asByteData(inner.offsetInBytes, inner.lengthInBytes); + final counter = view.getUint32(3, Endian.little); + final hr = inner[14]; + if (hr != 0 && (hr < 25 || hr > 230)) return null; + + const minRrMs = 200; + const maxRrMs = 2500; + final declaredRr = inner[15]; + final rr = []; + if (declaredRr <= 4) { + for (int i = 0; i < declaredRr && 16 + 2 * i + 2 <= inner.length; i++) { + final val = view.getInt16(16 + 2 * i, Endian.little); + if (val >= minRrMs && val <= maxRrMs) rr.add(val); + } + } + + double? ax; + double? ay; + double? az; + if (inner.length >= 49) { + final gx = view.getFloat32(37, Endian.little); + final gy = view.getFloat32(41, Endian.little); + final gz = view.getFloat32(45, Endian.little); + if (gx.isFinite && gy.isFinite && gz.isFinite) { + final magSq = gx * gx + gy * gy + gz * gz; + // Same gate as protocol's Gen5V18Decoder — but abstain on ax/ay/az + // instead of rejecting the whole record (HR/RR are independent fields). + if (magSq >= 0.25 && magSq <= 2.25) { + ax = gx; + ay = gy; + az = gz; + } + } + } + + return Sample( + tsEpoch: unix, + counter: counter, + hr: hr, + rrIntervalsMs: rr, + ax: ax, + ay: ay, + az: az, + ); +} + +/// Build gen5 SET_CLOCK_MAVERICK (0x92) / GET_CLOCK_GEN5 (0x93) payloads. +/// +/// Hardware evidence (fw 50.40.1.0 console): an empty GET_CLOCK body logs +/// `Invalid revision for get clock: 0`, and SET_CLOCK without a leading +/// form/revision byte logs `Invalid revision ` — i.e. the strap +/// treats body[0] as the command revision (same role as HELLO's `[0x01]` / +/// Alec's gen5 `b3`). Prepend [revision1] so the 8-byte time field starts at +/// body[1]. +@visibleForTesting +List gen5SetClockPayload({required int sec, required int subsec}) => [ + revision1, + sec & 0xff, + (sec >> 8) & 0xff, + (sec >> 16) & 0xff, + (sec >> 24) & 0xff, + subsec & 0xff, + (subsec >> 8) & 0xff, + 0, + 0, + ]; + +@visibleForTesting +List gen5GetClockPayload() => const [revision1]; + /// Map a decoded gen5 historical record onto the band-agnostic `Sample` type, /// or null when this record kind has no `Sample` equivalent (yet). /// @@ -113,6 +208,17 @@ Sample? sampleFromGen5Historical(Gen5HistoricalRecord? g) { ); } +/// Decode a gen5 historical inner frame to a band-agnostic [Sample], or null. +@visibleForTesting +Sample? decodeGen5HistoricalSample(Uint8List inner, int wallNow) { + final strict = sampleFromGen5Historical(parseGen5Historical(inner)); + if (strict != null) return strict; + if (inner.length > 1 && inner[1] == 18) { + return sampleFromGen5V18Lenient(inner, wallNow); + } + return null; +} + @visibleForTesting int countHistoricalBurstPackets({ required Map dataPacketCountsByRevision, @@ -1929,6 +2035,32 @@ class BleEngine { /// path is deliberate: the previous duplicate had drifted, silently losing /// the plausibility gate and freezing the frontier the stuck-strap / /// auto-continue policies read. + /// Set a historical frame aside in `raw_archive` — the never-pruned store for + /// bytes this build could not fully turn into a [Sample]. + /// + /// Routed through the drain when one is active so the write lands inside the + /// SAME transaction as the batch commit (safe-trim invariant: nothing the + /// band is told it may trim has been discarded). + void _archiveHistoricalFrame( + Frame frame, + int counter, { + required String reason, + }) { + final archive = ArchiveRecord( + counter: counter, + hex: _innerHex(frame.inner), + packetType: frame.inner.isNotEmpty ? frame.inner[0] : 0, + capturedAt: DateTime.now().millisecondsSinceEpoch, + reason: reason, + ); + final d = _drain; + if (d != null) { + d.onUndecodableRecord(archive); + } else { + unawaited(onArchiveRecord?.call(archive) ?? Future.value()); + } + } + void _ingestHistoricalFrame(Frame frame) { final pt = frame.packetType; if (pt != PacketType.historicalData) return; @@ -1950,6 +2082,7 @@ class BleEngine { // backfill (all received in one sync) splits into correct per-real-day // buckets instead of collapsing into one "today". Sample? sample; + final wallNow = DateTime.now().millisecondsSinceEpoch ~/ 1000; final isGen5 = _session?.band.isGen5 ?? false; if (isGen5) { // gen5 (WHOOP 5): `parseGen5Historical` dispatches across all four real @@ -1959,7 +2092,22 @@ class BleEngine { // own raw-buffer storage (a future db table), not a 1Hz Sample, so they // fall through to the undecodable archive below — that is honest // (correctly-identified-but-not-yet-stored), not a decode failure. - sample = sampleFromGen5Historical(parseGen5Historical(frame.inner)); + sample = decodeGen5HistoricalSample(frame.inner, wallNow); + // PARTIAL decode is not a full decode. The lenient v18 path deliberately + // keeps HR/RR while ABSTAINING on a gravity vector that failed the + // magnitude gate — but `raw_records` is gone and `decoded_onehz` has + // nowhere to put a null accel, so those gravity bytes would be discarded + // the moment we ACK the trim. Archive the frame as WELL as keeping the + // sample: same safe-trim transaction, and a future decoder can still + // recover what this one could not. Nothing is double-counted — + // `raw_archive` is a diagnostic store, never a derivation input. + if (sample != null && sample.ax == null) { + _archiveHistoricalFrame( + frame, + counter, + reason: 'partial_decode_v${recType}_no_gravity', + ); + } } else if (recType == Record.r24 || recType == Record.r12) { // Legacy decoder first, firmware-fallback chain second, undecodable // archive last — see FirmwareAwareR24Decoder. @@ -1999,19 +2147,11 @@ class BleEngine { // archive rides the SAME commit that runs before the batch-ACK, so nothing the // band trims has been discarded (safe-trim invariant intact). if (sample == null) { - final archive = ArchiveRecord( - counter: counter, - hex: _innerHex(frame.inner), - packetType: frame.inner.isNotEmpty ? frame.inner[0] : 0, - capturedAt: DateTime.now().millisecondsSinceEpoch, + _archiveHistoricalFrame( + frame, + counter, reason: 'undecodable_rec_v$recType', ); - final d = _drain; - if (d != null) { - d.onUndecodableRecord(archive); - } else { - unawaited(onArchiveRecord?.call(archive) ?? Future.value()); - } return; } // PLAUSIBILITY GATE + FRONTIER (RecordGate, shared with the detectors). @@ -2023,7 +2163,7 @@ class BleEngine { // Past this point [sample] is non-null — undecodable records returned above. if (!_recordGate.admit( sample.tsEpoch, - wallNow: DateTime.now().millisecondsSinceEpoch ~/ 1000, + wallNow: wallNow, sessionOldestUnix: _sessionOldestUnix, sessionNewestUnix: _sessionNewestUnix, )) { @@ -2890,26 +3030,26 @@ class BleEngine { final ms = DateTime.now().millisecondsSinceEpoch; final sec = ms ~/ 1000; final subsec = ((ms % 1000) * 32768) ~/ 1000; // 0..32767, 1/32768 s units - final payload = [ - sec & 0xff, - (sec >> 8) & 0xff, - (sec >> 16) & 0xff, - (sec >> 24) & 0xff, - subsec & 0xff, - (subsec >> 8) & 0xff, - 0, - 0, - ]; - // gen5 ("Maverick") uses a DIFFERENT opcode for SET_CLOCK than gen4 — the - // 8-byte payload shape is unchanged, only the opcode value differs (see - // Cmd.setClockMaverick's doc in protocol/constants.dart). Sending gen4's - // opcode 0x0A to a gen5 strap here would silently fail to latch the RTC, - // which then refuses to serve type-47 history — the exact symptom fixed. + // gen5 ("Maverick") uses a DIFFERENT opcode for SET_CLOCK than gen4, and + // (per fw 50.40.1.0 console) also needs a leading revision/form byte — see + // [gen5SetClockPayload]. Gen4 keeps the hardware-verified 8-byte body. final isGen5 = _session?.band.isGen5 ?? false; + final payload = isGen5 + ? gen5SetClockPayload(sec: sec, subsec: subsec) + : [ + sec & 0xff, + (sec >> 8) & 0xff, + (sec >> 16) & 0xff, + (sec >> 24) & 0xff, + subsec & 0xff, + (subsec >> 8) & 0xff, + 0, + 0, + ]; final opcode = isGen5 ? Cmd.setClockMaverick : Cmd.setClock; await _send(opcode, payload); _log('SET_CLOCK${isGen5 ? " (gen5 Maverick)" : ""} → sec=$sec ' - 'subsec=$subsec (WHOOP-exact 8B).'); + 'subsec=$subsec (${payload.length}B${isGen5 ? ", rev=$revision1" : ""}).'); // Read the RTC back so the GET_CLOCK response handler can confirm it latched // (and re-issue SET_CLOCK if the strap clock is still off — see _onDecoded). await getClock(); @@ -2917,11 +3057,15 @@ class BleEngine { /// Read the strap RTC. The response carries `clock_epoch`, handled where we /// verify drift and re-correlate the strap-RTC ↔ wall clock. gen5 uses its - /// own GET_CLOCK opcode (147) — see [setClock]. - Future getClock() => _send( - (_session?.band.isGen5 ?? false) ? Cmd.getClockGen5 : Cmd.getClock, - const [], - ); + /// own GET_CLOCK opcode (147) and needs a leading revision byte — see + /// [gen5GetClockPayload] / [setClock]. + Future getClock() { + final isGen5 = _session?.band.isGen5 ?? false; + return _send( + isGen5 ? Cmd.getClockGen5 : Cmd.getClock, + isGen5 ? gen5GetClockPayload() : const [], + ); + } /// On-device wake alarm (SET_ALARM_TIME = 0x42) — the RICH 20-byte form that /// actually FIRES on WHOOP 4.0: diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart index 40df505..e9a139f 100644 --- a/lib/compute/substrate.dart +++ b/lib/compute/substrate.dart @@ -16,6 +16,16 @@ import 'dart:math' as math; import 'package:openstrap_analytics/onehz.dart' as ana; import 'package:openstrap_protocol/openstrap_protocol.dart' as proto; +/// Minimum fraction of a nocturnal search window that must carry a REAL +/// gravity vector before accel-led (van Hees) sleep detection is trusted. +/// +/// Below this we do not run it at all and fall through to the HR-led window, +/// which is already the honest low-confidence degraded mode. Set at a half +/// rather than something tiny on purpose: van Hees picks the LONGEST immobile +/// block, and absent seconds are maximally "immobile", so a window that is +/// mostly absent would reliably hand the answer to the missing data. +const double kMinAccelCoverageForVanHees = 0.5; + /// The decoded 1 Hz substrate — the only decoded form (ARCHITECTURE_V2). /// /// All HR/accel/ADC arrays are parallel and 1:1 with [tsSec] (one sample per @@ -82,6 +92,36 @@ class Substrate { ana.AccelSample(tsSec[i] * 1000.0, ax[i], ay[i], az[i]) ]; + /// Whether second [i] carries a REAL gravity vector. + /// + /// `decoded_onehz.ax/ay/az` are `REAL NOT NULL`, so a record decoded without + /// a usable gravity vector (the gen5 v18 lenient path, which deliberately + /// abstains on accel while keeping HR/RR) is stored as exact `(0, 0, 0)`. + /// That is not a reading a real device can produce — a gravity vector always + /// has magnitude ~1 g, and every decoder that emits one gates on + /// `magSq >= 0.25` — so exact zero is an unambiguous ABSENT marker rather + /// than a measurement. + /// + /// This matters because absent accel does not merely go unused: a run of + /// `(0, 0, 0)` has a constant z-angle of exactly 0.0°, which the van Hees + /// rule reads as PERFECT IMMOBILITY. Eight hours of missing accel scores + /// 28 501 immobile seconds and yields a fabricated ~7.9 h sleep window, + /// fully staged. Absent input must produce no claim, never a confident one. + bool accelPresentAt(int i) => !(ax[i] == 0 && ay[i] == 0 && az[i] == 0); + + /// Fraction of [lo, hi) seconds carrying a real gravity vector (0..1). + /// Returns 0 for an empty range — no evidence, not "all present". + double accelPresentFraction(int lo, int hi) { + final a = lo < 0 ? 0 : lo; + final b = hi > tsSec.length ? tsSec.length : hi; + if (b <= a) return 0; + var present = 0; + for (var i = a; i < b; i++) { + if (accelPresentAt(i)) present++; + } + return present / (b - a); + } + /// 1 Hz HR as doubles (0 = off-skin). Parallel to [tsSec] / [accelSamples]. List hr1hz() => [for (final h in hr) h.toDouble()]; @@ -531,14 +571,33 @@ List calendarDays( ); src = ov.source; // 'manual' | 'confirmed' } else { - s = ana.segmentSleep( - accelSlice, - hrSlice, - hrBaseline: hrBaseline, - rrMs: rrMsSeg, - rrTsMs: rrTsSeg, - habitualMidsleepSec: habitualMidsleepSec, - ); + // Accel-led detection is only meaningful if we actually HAVE accel. + // Absent gravity is stored as exact (0,0,0) (see `accelPresentAt`) and + // van Hees scores a run of it as perfect immobility, so a night whose + // records all decoded without a gravity vector would otherwise produce + // a confident, fully-staged sleep window built entirely out of missing + // data. `immobilityMask` has no validity input to tell it otherwise — + // it is a pure index-wise angle rule, so neither a NaN sentinel (NaN + // comparisons are false, so the "angle changed" test never trips and + // it reads as immobile) nor omitting the seconds (no gap awareness) + // reaches it. The only honest move at this layer is not to let it + // anchor the window in the first place. + final accelCoverage = sub.accelPresentFraction(loS, hiS); + if (accelCoverage >= kMinAccelCoverageForVanHees) { + s = ana.segmentSleep( + accelSlice, + hrSlice, + hrBaseline: hrBaseline, + rrMs: rrMsSeg, + rrTsMs: rrTsSeg, + habitualMidsleepSec: habitualMidsleepSec, + ); + } else { + // Not an error and not "no sleep" — just no accel evidence. Fall + // through to the HR-led path below, which is exactly the degraded + // mode for this and is already marked low-confidence. + s = ana.SleepSegmentation.absent; + } src = 'auto'; if (!s.present) { // Approach 2: accel-led detection found nothing → HR-led fallback. diff --git a/lib/data/db.dart b/lib/data/db.dart index 33d11f7..a7950c5 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -11,6 +11,7 @@ import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; import 'package:openstrap_analytics/onehz.dart' as ana; import 'package:openstrap_protocol/openstrap_protocol.dart' as proto; @@ -1856,32 +1857,55 @@ class LocalDb { static String _localDayLabelFromEpoch(int epochSec) => _localDayLabel(DateTime.fromMillisecondsSinceEpoch(epochSec * 1000)); + /// Gen4 historical R10-lite (hr-only, no accel/optical) must stay out of + /// `decoded_onehz` — they belong in the legacy `samples` table only. + static bool _isGen4R10LiteHistorical(Uint8List inner) => + inner.isNotEmpty && + inner[0] == proto.PacketType.historicalData && + inner.length > 1 && + inner[1] == proto.Record.r10; + static Sample? _decodeOneHzSample(RawRecord raw, {Sample? preferred}) { - if (preferred != null && preferred.hasDecodedOneHz) return preferred; + // Parse hex when possible so Gen4 R10-lite can be rejected even when a + // complete preferred Sample is supplied. Invalid/placeholder hex (test + // fixtures, corrupt imports) must NOT abort before the preferred paths — + // commit 1f85b10 returned null on hexToBytes failure and zeroed + // decoded_onehz for every insertRecord that used non-hex placeholders. + Uint8List? bytes; try { - // Legacy decoder first, firmware-fallback chain second — see - // FirmwareAwareR24Decoder. This path only runs when no pre-decoded - // `preferred` sample was supplied (e.g. a raw-hex import/merge), so a - // fresh per-call instance is fine — no session state to preserve. - final r = proto.FirmwareAwareR24Decoder().decode( - proto.hexToBytes(raw.hex), - ); - if (r == null || r.tsEpoch <= 0) return null; - return Sample( - tsEpoch: r.tsEpoch, - counter: r.counter, - hr: r.hr, - rrIntervalsMs: List.from(r.rrIntervalsMs), - ax: r.accelG.isNotEmpty ? r.accelG[0] : 0, - ay: r.accelG.length > 1 ? r.accelG[1] : 0, - az: r.accelG.length > 2 ? r.accelG[2] : 0, - spo2RedRaw: r.spo2RedRaw, - spo2IrRaw: r.spo2IrRaw, - skinTempRaw: r.skinTempRaw, - ); - } catch (_) { - return null; + bytes = proto.hexToBytes(raw.hex); + } catch (_) {} + if (bytes != null && _isGen4R10LiteHistorical(bytes)) return null; + if (preferred != null && preferred.hasDecodedOneHz) return preferred; + if (bytes != null) { + try { + // Legacy decoder first, firmware-fallback chain second — see + // FirmwareAwareR24Decoder. This path only runs when no pre-decoded + // `preferred` sample was supplied (e.g. a raw-hex import/merge), so a + // fresh per-call instance is fine — no session state to preserve. + final r = proto.FirmwareAwareR24Decoder().decode(bytes); + if (r != null && r.tsEpoch > 0) { + return Sample( + tsEpoch: r.tsEpoch, + counter: r.counter, + hr: r.hr, + rrIntervalsMs: List.from(r.rrIntervalsMs), + ax: r.accelG.isNotEmpty ? r.accelG[0] : 0, + ay: r.accelG.length > 1 ? r.accelG[1] : 0, + az: r.accelG.length > 2 ? r.accelG[2] : 0, + spo2RedRaw: r.spo2RedRaw, + spo2IrRaw: r.spo2IrRaw, + skinTempRaw: r.skinTempRaw, + ); + } + } catch (_) {} + } + // Gen5 v18 / lenient samples carry HR/RR/gravity but lack gen4 optics — + // `hasDecodedOneHz` stays false, yet they are honest 1 Hz substrate rows. + if (preferred != null && preferred.tsEpoch > 0) { + return preferred; } + return null; } /// THE orphan guard for an INSERT-OR-REPLACE into `decoded_onehz`. diff --git a/test/gen5_decoded_onehz_persistence_test.dart b/test/gen5_decoded_onehz_persistence_test.dart new file mode 100644 index 0000000..f0d0646 --- /dev/null +++ b/test/gen5_decoded_onehz_persistence_test.dart @@ -0,0 +1,232 @@ +// Gen5 v18 samples must land in `decoded_onehz` via the preferred-sample +// fallback in LocalDb._decodeOneHzSample — they lack gen4 optics so R24 +// decode fails, but they are honest 1 Hz substrate rows. R10-lite hr-only +// records must stay excluded. + +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/ble_engine.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/models.dart'; +import 'package:openstrap_protocol/openstrap_protocol.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +String _bytesToHex(Uint8List bytes) => + bytes.map((b) => b.toRadixString(16).padLeft(2, '0')).join(); + +Uint8List _buildR10LiteInner({required int ts, required int counter, required int hr}) { + final inner = Uint8List(18); + inner[0] = PacketType.historicalData; + inner[1] = Record.r10; + inner.buffer.asByteData().setUint32(3, counter, Endian.little); + inner.buffer.asByteData().setUint32(7, ts, Endian.little); + inner[17] = hr; + return inner; +} + +/// Synthetic gen5 v18 lenient inner: valid unix@7 + HR, gravity fails gate. +Uint8List _buildGen5V18LenientInner({ + required int unix, + required int counter, + required int hr, + List rrMs = const [], +}) { + final inner = Uint8List(112); + inner[0] = PacketType.historicalData; + inner[1] = 18; + inner[2] = 0x80; + inner[3] = counter & 0xff; + inner[4] = (counter >> 8) & 0xff; + inner[5] = (counter >> 16) & 0xff; + inner[6] = (counter >> 24) & 0xff; + inner.buffer.asByteData().setUint32(7, unix, Endian.little); + inner[14] = hr; + inner[15] = rrMs.length.clamp(0, 4).toInt(); + final view = inner.buffer.asByteData(); + for (var i = 0; i < rrMs.length && i < 4; i++) { + view.setInt16(16 + 2 * i, rrMs[i], Endian.little); + } + view.setFloat32(33, 0.5, Endian.little); + view.setFloat32(37, 0.05, Endian.little); + view.setFloat32(41, 0.05, Endian.little); + view.setFloat32(45, 0.05, Endian.little); + return inner; +} + +void main() { + setUpAll(() async { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + LocalDb.dbName = 'openstrap_gen5_onehz_test.db'; + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + }); + + tearDownAll(() async { + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + }); + + group('gen5 → decoded_onehz persistence', () { + test('gen5 v18-shaped sample persists via preferred fallback (+ RR)', () async { + // Real fixture inner — same bytes as gen5_sample_mapping_test.dart. + final frameHex = + 'aa01740001003fb12f1280733d8401b69f266a66460066025a0265020000000' + '000007b0a8d656463ff0012163cf6a439bf2924fd3ed763fe3e3200aa000000' + '000000000000f7000901f10b0007010c020c000000000000000000000000000' + '00000000000000000000100656f1e1e0000009d61a7c00000003e862817'; + final frame = Uint8List.fromList( + List.generate(frameHex.length ~/ 2, (i) { + return int.parse(frameHex.substring(i * 2, i * 2 + 2), radix: 16); + }), + ); + final parsed = parseFrame(frame, profile: BandProfile.gen5)!; + final inner = parsed.inner; + final sample = sampleFromGen5Historical(parseGen5Historical(inner)); + expect(sample, isNotNull); + + const recTs = 1780916150; + final raw = RawRecord( + counter: sample!.counter, + packetType: PacketType.historicalData, + hex: _bytesToHex(inner), + capturedAt: recTs * 1000, + recTs: recTs, + ); + + await LocalDb.commitSyncBatch([raw], [sample]); + + final db = await LocalDb.instance; + final rows = await db.query( + 'decoded_onehz', + where: 'rec_ts = ?', + whereArgs: [recTs], + ); + expect(rows.length, 1); + expect(rows.first['hr'], 102); + expect(rows.first['counter'], sample.counter); + + final rr = await db.query( + 'decoded_rr', + where: 'counter = ?', + whereArgs: [sample.counter], + ); + expect(rr.length, 2); + expect([for (final r in rr) r['rr_ms']], containsAll([602, 613])); + }); + + // NOTE what this pins, and what it does NOT. `decoded_onehz.ax/ay/az` are + // REAL NOT NULL, so absent gravity has to be STORED as 0 — that is a schema + // constraint, not a claim about the wrist. Exact (0,0,0) is therefore the + // ABSENT marker (no real gravity vector has zero magnitude, and every decoder + // that emits one gates on magSq >= 0.25); `Substrate.accelPresentAt` is what + // stops it being read back as a measurement. See + // substrate_accel_absence_test.dart — without that, a night of these scores + // as perfect immobility and fabricates a fully-staged sleep window. + test('lenient v18 with null accel still persists (stored as 0)', () async { + const unix = 1785801600; + const counter = 42; + final inner = _buildGen5V18LenientInner(unix: unix, counter: counter, hr: 72); + final sample = sampleFromGen5V18Lenient(inner, unix); + expect(sample, isNotNull); + expect(sample!.ax, isNull); + + final raw = RawRecord( + counter: counter, + packetType: PacketType.historicalData, + hex: _bytesToHex(inner), + capturedAt: unix * 1000, + recTs: unix, + ); + await LocalDb.commitSyncBatch([raw], [sample]); + + final db = await LocalDb.instance; + final rows = await db.query( + 'decoded_onehz', + where: 'rec_ts = ?', + whereArgs: [unix], + ); + expect(rows.length, 1); + expect(rows.first['hr'], 72); + expect(rows.first['ax'], 0); + expect(rows.first['ay'], 0); + expect(rows.first['az'], 0); + }); + + test('R10-lite + complete preferred → no decoded_onehz row', () async { + const ts = 1780000100; + const counter = 99; + final inner = _buildR10LiteInner(ts: ts, counter: counter, hr: 65); + final preferred = Sample( + tsEpoch: ts, + counter: counter, + hr: 65, + ax: 0.1, + ay: -0.2, + az: 0.95, + spo2RedRaw: 100, + spo2IrRaw: 200, + skinTempRaw: 300, + ); + expect(preferred.hasDecodedOneHz, isTrue); + final raw = RawRecord( + counter: counter, + packetType: PacketType.historicalData, + hex: _bytesToHex(inner), + capturedAt: ts * 1000, + recTs: ts, + ); + + await LocalDb.commitSyncBatch([raw], [preferred]); + + final db = await LocalDb.instance; + final rows = await db.query( + 'decoded_onehz', + where: 'rec_ts = ?', + whereArgs: [ts], + ); + expect(rows, isEmpty); + }); + + test('full gen4 R24 sample still persists', () async { + const ts = 1780000200; + const counter = 5001; + final sample = Sample( + tsEpoch: ts, + counter: counter, + hr: 70, + rrIntervalsMs: [800], + ax: 0.1, + ay: -0.2, + az: 0.95, + spo2RedRaw: 100, + spo2IrRaw: 200, + skinTempRaw: 300, + ); + // Minimal non-R10 historical hex — R24 decode won't match, but preferred + // has full gen4 optics so _decodeOneHzSample returns it immediately. + final raw = RawRecord( + counter: counter, + packetType: PacketType.historicalData, + hex: '2f18' '00' * 20, + capturedAt: ts * 1000, + recTs: ts, + ); + + await LocalDb.commitSyncBatch([raw], [sample]); + + final db = await LocalDb.instance; + final rows = await db.query( + 'decoded_onehz', + where: 'rec_ts = ?', + whereArgs: [ts], + ); + expect(rows.length, 1); + expect(rows.first['hr'], 70); + expect(rows.first['spo2_red_raw'], 100); + }); + }); +} diff --git a/test/gen5_v18_hardware_lenient_test.dart b/test/gen5_v18_hardware_lenient_test.dart new file mode 100644 index 0000000..a323f9e --- /dev/null +++ b/test/gen5_v18_hardware_lenient_test.dart @@ -0,0 +1,97 @@ +// Pins WHOOP 5 hardware v18 + clock payload behaviour against real evidence +// from openstrap_export_1785863370590.db / openstrap_sync.log (fw 50.40.1.0). + +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/ble_engine.dart'; +import 'package:openstrap_protocol/openstrap_protocol.dart'; + +Uint8List hex(String s) { + final clean = s.replaceAll(' ', ''); + final out = Uint8List(clean.length ~/ 2); + for (int i = 0; i < out.length; i++) { + out[i] = int.parse(clean.substring(i * 2, i * 2 + 2), radix: 16); + } + return out; +} + +void main() { + group('gen5 clock payloads — fw 50.40.1.0 Invalid revision evidence', () { + test('SET_CLOCK prepends revision1 before the 8-byte time', () { + final p = gen5SetClockPayload(sec: 1785710096, subsec: 16351); + expect(p, hasLength(9)); + expect(p[0], revision1); + // Seconds LE start at body[1] — not body[0] (that was read as "revision"). + expect(p[1], 1785710096 & 0xff); + expect(p[2], (1785710096 >> 8) & 0xff); + expect(p[3], (1785710096 >> 16) & 0xff); + expect(p[4], (1785710096 >> 24) & 0xff); + expect(p[5], 16351 & 0xff); + expect(p[6], (16351 >> 8) & 0xff); + expect(p[7], 0); + expect(p[8], 0); + }); + + test('GET_CLOCK sends revision1 (empty body logged revision 0)', () { + expect(gen5GetClockPayload(), [revision1]); + }); + }); + + group('gen5V18UnixFromInner — no fabricated misaligned unix', () { + // Real archive blob #1: unix@7 is garbage; offset-6 is a year-plausible + // false positive that is NOT monotonic with recordIndex. + final inner = hex( + '2f1280540df701737c6a915c2f004d0000000000000000000021fb608c4d0000' + 'c330ebf63ecd4ad43f854b653e5298863da3017400000000000000000039014401' + '040d6003010c020c3100000000000000000000000000000000000000000000011f' + 'bed68080000000a80372c0000000', + ); + + test('strict protocol decode returns null (gravity/dyn gate)', () { + expect(parseGen5Historical(inner), isNull); + }); + + test('unix@7 garbage → abstain (do not invent offset-6 time)', () { + const wallNow = 1785863370; // export capture era + expect(gen5V18UnixFromInner(inner, wallNow), isNull); + expect(sampleFromGen5V18Lenient(inner, wallNow), isNull); + expect(decodeGen5HistoricalSample(inner, wallNow), isNull); + }); + }); + + group('decodeGen5HistoricalSample — lenient v18 production path', () { + // Synthetic: valid shared header unix@7 + HR + RR, gravity out of gate range. + // Layout matches Gen5HistoricalHeader + HR @14. + test('recovers HR/RR when strict decode fails gravity gate only', () { + final inner = Uint8List(112); + inner[0] = 0x2f; + inner[1] = 18; + inner[2] = 0x80; + // recordIndex = 1 + inner[3] = 1; + // unix = 1785801600 (2026-08-04 00:00:00 UTC) + const unix = 1785801600; + inner.buffer.asByteData().setUint32(7, unix, Endian.little); + inner[14] = 72; // HR + inner[15] = 1; // one RR interval + inner.buffer.asByteData().setInt16(16, 820, Endian.little); + // dynAccel = 0.5 (ok), gravity magnitude ~0.1 (fails 0.5..1.5 gate) + inner.buffer.asByteData().setFloat32(33, 0.5, Endian.little); + inner.buffer.asByteData().setFloat32(37, 0.05, Endian.little); + inner.buffer.asByteData().setFloat32(41, 0.05, Endian.little); + inner.buffer.asByteData().setFloat32(45, 0.05, Endian.little); + + expect(parseGen5Historical(inner), isNull); + const wallNow = unix; // plausible vs the synthetic timestamp + final sample = decodeGen5HistoricalSample(inner, wallNow); + expect(sample, isNotNull); + expect(sample!.tsEpoch, unix); + expect(sample.hr, 72); + expect(sample.rrIntervalsMs, [820]); + expect(sample.ax, isNull); + expect(sample.ay, isNull); + expect(sample.az, isNull); + }); + }); +} diff --git a/test/substrate_accel_absence_test.dart b/test/substrate_accel_absence_test.dart new file mode 100644 index 0000000..14c7028 --- /dev/null +++ b/test/substrate_accel_absence_test.dart @@ -0,0 +1,166 @@ +// P0 REGRESSION — absent gravity must not be read back as perfect stillness. +// +// `sampleFromGen5V18Lenient` correctly ABSTAINS on a gravity vector that fails +// the magnitude gate (ax/ay/az stay null) while keeping HR/RR. But +// `decoded_onehz.ax/ay/az` are REAL NOT NULL, so the persistence layer writes +// `decoded.ax ?? 0` and the substrate loader reads `?? 0` back — turning +// "we did not measure this" into "the wrist was at exactly (0,0,0)". +// +// That is not an inert default. zAngle(0,0,0) is exactly 0.0 in Dart (atan2 +// (0,0) == 0.0, NOT NaN), so a run of absent seconds has a PERFECTLY CONSTANT +// z-angle, which is the van Hees immobility criterion satisfied maximally. +// Measured against the pinned analytics: 8 h of (0,0,0) yields 28 501 immobile +// seconds and `vanHeesSleepWindow.present == true` — a fabricated ~7.9 h night, +// fully staged, from data that does not exist. The PR's own commit message +// notes the strict gate rejected EVERY v18 record on fw 50.40.1.0, so this is +// the ordinary case for that firmware, not a corner. +// +// Two sentinels were tried and REJECTED, both verified against the pinned +// analytics rather than assumed: +// * NaN — fails OPEN. The rule tests "did the angle change by >= thr", and +// every comparison against NaN is false, so it never trips: NaN scores the +// SAME 28 501 immobile seconds as zeros. +// * dropping the seconds — `immobilityMask` is a pure index-wise angle rule +// with no timestamp/gap awareness (unlike `nap.dart`'s `stillAt`), so it +// simply joins across the hole. +// Hence the gate below: absent accel must not be allowed to ANCHOR a window. + +import 'dart:math' as math; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/substrate.dart'; + +Substrate _sub({ + required int n, + required bool accelPresent, + int startSec = 1750000000, +}) { + final ts = []; + final hr = []; + final ax = []; + final ay = []; + final az = []; + for (var i = 0; i < n; i++) { + ts.add(startSec + i); + hr.add(58); + if (accelPresent) { + // A real, still-ish wrist: unit-magnitude gravity with a slight drift. + final rad = (i % 3) * 0.5 * math.pi / 180.0; + ax.add(math.sin(rad)); + ay.add(0.0); + az.add(math.cos(rad)); + } else { + // What the NOT NULL column forces for an abstaining decoder. + ax.add(0.0); + ay.add(0.0); + az.add(0.0); + } + } + return Substrate( + tsSec: ts, + hr: hr, + rrTsMs: const [], + rrMs: const [], + ax: ax, + ay: ay, + az: az, + spo2Red: List.filled(n, 0), + spo2Ir: List.filled(n, 0), + skinTemp: List.filled(n, 0), + skinContact: List.filled(n, 0), + ); +} + +void main() { + group('the hazard this guards against is real', () { + test( + 'zAngle(0,0,0) is 0.0, not NaN — so absent accel is maximally "still"', + () { + expect(ana.zAngle(0.0, 0.0, 0.0), 0.0); + }, + ); + + test( + '8 h of absent accel scores as an almost entirely immobile night', + () { + const n = 8 * 3600; + final s = _sub(n: n, accelPresent: false); + final m = ana.vanHeesSleepWindow(s.accelSamples()); + final win = m.value!; + final immobile = win.immobile.where((b) => b).length; + expect( + immobile, + greaterThan((n * 0.95).round()), + reason: 'this is why the gate exists — missing data reads as sleep', + ); + expect(m.present, isTrue); + }, + ); + + test( + 'a NaN sentinel would NOT have fixed it — comparisons against NaN are ' + 'false, so the "angle changed" test never trips', + () { + const n = 8 * 3600; + final base = 1750000000000.0; + final nan = [ + for (var i = 0; i < n; i++) + ana.AccelSample( + base + i * 1000, double.nan, double.nan, double.nan), + ]; + final immobile = + ana.vanHeesSleepWindow(nan).value!.immobile.where((b) => b).length; + expect( + immobile, + greaterThan((n * 0.95).round()), + reason: 'NaN fails OPEN here; documented so nobody "fixes" it that way', + ); + }, + ); + }); + + group('Substrate.accelPresentAt / accelPresentFraction', () { + test('exact (0,0,0) is absent; a real vector is present', () { + final absent = _sub(n: 10, accelPresent: false); + final present = _sub(n: 10, accelPresent: true); + expect(absent.accelPresentAt(0), isFalse); + expect(present.accelPresentAt(0), isTrue); + expect(absent.accelPresentFraction(0, 10), 0.0); + expect(present.accelPresentFraction(0, 10), 1.0); + }); + + test('an empty range reports 0 — no evidence, not "all present"', () { + final s = _sub(n: 10, accelPresent: true); + expect(s.accelPresentFraction(5, 5), 0.0); + expect(Substrate.empty.accelPresentFraction(0, 100), 0.0); + }); + + test('a mixed window reports the real fraction', () { + final s = _sub(n: 100, accelPresent: true); + for (var i = 0; i < 40; i++) { + s.ax[i] = 0.0; + s.ay[i] = 0.0; + s.az[i] = 0.0; + } + expect(s.accelPresentFraction(0, 100), closeTo(0.60, 1e-9)); + }); + + test( + 'the van Hees coverage floor rejects an all-absent night and accepts a ' + 'fully-measured one', + () { + final absent = _sub(n: 8 * 3600, accelPresent: false); + final present = _sub(n: 8 * 3600, accelPresent: true); + expect( + absent.accelPresentFraction(0, absent.length), + lessThan(kMinAccelCoverageForVanHees), + ); + expect( + present.accelPresentFraction(0, present.length), + greaterThanOrEqualTo(kMinAccelCoverageForVanHees), + ); + }, + ); + }); +}