feat(ai): hung-heavy-lease watchdog decision (pure) — sub of #13624 (#13760) (#13761)

Co-authored-by: tobiu <tobiasuhlig78@gmail.com>

Author: neo-opus-ada

ai/daemons/orchestrator/services/leaseWatchdog.mjs

@@ -0,0 +1,38 @@
+/**
+ * @module Neo.ai.daemons.orchestrator.services.leaseWatchdog
+ * @summary Pure decision for the hung-heavy-lease watchdog. The exclusive-heavy-maintenance lease
+ * releases on pid-death + TTL-expiry (`isLeaseStale` in `HeavyMaintenanceLeaseService`), but neither
+ * catches a holder that is ALIVE + within-TTL yet HUNG (sustained ~0% cpu — e.g. blocked on a wedged
+ * embedder). A hung holder otherwise monopolizes the lease until the full TTL, starving every other
+ * maintenance task (the orchestrator DRAIN). The orchestrator lease-monitor loop samples the active
+ * lease holder's cpu% over time and force-releases on a true verdict; THIS module is the pure verdict
+ * only — the ps-sampling and the force-release are the integration (kept out of this slice).
+ */
+
+/**
+ * @summary Decides whether the heavy-lease holder is HUNG — a sustained-idle process holding the lease
+ * without progressing. Returns `true` iff the last `minConsecutiveIdle` cpu samples are ALL at or below
+ * `idleThresholdPct`: a single idle sample between work bursts is normal, so only a sustained trailing
+ * run counts as a hang. Total + never-throws (it runs inside the orchestrator loop, where a throw would
+ * trap the daemon): a non-array, too-short, or malformed sample list returns `false` — fail-SAFE, the
+ * watchdog never force-releases on bad/insufficient data. A non-finite sample is treated as non-idle
+ * (same fail-safe: don't conclude "hung" from a missing reading). Exported + unit-tested.
+ * @param {Object} [options]
+ * @param {Number[]} [options.cpuPercentSamples=[]] Recent cpu% samples for the lease holder pid, oldest→newest.
+ * @param {Number} [options.idleThresholdPct=1] A sample at or below this cpu% counts as idle.
+ * @param {Number} [options.minConsecutiveIdle=3] Consecutive trailing idle samples required to call it hung.
+ * @returns {Boolean}
+ */
+export function isHungLeaseHolder({
+    cpuPercentSamples  = [],
+    idleThresholdPct   = 1,
+    minConsecutiveIdle = 3
+} = {}) {
+    if (!Array.isArray(cpuPercentSamples))                              return false;
+    if (!Number.isFinite(minConsecutiveIdle) || minConsecutiveIdle < 1) return false;
+    if (cpuPercentSamples.length < minConsecutiveIdle)                 return false;
+
+    return cpuPercentSamples
+        .slice(-minConsecutiveIdle)
+        .every(sample => Number.isFinite(sample) && sample <= idleThresholdPct);
+}

test/playwright/unit/ai/daemons/orchestrator/services/leaseWatchdog.spec.mjs

@@ -0,0 +1,40 @@
+import {test, expect}      from '@playwright/test';
+import {isHungLeaseHolder} from '../../../../../../../ai/daemons/orchestrator/services/leaseWatchdog.mjs';
+
+/**
+ * Coverage for the pure hung-heavy-lease watchdog decision (sub of the orchestrator heavy-maintenance
+ * epic). Catches the alive + within-TTL but sustained-idle holder the pid/TTL stale-check misses. The
+ * sustained-run semantics + the fail-SAFE-on-bad-data contract are pinned here.
+ */
+test.describe('ai/daemons/orchestrator/services/leaseWatchdog — isHungLeaseHolder', () => {
+    test('a sustained trailing-idle run → hung (true)', () => {
+        expect(isHungLeaseHolder({cpuPercentSamples: [42, 0, 0.2, 0.1]})).toBe(true);
+    });
+
+    test('an active sample anywhere in the trailing window → not hung (false)', () => {
+        expect(isHungLeaseHolder({cpuPercentSamples: [0, 0, 35]})).toBe(false);  // newest active
+        expect(isHungLeaseHolder({cpuPercentSamples: [0, 35, 0]})).toBe(false);  // active mid-window
+    });
+
+    test('too few samples → false (cannot conclude a sustained hang)', () => {
+        expect(isHungLeaseHolder({cpuPercentSamples: [0, 0]})).toBe(false);  // < minConsecutiveIdle (3)
+    });
+
+    test('boundary: a sample exactly at the threshold counts as idle', () => {
+        expect(isHungLeaseHolder({cpuPercentSamples: [1, 1, 1], idleThresholdPct: 1})).toBe(true);
+        expect(isHungLeaseHolder({cpuPercentSamples: [1.1, 1, 1], idleThresholdPct: 1})).toBe(false); // 1.1 > 1
+    });
+
+    test('respects a custom minConsecutiveIdle window', () => {
+        expect(isHungLeaseHolder({cpuPercentSamples: [0, 0], minConsecutiveIdle: 2})).toBe(true);
+        expect(isHungLeaseHolder({cpuPercentSamples: [5, 0], minConsecutiveIdle: 2})).toBe(false);
+    });
+
+    test('fail-SAFE on malformed / insufficient input (never force-release on bad data)', () => {
+        expect(isHungLeaseHolder()).toBe(false);                                          // no args
+        expect(isHungLeaseHolder({cpuPercentSamples: 'nope'})).toBe(false);               // non-array
+        expect(isHungLeaseHolder({cpuPercentSamples: []})).toBe(false);                   // empty
+        expect(isHungLeaseHolder({cpuPercentSamples: [0, 0, NaN]})).toBe(false);          // non-finite → not idle
+        expect(isHungLeaseHolder({cpuPercentSamples: [0, 0, 0], minConsecutiveIdle: 0})).toBe(false); // bad window
+    });
+});