Skip to content

Commit cd6a67f

Browse files
authored
fix(ai): diagnose active provider residency (#13942) (#13943)
* fix(ai): diagnose active provider residency (#13942) * fix(ai): simplify recovery config leaf reads (#13942)
1 parent 4974090 commit cd6a67f

10 files changed

Lines changed: 651 additions & 72 deletions

ai/config.template.mjs

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -459,21 +459,23 @@ class Config extends ConfigProvider {
459459
* `NEO_DEPLOYMENT_STATE_BRIDGE_INCLUDE_LOGS`,
460460
* `NEO_DEPLOYMENT_STATE_BRIDGE_LOG_TAIL`,
461461
* `NEO_DEPLOYMENT_STATE_BRIDGE_LOG_MAX_BYTES`,
462-
* `NEO_DEPLOYMENT_STATE_BRIDGE_STATS_SAMPLE_WINDOW`.
462+
* `NEO_DEPLOYMENT_STATE_BRIDGE_STATS_SAMPLE_WINDOW`,
463+
* `NEO_DEPLOYMENT_STATE_BRIDGE_PROVIDER_RESIDENCY_SERVICE_KEYS`.
463464
*
464465
* @type {Object}
465466
*/
466467
deploymentStateBridge: {
467-
enabled : leaf(false, 'NEO_DEPLOYMENT_STATE_BRIDGE_ENABLED', 'boolean'),
468-
snapshotPath : leaf(path.resolve(neoRootDir, '.neo-ai-data/deployment-state/snapshot.json'), 'NEO_DEPLOYMENT_STATE_BRIDGE_SNAPSHOT_PATH', 'string'),
469-
writeIntervalMs : leaf(30000, 'NEO_DEPLOYMENT_STATE_BRIDGE_WRITE_INTERVAL_MS', 'number'),
470-
staleAfterMs : leaf(2 * 60 * 1000, 'NEO_DEPLOYMENT_STATE_BRIDGE_STALE_AFTER_MS', 'number'),
471-
maxSnapshotBytes : leaf(256 * 1024, 'NEO_DEPLOYMENT_STATE_BRIDGE_MAX_BYTES', 'number'),
472-
allowedServices : leaf([], 'NEO_DEPLOYMENT_STATE_BRIDGE_ALLOWED_SERVICES', 'csv'),
473-
includeLogs : leaf(true, 'NEO_DEPLOYMENT_STATE_BRIDGE_INCLUDE_LOGS', 'boolean'),
474-
logTail : leaf(120, 'NEO_DEPLOYMENT_STATE_BRIDGE_LOG_TAIL', 'number'),
475-
logMaxBytes : leaf(32 * 1024, 'NEO_DEPLOYMENT_STATE_BRIDGE_LOG_MAX_BYTES', 'number'),
476-
statsSampleWindow: leaf(2, 'NEO_DEPLOYMENT_STATE_BRIDGE_STATS_SAMPLE_WINDOW', 'number')
468+
enabled : leaf(false, 'NEO_DEPLOYMENT_STATE_BRIDGE_ENABLED', 'boolean'),
469+
snapshotPath : leaf(path.resolve(neoRootDir, '.neo-ai-data/deployment-state/snapshot.json'), 'NEO_DEPLOYMENT_STATE_BRIDGE_SNAPSHOT_PATH', 'string'),
470+
writeIntervalMs : leaf(30000, 'NEO_DEPLOYMENT_STATE_BRIDGE_WRITE_INTERVAL_MS', 'number'),
471+
staleAfterMs : leaf(2 * 60 * 1000, 'NEO_DEPLOYMENT_STATE_BRIDGE_STALE_AFTER_MS', 'number'),
472+
maxSnapshotBytes : leaf(256 * 1024, 'NEO_DEPLOYMENT_STATE_BRIDGE_MAX_BYTES', 'number'),
473+
allowedServices : leaf([], 'NEO_DEPLOYMENT_STATE_BRIDGE_ALLOWED_SERVICES', 'csv'),
474+
includeLogs : leaf(true, 'NEO_DEPLOYMENT_STATE_BRIDGE_INCLUDE_LOGS', 'boolean'),
475+
logTail : leaf(120, 'NEO_DEPLOYMENT_STATE_BRIDGE_LOG_TAIL', 'number'),
476+
logMaxBytes : leaf(32 * 1024, 'NEO_DEPLOYMENT_STATE_BRIDGE_LOG_MAX_BYTES', 'number'),
477+
statsSampleWindow : leaf(2, 'NEO_DEPLOYMENT_STATE_BRIDGE_STATS_SAMPLE_WINDOW', 'number'),
478+
providerResidencyServiceKeys: leaf(['local-model', 'model'], 'NEO_DEPLOYMENT_STATE_BRIDGE_PROVIDER_RESIDENCY_SERVICE_KEYS', 'csv')
477479
},
478480
/**
479481
* Maintenance-loop intervals consumed by the orchestrator daemon.
@@ -683,13 +685,14 @@ class Config extends ConfigProvider {
683685
tenantRepoSyncEnabled: leaf(null, 'NEO_ORCHESTRATOR_TENANT_REPO_SYNC_ENABLED', 'boolean')
684686
},
685687
/**
686-
* ADR-0026 B1 recovery actuator envelope. Disabled and empty by default: the
687-
* orchestrator may only restart compose services or page deploy targets that are
688-
* explicitly named here by the operator.
688+
* Recovery actuator envelope. Disabled and empty by default: the
689+
* orchestrator may only recycle supervised tasks, restart compose services, or
690+
* page deploy targets that are explicitly named here by the operator.
689691
* @type {Object}
690692
*/
691693
recoveryActuator: {
692694
enabled : leaf(false, 'NEO_RECOVERY_ACTUATOR_ENABLED', 'boolean'),
695+
allowedSupervisedTasks : leaf([], 'NEO_RECOVERY_ACTUATOR_SUPERVISED_TASKS', 'csv'),
693696
allowedComposeServices : leaf([], 'NEO_RECOVERY_ACTUATOR_COMPOSE_SERVICES', 'csv'),
694697
allowedDeployTargets : leaf([], 'NEO_RECOVERY_ACTUATOR_DEPLOY_TARGETS', 'csv'),
695698
healAttemptsPath : leaf(path.resolve(neoRootDir, '.neo-ai-data/orchestrator-daemon/heal-attempts.json'), 'NEO_RECOVERY_ACTUATOR_HEAL_ATTEMPTS_PATH', 'string'),

ai/daemons/orchestrator/Orchestrator.mjs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,7 @@ export class Orchestrator extends Base {
337337
dataDir : this.dataDir,
338338
deploymentRuntimeAccessService: this.deploymentRuntimeAccessService,
339339
healthService : this.healthService,
340+
processSupervisorService : this.processSupervisorService,
340341
writeLog : this.processSupervisorWriteLog,
341342
actuatorConfig : AiConfig.orchestrator.recoveryActuator
342343
});
@@ -353,6 +354,11 @@ export class Orchestrator extends Base {
353354
this.processSupervisorService.taskDefinitions = value;
354355
this.maintenanceBackpressureService.taskDefinitions = value;
355356
}
357+
afterSetProcessSupervisorService(value, oldValue) {
358+
if (this.recoveryActuatorService) {
359+
this.recoveryActuatorService.processSupervisorService = value;
360+
}
361+
}
356362
afterSetTaskStateService(value, oldValue) {
357363
if (oldValue === undefined) return;
358364
this.processSupervisorService.taskStateService = value;

ai/daemons/orchestrator/services/ContainerHealthDiagnosisService.mjs

Lines changed: 198 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import Base from '../../../../src/core/Base.mjs';
22

3-
import {createRecoveryDiagnosisEvent} from '../../../services/memory-core/helpers/recoveryRunStateStore.mjs';
3+
import {
4+
createRecoveryDiagnosisEvent,
5+
createRecoveryTargetIdentity
6+
} from '../../../services/memory-core/helpers/recoveryRunStateStore.mjs';
47

58
export const CONTAINER_HEALTH_FACT_TYPES = Object.freeze({
69
configDrift : 'config-drift',
@@ -9,6 +12,7 @@ export const CONTAINER_HEALTH_FACT_TYPES = Object.freeze({
912
endpointProbeFailed: 'endpoint-probe-failed',
1013
evalContention : 'ollama-eval-contention',
1114
memorySaturation : 'memory-saturation',
15+
providerResidency : 'provider-residency-degraded',
1216
resourceSaturation : 'resource-saturation',
1317
runtimeReadFailed : 'runtime-read-failed'
1418
});
@@ -86,6 +90,7 @@ export class ContainerHealthDiagnosisService extends Base {
8690
* @param {Object|null} [options.endpointProbe=null] Direct service probe result.
8791
* @param {Object|null} [options.configCheck=null] Config correctness result.
8892
* @param {Object|null} [options.ollamaEvalAttribution=null] Native Ollama eval attribution.
93+
* @param {Object|null} [options.providerResidency=null] Active-provider residency observation.
8994
* @param {Number} [options.observedAt] Epoch milliseconds for the observation.
9095
* @returns {Object} Diagnosis decision with advisory facts and optional diagnosis event.
9196
*/
@@ -97,6 +102,7 @@ export class ContainerHealthDiagnosisService extends Base {
97102
endpointProbe = null,
98103
configCheck = null,
99104
ollamaEvalAttribution = null,
105+
providerResidency = null,
100106
observedAt = this.now()
101107
} = {}) {
102108
this.validateServiceKey(serviceKey);
@@ -106,7 +112,8 @@ export class ContainerHealthDiagnosisService extends Base {
106112
...this.collectStatsFacts({serviceKey, stats, statsSamples, observedAt}),
107113
...this.collectEndpointProbeFacts({serviceKey, endpointProbe, observedAt}),
108114
...this.collectConfigFacts({serviceKey, configCheck, observedAt}),
109-
...this.collectEvalAttributionFacts({serviceKey, ollamaEvalAttribution, observedAt})
115+
...this.collectEvalAttributionFacts({serviceKey, ollamaEvalAttribution, observedAt}),
116+
...this.collectProviderResidencyFacts({serviceKey, providerResidency, observedAt})
110117
];
111118

112119
const classification = this.classifyFacts({facts});
@@ -123,7 +130,7 @@ export class ContainerHealthDiagnosisService extends Base {
123130
diagnosisId : this.createDiagnosisId({serviceKey, observedAt, recoveryClass: classification.recoveryClass}),
124131
recoveryClass : classification.recoveryClass,
125132
confidence : classification.confidence,
126-
targetIdentity: {kind: 'compose-service', id: serviceKey},
133+
targetIdentity: this.resolveDiagnosisTargetIdentity({serviceKey, classification}),
127134
evidenceFacts : classification.evidenceFacts,
128135
observedAt,
129136
source : 'container-health-diagnostics',
@@ -156,6 +163,7 @@ export class ContainerHealthDiagnosisService extends Base {
156163
* @param {Function|null} [options.endpointProbeFn=null] Optional direct probe callback.
157164
* @param {Function|null} [options.configCheckFn=null] Optional config-check callback.
158165
* @param {Object|null} [options.ollamaEvalAttribution=null] Optional eval attribution payload.
166+
* @param {Object|null} [options.providerResidency=null] Optional active-provider residency observation.
159167
* @param {Object[]|null} [options.statsSamples=null] Optional stats samples spanning the window.
160168
* @param {Number} [options.observedAt] Epoch milliseconds for the observation.
161169
* @returns {Promise<Object>} Diagnosis decision.
@@ -166,6 +174,7 @@ export class ContainerHealthDiagnosisService extends Base {
166174
endpointProbeFn = null,
167175
configCheckFn = null,
168176
ollamaEvalAttribution = null,
177+
providerResidency = null,
169178
statsSamples = null,
170179
observedAt = this.now()
171180
} = {}) {
@@ -203,6 +212,7 @@ export class ContainerHealthDiagnosisService extends Base {
203212
endpointProbe,
204213
configCheck,
205214
ollamaEvalAttribution,
215+
providerResidency,
206216
observedAt
207217
});
208218

@@ -389,20 +399,84 @@ export class ContainerHealthDiagnosisService extends Base {
389399
})];
390400
}
391401

402+
/**
403+
* Collects active-provider residency facts from provider-specific probes.
404+
*
405+
* Diagnostics own normalization only: callers supply LMS, Ollama, or future-provider facts;
406+
* recovery later routes by the typed target identity.
407+
*
408+
* @param {Object} options
409+
* @returns {Object[]}
410+
*/
411+
collectProviderResidencyFacts({serviceKey, providerResidency, observedAt}) {
412+
if (!providerResidency || typeof providerResidency !== 'object') return [];
413+
414+
const reasonCode = this.getProviderResidencyReasonCode(providerResidency);
415+
if (!reasonCode) return [];
416+
417+
return [this.createFact({
418+
type : CONTAINER_HEALTH_FACT_TYPES.providerResidency,
419+
serviceKey,
420+
observedAt,
421+
severity : ['extra-residents', 'provider-probe-failed'].includes(reasonCode) ? 'warning' : 'critical',
422+
authoritative: !['extra-residents', 'provider-probe-failed'].includes(reasonCode),
423+
details : {
424+
provider : typeof providerResidency.provider === 'string' ? providerResidency.provider : null,
425+
host : typeof providerResidency.host === 'string' ? providerResidency.host : null,
426+
message : typeof providerResidency.message === 'string' ? providerResidency.message : null,
427+
reasonCode,
428+
ready : typeof providerResidency.ready === 'boolean' ? providerResidency.ready : null,
429+
degraded : typeof providerResidency.degraded === 'boolean' ? providerResidency.degraded : null,
430+
targetIdentity : this.readProviderTargetIdentity(providerResidency.targetIdentity),
431+
requiredModels : toSafeArray(providerResidency.requiredModels),
432+
availableModels : toSafeArray(providerResidency.availableModels),
433+
missingModels : toSafeArray(providerResidency.missingModels),
434+
insufficientContextModels: toSafeArray(providerResidency.insufficientContextModels),
435+
extraModels : toSafeArray(providerResidency.extraModels),
436+
failedModels : toSafeArray(providerResidency.failedModels),
437+
loadedContexts : toSafeObject(providerResidency.loadedContexts),
438+
observedRequiredCount : toSafeNumber(providerResidency.observedRequiredCount),
439+
requiredResidentModels : toSafeNumber(providerResidency.requiredResidentModels),
440+
residentButNotServing : providerResidency.residentButNotServing === true
441+
}
442+
})];
443+
}
444+
392445
/**
393446
* Classifies facts into the recovery diagnosis contract.
394447
* @param {Object} options
395448
* @returns {Object|null}
396449
*/
397450
classifyFacts({facts}) {
398-
if (facts.some(fact => fact.type === CONTAINER_HEALTH_FACT_TYPES.configDrift)) {
399-
const evidenceFacts = facts.filter(fact => fact.type === CONTAINER_HEALTH_FACT_TYPES.configDrift);
451+
const configDriftFacts = facts.filter(fact =>
452+
fact.type === CONTAINER_HEALTH_FACT_TYPES.configDrift ||
453+
this.isProviderResidencyConfigDrift(fact)
454+
);
455+
456+
if (configDriftFacts.length > 0) {
457+
const providerTargetIdentity = this.resolveProviderFactTargetIdentity(configDriftFacts.find(fact =>
458+
fact.type === CONTAINER_HEALTH_FACT_TYPES.providerResidency
459+
));
460+
400461
return {
401-
recoveryClass: 'config-drift',
402-
actionClass : CONTAINER_HEALTH_ACTION_CLASSES.escalate,
403-
confidence : 0.95,
404-
evidenceFacts,
405-
reason : 'config-drift-escalate'
462+
recoveryClass : 'config-drift',
463+
actionClass : CONTAINER_HEALTH_ACTION_CLASSES.escalate,
464+
confidence : 0.95,
465+
evidenceFacts : configDriftFacts,
466+
reason : 'config-drift-escalate',
467+
targetIdentity: providerTargetIdentity
468+
};
469+
}
470+
471+
const providerCrashFacts = facts.filter(fact => this.isProviderResidentButNotServing(fact));
472+
if (providerCrashFacts.length > 0) {
473+
return {
474+
recoveryClass : 'crash',
475+
actionClass : CONTAINER_HEALTH_ACTION_CLASSES.restart,
476+
confidence : 0.85,
477+
evidenceFacts : this.selectEvidenceFacts(facts, providerCrashFacts),
478+
reason : 'provider-resident-not-serving',
479+
targetIdentity: this.resolveProviderFactTargetIdentity(providerCrashFacts[0])
406480
};
407481
}
408482

@@ -514,7 +588,7 @@ export class ContainerHealthDiagnosisService extends Base {
514588
schemaVersion : 1,
515589
recordType : 'container-health-diagnosis-decision',
516590
serviceKey,
517-
targetIdentity: {kind: 'compose-service', id: serviceKey},
591+
targetIdentity: diagnosis?.targetIdentity || {kind: 'compose-service', id: serviceKey},
518592
observedAt,
519593
status,
520594
actionClass,
@@ -553,6 +627,103 @@ export class ContainerHealthDiagnosisService extends Base {
553627
throw new TypeError(`Container health diagnosis serviceKey '${serviceKey}' contains unsupported characters`);
554628
}
555629
}
630+
631+
/**
632+
* Resolves the diagnosis target without letting raw probe data bypass the target enum.
633+
* @param {Object} options
634+
* @returns {Object}
635+
*/
636+
resolveDiagnosisTargetIdentity({serviceKey, classification}) {
637+
return classification?.targetIdentity || createRecoveryTargetIdentity({
638+
kind: 'compose-service',
639+
id : serviceKey
640+
});
641+
}
642+
643+
/**
644+
* Determines the provider-residency reason code that diagnostics can classify.
645+
* @param {Object} providerResidency Active-provider residency observation.
646+
* @returns {String|null}
647+
*/
648+
getProviderResidencyReasonCode(providerResidency) {
649+
if (providerResidency.residentButNotServing === true) {
650+
return 'resident-not-serving';
651+
}
652+
if (toSafeArray(providerResidency.missingModels).length > 0) {
653+
return 'missing-required-model';
654+
}
655+
if (toSafeArray(providerResidency.insufficientContextModels).length > 0) {
656+
return 'insufficient-context';
657+
}
658+
if (toSafeArray(providerResidency.failedModels).length > 0) {
659+
return 'provider-warmup-failed';
660+
}
661+
if (providerResidency.supported === false) {
662+
return 'unsupported-provider';
663+
}
664+
if (providerResidency.probeFailed === true) {
665+
return 'provider-probe-failed';
666+
}
667+
if (providerResidency.ready === false) {
668+
return 'provider-not-ready';
669+
}
670+
if (toSafeArray(providerResidency.extraModels).length > 0) {
671+
return 'extra-residents';
672+
}
673+
674+
return null;
675+
}
676+
677+
/**
678+
* Checks whether a provider-residency fact should page instead of looping recovery.
679+
* @param {Object} fact Diagnosis fact.
680+
* @returns {Boolean}
681+
*/
682+
isProviderResidencyConfigDrift(fact) {
683+
return fact.type === CONTAINER_HEALTH_FACT_TYPES.providerResidency && [
684+
'insufficient-context',
685+
'missing-required-model',
686+
'provider-not-ready',
687+
'provider-warmup-failed',
688+
'unsupported-provider'
689+
].includes(fact.details?.reasonCode);
690+
}
691+
692+
/**
693+
* Checks whether a provider-residency fact proves a resident-but-not-serving fault.
694+
* @param {Object} fact Diagnosis fact.
695+
* @returns {Boolean}
696+
*/
697+
isProviderResidentButNotServing(fact) {
698+
return fact.type === CONTAINER_HEALTH_FACT_TYPES.providerResidency &&
699+
fact.details?.reasonCode === 'resident-not-serving';
700+
}
701+
702+
/**
703+
* Reads and validates a provider fact target identity.
704+
* @param {Object|null} targetIdentity Target identity candidate.
705+
* @returns {Object|null}
706+
*/
707+
readProviderTargetIdentity(targetIdentity) {
708+
if (!targetIdentity || typeof targetIdentity !== 'object') return null;
709+
710+
try {
711+
return createRecoveryTargetIdentity(targetIdentity);
712+
} catch {
713+
return null;
714+
}
715+
}
716+
717+
/**
718+
* Resolves the first typed provider fact target identity.
719+
* @param {Object} fact Diagnosis fact.
720+
* @returns {Object|null}
721+
*/
722+
resolveProviderFactTargetIdentity(fact) {
723+
if (!fact) return null;
724+
725+
return this.readProviderTargetIdentity(fact.details?.targetIdentity);
726+
}
556727
}
557728

558729
export function calculateDockerCpuPercent(stats) {
@@ -625,4 +796,20 @@ function toSafeScalar(value) {
625796
return JSON.stringify(value).slice(0, 256);
626797
}
627798

799+
function toSafeArray(value) {
800+
return Array.isArray(value)
801+
? value.map(item => toSafeScalar(item)).filter(item => item !== null)
802+
: [];
803+
}
804+
805+
function toSafeObject(value) {
806+
if (!value || typeof value !== 'object' || Array.isArray(value)) return {};
807+
808+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, toSafeScalar(item)]));
809+
}
810+
811+
function toSafeNumber(value) {
812+
return Number.isFinite(value) ? value : null;
813+
}
814+
628815
export default Neo.setupClass(ContainerHealthDiagnosisService);

0 commit comments

Comments
 (0)