Skip to content

Commit 6fc44c4

Browse files
authored
fix(ai): enable recovery actuator by default (#13952) (#13953)
* fix(ai): enable recovery actuator by default (#13952) * docs(ai): document recovery actuator opt-out default (#13952) * fix(ai): enable recovery targets by default (#13952)
1 parent d09f416 commit 6fc44c4

6 files changed

Lines changed: 194 additions & 107 deletions

File tree

ai/config.template.mjs

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -692,16 +692,17 @@ class Config extends ConfigProvider {
692692
tenantRepoSyncEnabled: leaf(null, 'NEO_ORCHESTRATOR_TENANT_REPO_SYNC_ENABLED', 'boolean')
693693
},
694694
/**
695-
* Recovery actuator envelope. Disabled and empty by default: the
696-
* orchestrator may only recycle supervised tasks, restart compose services, or
697-
* page deploy targets that are explicitly named here by the operator.
695+
* Recovery actuator envelope. Enabled by default so deployed immune-system
696+
* lanes can heal without per-deployment recovery target allowlists. Operators
697+
* can block specific supervised tasks, compose services, or deploy targets while
698+
* the runtime-access holder still gates compose services to known labels.
698699
* @type {Object}
699700
*/
700701
recoveryActuator: {
701-
enabled : leaf(false, 'NEO_RECOVERY_ACTUATOR_ENABLED', 'boolean'),
702-
allowedSupervisedTasks : leaf([], 'NEO_RECOVERY_ACTUATOR_SUPERVISED_TASKS', 'csv'),
703-
allowedComposeServices : leaf([], 'NEO_RECOVERY_ACTUATOR_COMPOSE_SERVICES', 'csv'),
704-
allowedDeployTargets : leaf([], 'NEO_RECOVERY_ACTUATOR_DEPLOY_TARGETS', 'csv'),
702+
enabled : leaf(true, 'NEO_RECOVERY_ACTUATOR_ENABLED', 'boolean'),
703+
blockedSupervisedTasks : leaf([], 'NEO_RECOVERY_ACTUATOR_BLOCKED_SUPERVISED_TASKS', 'csv'),
704+
blockedComposeServices : leaf([], 'NEO_RECOVERY_ACTUATOR_BLOCKED_COMPOSE_SERVICES', 'csv'),
705+
blockedDeployTargets : leaf([], 'NEO_RECOVERY_ACTUATOR_BLOCKED_DEPLOY_TARGETS', 'csv'),
705706
healAttemptsPath : leaf(path.resolve(neoRootDir, '.neo-ai-data/orchestrator-daemon/heal-attempts.json'), 'NEO_RECOVERY_ACTUATOR_HEAL_ATTEMPTS_PATH', 'string'),
706707
recoveryRunStateDir : leaf(path.resolve(neoRootDir, '.neo-ai-data/orchestrator-daemon/recovery-runs'), 'NEO_RECOVERY_ACTUATOR_RUN_STATE_DIR', 'string'),
707708
recoveryRunRetentionLimit : leaf(100, 'NEO_RECOVERY_ACTUATOR_RUN_RETENTION_LIMIT', 'number'),

ai/daemons/orchestrator/services/RecoveryActuatorService.mjs

Lines changed: 71 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -13,22 +13,22 @@ import {
1313
} from '../../../services/memory-core/helpers/recoveryRunStateStore.mjs';
1414
import {DEFAULT_DATA_DIR} from '../taskDefinitions.mjs';
1515

16-
const DEFAULT_ACTIONS = Object.freeze(['restart', 'redeploy', 'page', 'warm-provider']);
16+
const DEFAULT_ACTIONS = Object.freeze(['restart', 'redeploy', 'page', 'warm-provider']);
17+
const DEFAULT_DEPLOY_TARGETS = Object.freeze(['cloud-deploy']);
1718

1819
/**
19-
* @summary Normalizes string/object allowlist entries into stable recovery targets.
20+
* @summary Normalizes string/object recovery-target entries into stable descriptors.
2021
*
2122
* String entries use the same value for `serviceKey` and `id`. Object entries may use an
22-
* explicit `serviceKey` plus either `id`, `taskName`, `composeService`, or `deployTarget`. The actuator
23-
* never accepts a runtime target that cannot be traced back to one of these normalized entries.
23+
* explicit `serviceKey` plus either `id`, `taskName`, `composeService`, or `deployTarget`.
2424
*
25-
* @param {Array<String|Object>} entries Configured allowlist entries.
25+
* @param {Array<String|Object>} entries Configured recovery target entries.
2626
* @param {String} kind Recovery target kind.
2727
* @returns {Object[]} Normalized target descriptors.
2828
*/
29-
export function normalizeRecoveryActuatorAllowlist(entries, kind) {
29+
export function normalizeRecoveryActuatorTargets(entries, kind) {
3030
if (!Array.isArray(entries)) {
31-
return [];
31+
throw new TypeError(`Recovery actuator ${kind} targets must be an array.`);
3232
}
3333

3434
const targets = [];
@@ -64,13 +64,28 @@ export function normalizeRecoveryActuatorAllowlist(entries, kind) {
6464
return targets;
6565
}
6666

67+
/**
68+
* @summary Checks whether a recovery target is blocked by the operator's opt-out list.
69+
* @param {Object} target Normalized target descriptor.
70+
* @param {Object[]} blockedTargets Normalized blocklist descriptors.
71+
* @returns {Boolean}
72+
*/
73+
export function isRecoveryActuatorTargetBlocked(target, blockedTargets) {
74+
return blockedTargets.some(blocked => (
75+
blocked.serviceKey === target.serviceKey ||
76+
blocked.serviceKey === target.id ||
77+
blocked.id === target.serviceKey ||
78+
blocked.id === target.id
79+
));
80+
}
81+
6782
/**
6883
* @class Neo.ai.daemons.services.RecoveryActuatorService
6984
* @extends Neo.core.Base
7085
*
7186
* B1 privileged recovery actuator for ADR-0026. The service is controller-blind:
7287
* callers pass an already-selected action, and this class only answers whether the
73-
* actuator allowlist + persisted anti-thrash envelope admits it. Compose-service
88+
* recovery target registry + persisted anti-thrash envelope admits it. Compose-service
7489
* lifecycle writes are delegated to the shared L0 deployment-runtime access holder,
7590
* keeping Docker socket access and container identity resolution out of this B1 class.
7691
*/
@@ -145,25 +160,55 @@ export class RecoveryActuatorService extends Base {
145160
return this.cfg.recoveryRunStateDir;
146161
}
147162

148-
/** @summary Resolves the allowlisted compose-service actuator targets. */
149-
get allowedComposeServices() {
150-
return normalizeRecoveryActuatorAllowlist(this.cfg.allowedComposeServices, 'compose-service');
163+
/** @summary Resolves the configured compose-service recovery blocklist. */
164+
get blockedComposeServices() {
165+
return normalizeRecoveryActuatorTargets(this.cfg.blockedComposeServices, 'compose-service');
166+
}
167+
168+
/** @summary Resolves the configured supervised-task recovery blocklist. */
169+
get blockedSupervisedTasks() {
170+
return normalizeRecoveryActuatorTargets(this.cfg.blockedSupervisedTasks, 'supervised-task');
151171
}
152172

153-
/** @summary Resolves the allowlisted supervised-task actuator targets. */
154-
get allowedSupervisedTasks() {
155-
return normalizeRecoveryActuatorAllowlist(this.cfg.allowedSupervisedTasks, 'supervised-task');
173+
/** @summary Resolves the configured deploy-target recovery blocklist. */
174+
get blockedDeployTargets() {
175+
return normalizeRecoveryActuatorTargets(this.cfg.blockedDeployTargets, 'deploy-target');
176+
}
177+
178+
/** @summary Resolves all currently-known supervised-task recovery targets except blocked ones. */
179+
get supervisedTaskTargets() {
180+
const taskNames = Object.keys(this.processSupervisorService?.taskDefinitions || {});
181+
182+
return taskNames
183+
.map(taskName => ({
184+
kind : 'supervised-task',
185+
serviceKey: taskName,
186+
id : taskName,
187+
taskName
188+
}))
189+
.filter(target => !isRecoveryActuatorTargetBlocked(target, this.blockedSupervisedTasks));
190+
}
191+
192+
/** @summary Resolves all runtime-access compose-service recovery targets except blocked ones. */
193+
get composeServiceTargets() {
194+
const runtimeAccessConfig = this.deploymentRuntimeAccessService
195+
? this.deploymentRuntimeAccessService.runtimeAccessConfig
196+
: AiConfig.orchestrator.deploymentRuntimeAccess;
197+
198+
return normalizeRecoveryActuatorTargets(runtimeAccessConfig.allowedServices, 'compose-service')
199+
.filter(target => !isRecoveryActuatorTargetBlocked(target, this.blockedComposeServices));
156200
}
157201

158-
/** @summary Resolves the allowlisted deploy-target page/escalation targets. */
159-
get allowedDeployTargets() {
160-
return normalizeRecoveryActuatorAllowlist(this.cfg.allowedDeployTargets, 'deploy-target');
202+
/** @summary Resolves all built-in deploy-target recovery targets except blocked ones. */
203+
get deployTargets() {
204+
return normalizeRecoveryActuatorTargets(DEFAULT_DEPLOY_TARGETS, 'deploy-target')
205+
.filter(target => !isRecoveryActuatorTargetBlocked(target, this.blockedDeployTargets));
161206
}
162207

163208
/**
164-
* @summary Applies one bounded recovery action if the allowlist and anti-thrash envelope admit it.
209+
* @summary Applies one bounded recovery action if the target registry and anti-thrash envelope admit it.
165210
*
166-
* @param {String} serviceKey Stable allowlisted recovery target key.
211+
* @param {String} serviceKey Stable recovery target key.
167212
* @param {String} action restart | redeploy | page | warm-provider.
168213
* @param {Object} [options]
169214
* @param {Object|null} [options.diagnosisEvent=null] Optional ADR-0025 diagnosis event.
@@ -193,7 +238,7 @@ export class RecoveryActuatorService extends Base {
193238
const target = this.resolveTarget({serviceKey, action, targetIdentity});
194239

195240
if (!target) {
196-
return this.rejectAction({serviceKey, action, now, reasonCode: 'target-not-allowlisted', targetIdentity});
241+
return this.rejectAction({serviceKey, action, now, reasonCode: 'target-not-recoverable', targetIdentity});
197242
}
198243

199244
if (!this.isActionAllowedForTarget({action, target})) {
@@ -312,15 +357,15 @@ export class RecoveryActuatorService extends Base {
312357
}
313358

314359
/**
315-
* @summary Resolves the strict target allowlist entry for a service key and optional identity.
360+
* @summary Resolves the strict recovery target entry for a service key and optional identity.
316361
* @param {Object} options
317362
* @returns {Object|null}
318363
*/
319364
resolveTarget({serviceKey, targetIdentity}) {
320365
const candidates = [
321-
...this.allowedComposeServices,
322-
...this.allowedSupervisedTasks,
323-
...this.allowedDeployTargets
366+
...this.composeServiceTargets,
367+
...this.supervisedTaskTargets,
368+
...this.deployTargets
324369
].filter(target => target.serviceKey === serviceKey || target.id === serviceKey);
325370

326371
if (targetIdentity) {
@@ -375,7 +420,7 @@ export class RecoveryActuatorService extends Base {
375420
}
376421

377422
/**
378-
* @summary Recycles one allowlisted supervised task through the B0 process supervisor.
423+
* @summary Recycles one known supervised task through the B0 process supervisor.
379424
* @param {Object} options
380425
* @returns {Promise<Object>}
381426
*/
@@ -400,7 +445,7 @@ export class RecoveryActuatorService extends Base {
400445
}
401446

402447
/**
403-
* @summary Restarts one allowlisted compose service through the L0 lifecycle-write envelope.
448+
* @summary Restarts one known compose service through the L0 lifecycle-write envelope.
404449
* @param {Object} options
405450
* @returns {Promise<Object>}
406451
*/

0 commit comments

Comments
 (0)