diff --git a/engine/src/engine/cut_divergence.rs b/engine/src/engine/cut_divergence.rs index 0d80863..c10794e 100644 --- a/engine/src/engine/cut_divergence.rs +++ b/engine/src/engine/cut_divergence.rs @@ -81,7 +81,7 @@ fn deterministic_targets(chains: &[&ProvenChain]) -> BTreeSet { if target.node == chain.entry { continue; } - if quarantine_workload_link(target).is_some() { + if quarantine_workload_link(&target.node, &target.labels).is_some() { targets.insert(target.node.0.clone()); } } diff --git a/engine/src/engine/metrics.rs b/engine/src/engine/metrics.rs index 5a76eac..7a40e00 100644 --- a/engine/src/engine/metrics.rs +++ b/engine/src/engine/metrics.rs @@ -105,6 +105,16 @@ pub(super) struct EngineMetrics { /// cumulative, alert-able counter recorded ONCE per edge (never per pass), so an /// operator can page on "this fired at all" rather than poll the gauge. pub(super) break_glass_transitions: opentelemetry::metrics::Counter, + /// `ProposedAction::ContainNode` (ADR-0040) events, by `event` + /// (`proposed`/`applied`/`reverted`/`rail_refused`) and, for a `rail_refused` event, + /// `reason` (`control-plane`/`one-node-cap`/`worker-floor`/`unlabelled`/`not-owned` — + /// `respond::actuator::node_containment::RailRefusal::metric_reason`). A separate + /// counter from the generic [`Self::mitigations`] one: `ContainNode` is + /// `is_additive_live() == false`, so it never reaches `mitigations`' `applied`/ + /// `reverted` labels through the generic auto-apply path — and its deterministic rails + /// must be observable even in shadow (no `node` arming rung wired yet), so a refusal is + /// counted regardless of whether anything is armed. + pub(super) contain_node: opentelemetry::metrics::Counter, } impl EngineMetrics { @@ -226,6 +236,13 @@ impl EngineMetrics { .u64_counter("protector.engine.break_glass_transitions") .with_description("Break-glass engage/clear transitions, by state.") .build(), + contain_node: m + .u64_counter("protector.engine.contain_node") + .with_description( + "ContainNode (ADR-0040) events by event (proposed/applied/reverted/\ + rail_refused) and, for a refusal, reason.", + ) + .build(), } } @@ -241,6 +258,21 @@ impl EngineMetrics { .add(1, &[opentelemetry::KeyValue::new("state", state)]); } + /// Record one `ContainNode` actuation event (ADR-0040): `event` is one of + /// `proposed`/`applied`/`reverted`/`rail_refused`; `reason` is `Some` only for + /// `rail_refused` — the refusal-reason label + /// (`respond::actuator::node_containment::RailRefusal::metric_reason`) alert rules key + /// on. Fires unconditionally — this counter carries no arming/mode gate of its own, so a + /// `rail_refused` event is exactly as countable in shadow as it would be once a `node` + /// arming rung exists. + pub(super) fn record_contain_node(&self, event: &'static str, reason: Option<&'static str>) { + let mut attrs = vec![opentelemetry::KeyValue::new("event", event)]; + if let Some(reason) = reason { + attrs.push(opentelemetry::KeyValue::new("reason", reason)); + } + self.contain_node.add(1, &attrs); + } + /// Mirror this pass's runtime-corroboration coverage into the OTLP gauges. A pure /// mirror of already-derived state: it takes the SAME [`RuntimeCoverage`] the dashboard reads /// (the caller passes back what `stamp_runtime_coverage` just stored), so the two can never @@ -290,9 +322,31 @@ fn coverage_gauge_values(coverage: &RuntimeCoverage) -> CoverageGaugeValues { mod tests { use std::collections::{BTreeMap, BTreeSet}; - use super::coverage_gauge_values; + use super::{EngineMetrics, coverage_gauge_values}; use crate::engine::state::{LiveNode, derive_runtime_coverage}; + /// `record_contain_node` takes no `EnabledActions`/mode/scope parameter — only the + /// event and an optional reason — so a `rail_refused` event is recordable with nothing + /// armed at all, exactly the "must be observable in shadow" requirement (ADR-0040 §5). + /// A smoke test: constructing the no-op global meter and recording every event this + /// ticket adds (with and without a reason) must not panic. + #[test] + fn record_contain_node_fires_every_event_with_no_armed_state_required() { + let metrics = EngineMetrics::new(); + metrics.record_contain_node("proposed", None); + metrics.record_contain_node("applied", None); + metrics.record_contain_node("reverted", None); + for reason in [ + "control-plane", + "one-node-cap", + "worker-floor", + "unlabelled", + "not-owned", + ] { + metrics.record_contain_node("rail_refused", Some(reason)); + } + } + /// Build a `RuntimeCoverage` from the SAME `derive_runtime_coverage` the dashboard uses, so the /// mirror is tested against the real derivation, not a hand-built stand-in. fn coverage(expected: &[&str], live: &[(&str, LiveNode)]) -> super::RuntimeCoverage { diff --git a/engine/src/engine/mod.rs b/engine/src/engine/mod.rs index 256507d..7c3c079 100644 --- a/engine/src/engine/mod.rs +++ b/engine/src/engine/mod.rs @@ -98,6 +98,7 @@ use observe::adapter::Adapter; use observe::health::{Health, PodStatusHealth}; use respond::Mitigation; use respond::MitigationLedger; +use respond::ProposedAction; use respond::actuator::{ ActionLog, ActuationScope, Actuator, Decision, EnabledActions, decide, predict_blast_radius, }; @@ -656,6 +657,19 @@ impl Engine { .iter() .map(|m| m.cut_signature()) .collect(); + // ADR-0040 actuation metrics: a newly-proposed `ContainNode` mitigation is real, + // genuine data today (the `boundary_break` trigger + menu resolver already run + // unconditionally, ADR-0040 §1-3) — unlike the deterministic rails + // (`respond::actuator::node_containment::cordon_decision`/`revert_decision`), which + // need an observed `NodeFact` fleet the engine does not watch yet (that module's own + // doc), so evaluating them here would mean gating against fabricated "no data" and + // silently reading as always-pass. `applied`/`reverted`/`rail_refused` wire in once + // that observation lands. + for mitigation in &ledger_delta.proposed { + if mitigation.action == ProposedAction::ContainNode { + self.metrics.record_contain_node("proposed", None); + } + } // The break-glass kill switch (ADR-0021's enforcement gate, fast path): checked fresh // every pass, narrowing `self.active` down for THIS pass alone when engaged. See diff --git a/engine/src/engine/reason/adjudicate/incident/menu.rs b/engine/src/engine/reason/adjudicate/incident/menu.rs index 5d04db4..010f4ee 100644 --- a/engine/src/engine/reason/adjudicate/incident/menu.rs +++ b/engine/src/engine/reason/adjudicate/incident/menu.rs @@ -172,8 +172,8 @@ pub fn build_menu( if target.node == chain.entry { continue; } - let fallback = - quarantine_workload_link(target).map(|cut| (cut, ProposedAction::QuarantineWorkload)); + let fallback = quarantine_workload_link(&target.node, &target.labels) + .map(|cut| (cut, ProposedAction::QuarantineWorkload)); match escalate(&target.node, fallback, graph, model_attack) { Some((cut, action)) => { selectable.push(menu_line(target.node.clone(), cut, action, graph, health)); diff --git a/engine/src/engine/respond/actuator/mod.rs b/engine/src/engine/respond/actuator/mod.rs index 02ed6a7..8f42fc9 100644 --- a/engine/src/engine/respond/actuator/mod.rs +++ b/engine/src/engine/respond/actuator/mod.rs @@ -41,6 +41,11 @@ use crate::engine::observe::health::{Health, HealthReport}; use render::workload_namespace; pub mod arming_ladder; +// The ADR-0040 node-containment actuator: the cordon + co-resident default-deny renderers +// and the deterministic rails (control-plane exclusion, one-node cap, worker floor, +// ownership-gated revert). Standalone module — see its own doc for why it is unit-tested +// but not wired into the live per-pass loop by this ticket. +pub mod node_containment; // The read-only pre-arm scope-simulation projection (ADR-0021/ADR-0016): "what fires and // what it severs if `enforceScope` were this scope, right now" — a pure view over the SAME // per-mitigation blast data this module's own `decide`/`predict_blast_radius` compute. diff --git a/engine/src/engine/respond/actuator/node_containment.rs b/engine/src/engine/respond/actuator/node_containment.rs new file mode 100644 index 0000000..016776f --- /dev/null +++ b/engine/src/engine/respond/actuator/node_containment.rs @@ -0,0 +1,228 @@ +//! The [`ProposedAction::ContainNode`] actuator (ADR-0040 §4/§5): the cordon + co-resident +//! default-deny rendering, and the deterministic rails that gate it. Split out of the +//! actuator module root purely to keep every file under the 1,000-line cap (repo CLAUDE.md). +//! +//! **This module is unit-tested, wired nowhere live yet.** `ContainNode` is +//! `is_additive_live() == false` ([`ProposedAction::is_additive_live`]), so +//! [`super::decide`] already routes every `ContainNode` mitigation to +//! [`super::Decision::Forbidden`] regardless of what these rails would say — there is no +//! `node` arming rung to escalate past (ADR-0040 §6, a separate ticket), so nothing here can +//! become live-armable through this module alone. What IS delivered: +//! +//! - [`render_cordon`]/[`render_uncordon`]: the pure `Node.spec.unschedulable` patch, +//! carrying [`CORDON_OWNER_ANNOTATION`] so a revert only ever lifts a cordon protector +//! itself placed (never a human's or the autoscaler's, ADR-0040 §5). +//! - [`co_resident_denies`]: the co-resident default-deny sweep, reusing +//! [`crate::engine::respond::quarantine_workload_link`]'s exact self-reference shape (and +//! therefore [`super::render_isolation`]'s renderer) per co-resident LABELLED workload +//! ([`crate::engine::respond::co_resident_workloads`]) — an unlabeled pod declines exactly +//! like every other quarantine candidate. +//! - [`cordon_decision`]/[`revert_decision`]: the deterministic rails (control-plane +//! exclusion, one-node cap, the two-worker floor, ownership-gated revert), pure over a +//! [`NodeFact`] fleet so they're unit-testable without a live cluster and independent of +//! any arming/enabled state — a rail refusal is exactly as meaningful in shadow as it +//! would be armed. +//! - [`live`]'s [`NodeContainmentActuator`]: the cluster-facing apply/revert glue a future +//! ticket's break-glass/self-revert verification and rung-3 wiring calls into. Thin and +//! untested against a real cluster, like [`super::KubeActuator`]/[`super::IsolationActuator`] +//! — [`render_cordon`]/[`render_uncordon`] are the unit-tested pure half. +//! +//! **Node role/schedulability observation is a follow-up, not this ticket.** [`NodeFact`] +//! is the fleet-state shape the rails need, but nothing in the engine watches Kubernetes +//! `Node` objects today — only `Pod.spec.nodeName`-derived placement (the placement +//! adapter, ADR-0040 §3), which needs no new RBAC. Populating a +//! real `NodeFact` fleet needs a `nodes` `get/list/watch` grant this ticket deliberately +//! does not add (ADR-0040 §7 ships the actuator split from the chart/RBAC change; the +//! ticket that adds this observation is the natural place to also wire these rails into +//! `Engine::process`'s per-pass loop). Evaluating a rail against a fabricated "no data" +//! fleet would silently default it to PASS — exactly the "weakening the rail" the ADR's +//! build-settled note warns against — so this module is deliberately not wired into the +//! live per-pass loop until real fleet data exists. + +use crate::engine::graph::{NodeKey, SecurityGraph}; +use crate::engine::respond::{ + Mitigation, ProposedAction, co_resident_workloads, quarantine_workload_link, +}; + +mod live; +pub use live::NodeContainmentActuator; + +/// The annotation a cordon carries to record that PROTECTOR placed it (ADR-0040 §5). A +/// revert only lifts a cordon carrying this — never a human's or the cluster +/// autoscaler's own cordon — so the engine can never fight another cordon owner. +/// `protector.jeffl.es/*` is the repo's existing annotation namespace (the egress adapter's +/// `EGRESS_ANNOTATION` is the sibling use). +pub const CORDON_OWNER_ANNOTATION: &str = "protector.jeffl.es/cordoned-by"; + +/// The fixed ownership-annotation value protector's own cordons carry. +pub const CORDON_OWNER_VALUE: &str = "protector"; + +/// Render the cordon patch for a [`ProposedAction::ContainNode`] `mitigation` (ADR-0040 +/// §4): `Node.spec.unschedulable = true`, carrying [`CORDON_OWNER_ANNOTATION`]. `None` for +/// any other action — the actuator render path's own convention +/// ([`super::render_deny`]/[`super::render_isolation`] self-guard the same way), so this +/// joins them as the ContainNode line the render allowlist was previously missing. The +/// target host is `mitigation.cut.from.short()` — [`contain_node_link`](crate::engine::respond::contain_node_link) +/// keys a `ContainNode` cut on a `host/` self-reference, and `short()` strips the +/// `host/` kind prefix. Applied via server-side apply under the `protector` field manager +/// ([`live::NodeContainmentActuator`]) — the manifest declares only these two fields, so SSA +/// never contends with any other manager's claim on the rest of the object. +pub fn render_cordon(mitigation: &Mitigation) -> Option { + if mitigation.action != ProposedAction::ContainNode { + return None; + } + let host_name = mitigation.cut.from.short(); + Some(serde_json::json!({ + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": host_name, + "annotations": { CORDON_OWNER_ANNOTATION: CORDON_OWNER_VALUE } + }, + "spec": { "unschedulable": true } + })) +} + +/// Render the uncordon patch for a [`ProposedAction::ContainNode`] `mitigation`: +/// `Node.spec.unschedulable = false`, with the ownership annotation OMITTED — under the +/// SAME `protector` field manager [`render_cordon`] applies through, omitting a previously- +/// declared field releases it, so re-applying this removes the annotation rather than +/// leaving a stale "protector cordoned this" marker on a node that is no longer cordoned. +/// `None` for any other action, mirroring [`render_cordon`]. +pub fn render_uncordon(mitigation: &Mitigation) -> Option { + if mitigation.action != ProposedAction::ContainNode { + return None; + } + let host_name = mitigation.cut.from.short(); + Some(serde_json::json!({ + "apiVersion": "v1", + "kind": "Node", + "metadata": { "name": host_name }, + "spec": { "unschedulable": false } + })) +} + +/// The co-resident default-deny sweep for a `ContainNode` mitigation on `host` (ADR-0040 +/// §4): one [`ProposedAction::QuarantineWorkload`] mitigation per co-resident LABELLED +/// workload, built through the exact SAME [`quarantine_workload_link`] self-reference shape +/// (and therefore [`super::render_isolation`]'s renderer) the chain-based workload +/// quarantine already uses, so the two paths can never diverge on how a pod-scoped deny is +/// rendered. `justifications` is empty on every returned mitigation: these are +/// node-containment-triggered, not chain-justified in the ledger's own +/// [`crate::engine::respond::MitigationLedger`] sense. +pub fn co_resident_denies(graph: &SecurityGraph, host: &NodeKey) -> Vec { + co_resident_workloads(graph, host) + .into_iter() + .filter_map(|(node, labels)| quarantine_workload_link(&node, &labels)) + .map(|cut| Mitigation { + cut, + action: ProposedAction::QuarantineWorkload, + justifications: Vec::new(), + }) + .collect() +} + +/// A per-pass fact for one node in the fleet — the shape [`cordon_decision`]/ +/// [`revert_decision`] need, sourced from an observed Kubernetes `Node` (name; the +/// `node-role.kubernetes.io/control-plane` label; `spec.unschedulable`; whether +/// [`CORDON_OWNER_ANNOTATION`] is set to [`CORDON_OWNER_VALUE`]) once that observation is +/// wired (see this module's doc — a follow-up). Deliberately a plain data type, not the +/// graph's [`crate::engine::graph::Node::Host`], so the rail predicates stay pure and +/// unit-testable over hand-built fixtures without a full [`SecurityGraph`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NodeFact { + pub name: String, + /// Carries a control-plane role label — VISION's "protector cannot touch the control + /// plane" (ADR-0040 §5). + pub control_plane: bool, + /// `!Node.spec.unschedulable` — true unless something (protector, a human, the + /// autoscaler) has already cordoned it. + pub schedulable: bool, + /// [`CORDON_OWNER_ANNOTATION`] is set to [`CORDON_OWNER_VALUE`] on this node right now. + pub owned_by_protector: bool, +} + +/// Why a deterministic node-containment rail refused (ADR-0040 §5) — the fixed vocabulary +/// [`Self::metric_reason`] labels the `rail_refused` metric with +/// ([`crate::engine::metrics::EngineMetrics::record_contain_node`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RailRefusal { + /// The target node carries a control-plane role — never cordoned (VISION). + ControlPlane, + /// Some OTHER node is already cordoned by protector — at most one at a time. + OneNodeCap, + /// Cordoning the target would leave fewer than two schedulable workers. + WorkerFloor, + /// A co-resident pod carries no labels — declined rather than widened to a namespace. + Unlabelled, + /// The target node does not carry protector's own cordon-ownership annotation — never + /// revert a cordon protector didn't place. + NotOwned, +} + +impl RailRefusal { + /// The metrics `reason` label this refusal carries — the fixed vocabulary the + /// actuation-metrics ticket requirement specifies. + pub fn metric_reason(&self) -> &'static str { + match self { + Self::ControlPlane => "control-plane", + Self::OneNodeCap => "one-node-cap", + Self::WorkerFloor => "worker-floor", + Self::Unlabelled => "unlabelled", + Self::NotOwned => "not-owned", + } + } +} + +/// The minimum number of schedulable, non-control-plane workers a cordon must leave behind +/// (ADR-0040 §5, build-settled 2026-08-02: "a floor that leaves a single worker is an +/// outage, not damage-limitation" — kept at 2 even on a small fleet where this can make +/// `ContainNode` correctly, permanently inert). +const WORKER_FLOOR: usize = 2; + +/// Whether cordoning `target` is deterministically allowed, over the CURRENT `fleet` +/// (ADR-0040 §5's three cordon rails, checked in the order a human reviewing a refusal would +/// expect: is this even a candidate node, is protector already committed elsewhere, would +/// this cordon itself cause an outage). `fleet` must include `target`'s own current entry — +/// the worker-floor count is `fleet` minus `target`, not a separately-supplied total, so the +/// two can never drift apart. +/// +/// Pure and independent of any [`super::EnabledActions`]/arming state by construction — the +/// rail is exactly as meaningful evaluated in shadow (nothing armed) as it would be once a +/// `node` rung exists, which is how a rail refusal can be counted regardless of mode +/// (this module's doc, and the actuation-metrics ticket requirement). +pub fn cordon_decision(target: &NodeFact, fleet: &[NodeFact]) -> Result<(), RailRefusal> { + if target.control_plane { + return Err(RailRefusal::ControlPlane); + } + let already_cordoned_elsewhere = fleet + .iter() + .any(|n| n.name != target.name && n.owned_by_protector && !n.schedulable); + if already_cordoned_elsewhere { + return Err(RailRefusal::OneNodeCap); + } + let workers_after = fleet + .iter() + .filter(|n| n.name != target.name && !n.control_plane && n.schedulable) + .count(); + if workers_after < WORKER_FLOOR { + return Err(RailRefusal::WorkerFloor); + } + Ok(()) +} + +/// Whether reverting (uncordoning) `target` is allowed: ownership-gated, and ONLY +/// ownership-gated (ADR-0040 §5) — a node protector never cordoned, or one a human/the +/// autoscaler has since re-cordoned over protector's own lifted control, must never be +/// touched. Unlike [`cordon_decision`] there is no control-plane/floor check here: lifting a +/// cordon can never cause the damage those rails guard against. +pub fn revert_decision(target: &NodeFact) -> Result<(), RailRefusal> { + if target.owned_by_protector { + Ok(()) + } else { + Err(RailRefusal::NotOwned) + } +} + +#[cfg(test)] +mod tests; diff --git a/engine/src/engine/respond/actuator/node_containment/live.rs b/engine/src/engine/respond/actuator/node_containment/live.rs new file mode 100644 index 0000000..79aa4f0 --- /dev/null +++ b/engine/src/engine/respond/actuator/node_containment/live.rs @@ -0,0 +1,128 @@ +//! The cluster-facing glue for `ProposedAction::ContainNode`: cordon/uncordon the target +//! `Node`, and drive the co-resident default-deny sweep through the SAME +//! [`IsolationActuator`] the chain-based workload quarantine already uses. Thin and +//! exercised only against a real cluster — like the `kube` module's live actuators — with +//! [`super::render_cordon`]/[`super::render_uncordon`] as the unit-tested pure half. +//! +//! **Not wired into anything live in this ticket.** No `node` arming rung exists yet +//! (ADR-0040 §6, a separate ticket) and `ContainNode`'s `is_additive_live() == false` means +//! [`super::super::decide`] never routes here through the generic auto-apply path either — +//! this exists so a future human-approval/break-glass-revert path has a real apply/revert to +//! call, callable and testable in isolation today. + +use crate::engine::respond::Mitigation; +use crate::engine::respond::actuator::{Actuation, Actuator, IsolationActuator, cut_label}; + +use super::{NodeFact, render_cordon, render_uncordon, revert_decision}; + +/// A dynamic `Api` for the cluster-scoped core `Node` resource. +fn node_api(client: &kube::Client) -> kube::Api { + let gvk = kube::core::GroupVersionKind::gvk("", "v1", "Node"); + let ar = kube::core::ApiResource::from_gvk(&gvk); + kube::Api::all_with(client.clone(), &ar) +} + +/// Applies/reverts a `ProposedAction::ContainNode` mitigation: the cordon patch on the +/// target `Host`, plus the co-resident default-deny sweep ([`super::co_resident_denies`]) +/// via the shared [`IsolationActuator`] path. Unlike every other [`Actuator`] +/// implementation, this one action maps to MANY cluster objects (one cordon, one +/// `NetworkPolicy` per co-resident labelled pod), so it deliberately does not implement the +/// single-mitigation [`Actuator`] trait — its methods take the co-resident set explicitly +/// instead. +pub struct NodeContainmentActuator { + client: kube::Client, +} + +impl NodeContainmentActuator { + pub fn new(client: kube::Client) -> Self { + Self { client } + } + + /// Cordon `mitigation`'s target host and default-deny every co-resident mitigation in + /// `co_resident` (built by [`super::co_resident_denies`]). Best-effort: a co-resident + /// deny failure is logged by [`IsolationActuator`] and does not roll back the cordon — a + /// partial containment (node cordoned, some pods still reachable) is safer than none. + pub async fn apply(&self, mitigation: &Mitigation, co_resident: &[Mitigation]) -> Actuation { + let Some(manifest) = render_cordon(mitigation) else { + tracing::warn!(cut = %cut_label(mitigation), "not a ContainNode mitigation; nothing to cordon"); + return Actuation::DryRun; + }; + let host_name = mitigation.cut.from.short(); + if !self.patch_node(host_name, &manifest, "cordon").await { + return Actuation::DryRun; + } + tracing::info!(node = %host_name, "cordoned node (ADR-0040 containment)"); + let isolation = IsolationActuator::new(self.client.clone()); + for co_resident_mitigation in co_resident { + isolation.apply(co_resident_mitigation).await; + } + Actuation::Applied + } + + /// Uncordon `mitigation`'s target host and lift every co-resident deny in + /// `co_resident`. Self-gated on [`super::revert_decision`]: `target` is the observed + /// [`NodeFact`] for the host, and this method short-circuits to [`Actuation::DryRun`] + /// unless protector owns the cordon — the ownership rail cannot be bypassed by a + /// forgetful caller, so the highest-blast action is safe by construction rather than by + /// caller discipline. The break-glass/self-revert path built on this (ADR-0040 §6) calls + /// exactly this method, so the gate lives here, not only in a doc-comment contract. + pub async fn revert( + &self, + mitigation: &Mitigation, + target: &NodeFact, + co_resident: &[Mitigation], + ) -> Actuation { + if let Err(refusal) = revert_decision(target) { + tracing::warn!( + node = %target.name, + reason = refusal.metric_reason(), + "revert refused by ownership rail; not uncordoning", + ); + return Actuation::DryRun; + } + let Some(manifest) = render_uncordon(mitigation) else { + tracing::warn!(cut = %cut_label(mitigation), "not a ContainNode mitigation; nothing to uncordon"); + return Actuation::DryRun; + }; + let host_name = mitigation.cut.from.short(); + if !self.patch_node(host_name, &manifest, "uncordon").await { + return Actuation::DryRun; + } + tracing::info!(node = %host_name, "uncordoned node (ADR-0040 revert)"); + let isolation = IsolationActuator::new(self.client.clone()); + for co_resident_mitigation in co_resident { + isolation.revert(co_resident_mitigation).await; + } + Actuation::Reverted + } + + /// Server-side-apply `manifest` (a [`super::render_cordon`]/[`super::render_uncordon`] + /// patch for `host_name`) against the target `Node`, under the `protector` field + /// manager so cordon and uncordon never contend with any other field the object + /// carries. Returns whether the patch succeeded. + async fn patch_node( + &self, + host_name: &str, + manifest: &serde_json::Value, + verb: &'static str, + ) -> bool { + let object: kube::core::DynamicObject = match serde_json::from_value(manifest.clone()) { + Ok(o) => o, + Err(error) => { + tracing::error!(%error, verb, "failed to build Node patch"); + return false; + } + }; + let params = kube::api::PatchParams::apply("protector").force(); + match node_api(&self.client) + .patch(host_name, ¶ms, &kube::api::Patch::Apply(&object)) + .await + { + Ok(_) => true, + Err(error) => { + tracing::error!(%error, node = %host_name, verb, "failed to patch node"); + false + } + } + } +} diff --git a/engine/src/engine/respond/actuator/node_containment/tests.rs b/engine/src/engine/respond/actuator/node_containment/tests.rs new file mode 100644 index 0000000..a856cf8 --- /dev/null +++ b/engine/src/engine/respond/actuator/node_containment/tests.rs @@ -0,0 +1,292 @@ +//! Unit tests for the ADR-0040 node-containment actuator: the cordon/uncordon renderers, +//! the co-resident default-deny sweep, and each deterministic rail — plus the standing +//! invariant that `ContainNode` still never auto-applies (no `node` rung wired). + +use super::*; +use crate::engine::observe::Snapshot; +use crate::engine::observe::adapter::{build_graph, default_adapters}; +use crate::engine::reason::proof::Link; +use serde_json::json; + +/// A `ContainNode` mitigation self-referencing `host/` — the exact shape +/// [`crate::engine::respond::contain_node_link`] builds. +fn contain_node_mitigation(host_name: &str) -> Mitigation { + let host = NodeKey(format!("host/{host_name}")); + Mitigation { + cut: Link { + from: host.clone(), + to: host, + relation: "contain-node".to_string(), + technique: None, + from_labels: Default::default(), + to_labels: Default::default(), + }, + action: ProposedAction::ContainNode, + justifications: vec![], + } +} + +fn scheduled_pod( + name: &str, + node_name: &str, + labels: serde_json::Value, +) -> k8s_openapi::api::core::v1::Pod { + serde_json::from_value(json!({ + "apiVersion": "v1", "kind": "Pod", + "metadata": {"name": name, "namespace": "app", "labels": labels}, + "spec": { + "nodeName": node_name, + "containers": [{"name": name, "image": format!("{name}:1")}] + } + })) + .expect("valid Pod fixture") +} + +fn fact(name: &str, control_plane: bool, schedulable: bool, owned: bool) -> NodeFact { + NodeFact { + name: name.to_string(), + control_plane, + schedulable, + owned_by_protector: owned, + } +} + +// --- render_cordon / render_uncordon --- + +#[test] +fn render_cordon_sets_unschedulable_and_the_ownership_annotation() { + let manifest = render_cordon(&contain_node_mitigation("node-1")).expect("ContainNode renders"); + assert_eq!(manifest["kind"], "Node"); + assert_eq!(manifest["metadata"]["name"], "node-1"); + assert_eq!(manifest["spec"]["unschedulable"], true); + assert_eq!( + manifest["metadata"]["annotations"][CORDON_OWNER_ANNOTATION], + CORDON_OWNER_VALUE + ); +} + +#[test] +fn render_uncordon_clears_unschedulable_and_omits_the_ownership_annotation() { + let manifest = + render_uncordon(&contain_node_mitigation("node-1")).expect("ContainNode renders"); + assert_eq!(manifest["spec"]["unschedulable"], false); + // Omitted, not null: under server-side apply this is how the SAME field manager + // that set the annotation RELEASES it on re-apply (see the module doc). + assert!(manifest["metadata"].get("annotations").is_none()); +} + +#[test] +fn render_cordon_and_uncordon_are_none_for_a_non_contain_node_action() { + let other = Mitigation { + cut: Link { + from: NodeKey("workload/app/Pod/web".into()), + to: NodeKey("workload/app/Pod/web".into()), + relation: "quarantine-workload".to_string(), + technique: None, + from_labels: [("app".to_string(), "web".to_string())].into(), + to_labels: [("app".to_string(), "web".to_string())].into(), + }, + action: ProposedAction::QuarantineWorkload, + justifications: vec![], + }; + assert!(render_cordon(&other).is_none()); + assert!(render_uncordon(&other).is_none()); +} + +// --- co_resident_denies --- + +#[test] +fn co_resident_denies_covers_every_labelled_pod_on_the_host() { + let snap = Snapshot { + pods: vec![ + scheduled_pod("victim", "node-1", json!({"app": "victim"})), + scheduled_pod("neighbor", "node-1", json!({"app": "neighbor"})), + scheduled_pod("elsewhere", "node-2", json!({"app": "elsewhere"})), + ], + ..Default::default() + }; + let graph = build_graph(&snap, &default_adapters()); + let host = NodeKey("host/node-1".into()); + + let denies = co_resident_denies(&graph, &host); + assert_eq!(denies.len(), 2, "only node-1's two co-resident pods"); + for m in &denies { + assert_eq!(m.action, ProposedAction::QuarantineWorkload); + assert!(m.justifications.is_empty()); + } + let targets: std::collections::BTreeSet<_> = + denies.iter().map(|m| m.cut.from.0.clone()).collect(); + assert!(targets.contains("workload/app/Pod/victim")); + assert!(targets.contains("workload/app/Pod/neighbor")); +} + +#[test] +fn co_resident_denies_declines_an_unlabelled_pod() { + let snap = Snapshot { + pods: vec![scheduled_pod("bare", "node-1", json!({}))], + ..Default::default() + }; + let graph = build_graph(&snap, &default_adapters()); + let host = NodeKey("host/node-1".into()); + + assert!( + co_resident_denies(&graph, &host).is_empty(), + "an unlabelled co-resident pod is declined, never widened to a namespace" + ); +} + +#[test] +fn co_resident_denies_is_empty_for_a_host_absent_from_the_graph() { + let graph = crate::engine::graph::SecurityGraph::new(); + let host = NodeKey("host/nonexistent".into()); + assert!(co_resident_denies(&graph, &host).is_empty()); +} + +// --- RailRefusal metric labels (the fixed vocabulary the metric reason keys on) --- + +#[test] +fn every_rail_refusal_has_its_specified_metric_reason() { + assert_eq!(RailRefusal::ControlPlane.metric_reason(), "control-plane"); + assert_eq!(RailRefusal::OneNodeCap.metric_reason(), "one-node-cap"); + assert_eq!(RailRefusal::WorkerFloor.metric_reason(), "worker-floor"); + assert_eq!(RailRefusal::Unlabelled.metric_reason(), "unlabelled"); + assert_eq!(RailRefusal::NotOwned.metric_reason(), "not-owned"); +} + +// --- cordon_decision rails --- + +#[test] +fn cordon_decision_allows_a_worker_with_a_healthy_fleet() { + let target = fact("node-1", false, true, false); + let fleet = vec![ + target.clone(), + fact("node-2", false, true, false), + fact("node-3", false, true, false), + fact("node-4", false, true, false), + fact("cp-1", true, true, false), + ]; + assert_eq!(cordon_decision(&target, &fleet), Ok(())); +} + +#[test] +fn cordon_decision_never_cordons_a_control_plane_node() { + let target = fact("cp-1", true, true, false); + let fleet = vec![ + target.clone(), + fact("node-1", false, true, false), + fact("node-2", false, true, false), + fact("node-3", false, true, false), + ]; + assert_eq!( + cordon_decision(&target, &fleet), + Err(RailRefusal::ControlPlane) + ); +} + +#[test] +fn cordon_decision_refuses_a_second_concurrent_cordon() { + let target = fact("node-2", false, true, false); + let fleet = vec![ + target.clone(), + // node-1 is ALREADY cordoned by protector (unschedulable + owned). + fact("node-1", false, false, true), + fact("node-3", false, true, false), + fact("node-4", false, true, false), + ]; + assert_eq!( + cordon_decision(&target, &fleet), + Err(RailRefusal::OneNodeCap) + ); +} + +#[test] +fn cordon_decision_ignores_a_node_cordoned_by_someone_other_than_protector() { + // node-1 is unschedulable but NOT owned by protector (a human/autoscaler cordon) — + // the one-node cap only tracks protector's OWN standing cordon. + let target = fact("node-2", false, true, false); + let fleet = vec![ + target.clone(), + fact("node-1", false, false, false), + fact("node-3", false, true, false), + fact("node-4", false, true, false), + ]; + assert_eq!(cordon_decision(&target, &fleet), Ok(())); +} + +#[test] +fn cordon_decision_refuses_when_it_would_leave_fewer_than_two_schedulable_workers() { + // Only two workers total (target + one other) — cordoning target leaves one. + let target = fact("node-1", false, true, false); + let fleet = vec![ + target.clone(), + fact("node-2", false, true, false), + fact("cp-1", true, true, false), + ]; + assert_eq!( + cordon_decision(&target, &fleet), + Err(RailRefusal::WorkerFloor) + ); +} + +#[test] +fn cordon_decision_worker_floor_excludes_already_unschedulable_and_control_plane_nodes() { + // Three other entries, but only one is a schedulable, non-control-plane worker — + // the floor must count REAL headroom, not raw fleet size. + let target = fact("node-1", false, true, false); + let fleet = vec![ + target.clone(), + fact("node-2", false, true, false), // the one real worker left + fact("node-3", false, false, false), // already unschedulable — doesn't count + fact("cp-1", true, true, false), // control-plane — doesn't count + ]; + assert_eq!( + cordon_decision(&target, &fleet), + Err(RailRefusal::WorkerFloor) + ); +} + +#[test] +fn cordon_decision_at_exactly_the_floor_is_allowed() { + // Cordoning target leaves EXACTLY two schedulable workers — the floor is inclusive. + let target = fact("node-1", false, true, false); + let fleet = vec![ + target.clone(), + fact("node-2", false, true, false), + fact("node-3", false, true, false), + fact("cp-1", true, true, false), + ]; + assert_eq!(cordon_decision(&target, &fleet), Ok(())); +} + +// --- revert_decision: ownership-gated --- + +#[test] +fn revert_decision_allows_uncordoning_a_node_protector_owns() { + let target = fact("node-1", false, false, true); + assert_eq!(revert_decision(&target), Ok(())); +} + +#[test] +fn revert_decision_refuses_a_node_protector_never_cordoned() { + let target = fact("node-1", false, false, false); + assert_eq!(revert_decision(&target), Err(RailRefusal::NotOwned)); +} + +// --- rails are pure over the fleet, independent of any arming/enabled state --- + +#[test] +fn rail_decisions_take_no_arming_state_and_so_are_exactly_as_meaningful_in_shadow() { + // `cordon_decision`/`revert_decision` accept only `NodeFact`/fleet data — no + // `EnabledActions`, no `ActuationScope`, no mode. A refusal computed here is + // identically valid whether or not anything is armed, which is what lets a + // rail-refused event be counted "in shadow" (nothing armed) exactly as it would be + // once a `node` rung exists. + let control_plane = fact("cp-1", true, true, false); + let fleet = vec![control_plane.clone()]; + assert_eq!( + cordon_decision(&control_plane, &fleet), + Err(RailRefusal::ControlPlane) + ); + let unowned = fact("node-1", false, false, false); + assert_eq!(revert_decision(&unowned), Err(RailRefusal::NotOwned)); +} diff --git a/engine/src/engine/respond/actuator/tests.rs b/engine/src/engine/respond/actuator/tests.rs index 5e2dc20..df2e81b 100644 --- a/engine/src/engine/respond/actuator/tests.rs +++ b/engine/src/engine/respond/actuator/tests.rs @@ -271,6 +271,33 @@ fn decide_forbids_subtractive_rbac() { )); } +/// ADR-0040 §5/§6: `ContainNode` must never auto-apply through the generic `decide()` path +/// — there is no `node` arming rung to escalate past yet, and `is_additive_live() == false` +/// means [`decide`] forbids it structurally, before even reaching the enabled/scope checks. +/// Enabling it here (something no real config can do today — there is no operator-facing +/// `node` class name, `actuator::actions_from_name` never maps to it) proves the forbid is +/// unconditional, not just "nobody happens to enable it". +#[test] +fn decide_forbids_contain_node_even_when_the_class_would_be_enabled() { + let m = mitigation( + "host/node-1", + "contain-node", + "host/node-1", + ProposedAction::ContainNode, + ); + assert!(!EnabledActions::none().is_enabled(ProposedAction::ContainNode)); + let enabled = EnabledActions::none().enable(ProposedAction::ContainNode); + assert!(matches!( + decide( + &m, + &enabled, + &ActuationScope::unscoped(), + &BlastRadius::default() + ), + Decision::Forbidden(_) + )); +} + #[test] fn decide_network_needs_corroboration_and_active_to_auto_apply() { let net = || Mitigation { diff --git a/engine/src/engine/respond/contain_node_tests.rs b/engine/src/engine/respond/contain_node_tests.rs index 3d7937b..71a1ae6 100644 --- a/engine/src/engine/respond/contain_node_tests.rs +++ b/engine/src/engine/respond/contain_node_tests.rs @@ -140,3 +140,41 @@ fn self_severance_is_false_for_a_host_absent_from_the_graph() { let graph = crate::engine::graph::SecurityGraph::new(); assert!(!self_severance(&graph, &NodeKey("host/nonexistent".into()))); } + +#[test] +fn co_resident_workloads_returns_every_workload_scheduled_on_the_host() { + let snap = Snapshot { + pods: vec![ + scheduled_pod("victim", "node-1", json!({"app": "victim"})), + scheduled_pod("neighbor", "node-1", json!({})), + scheduled_pod("elsewhere", "node-2", json!({"app": "elsewhere"})), + ], + ..Default::default() + }; + let graph = build_graph(&snap, &default_adapters()); + let host = NodeKey("host/node-1".into()); + + let mut co_resident = co_resident_workloads(&graph, &host); + co_resident.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + co_resident, + vec![ + ( + NodeKey("workload/app/Pod/neighbor".into()), + std::collections::BTreeMap::new() + ), + ( + NodeKey("workload/app/Pod/victim".into()), + [("app".to_string(), "victim".to_string())].into() + ), + ], + "node-2's pod is excluded; the unlabeled neighbor is still RETURNED (declining it \ + is quarantine_workload_link's job, not this walk's)" + ); +} + +#[test] +fn co_resident_workloads_is_empty_for_a_host_absent_from_the_graph() { + let graph = crate::engine::graph::SecurityGraph::new(); + assert!(co_resident_workloads(&graph, &NodeKey("host/nonexistent".into())).is_empty()); +} diff --git a/engine/src/engine/respond/mod.rs b/engine/src/engine/respond/mod.rs index 72b0601..43e9a11 100644 --- a/engine/src/engine/respond/mod.rs +++ b/engine/src/engine/respond/mod.rs @@ -23,7 +23,7 @@ use petgraph::visit::EdgeRef; use crate::engine::graph::attack::AttackRef; use crate::engine::graph::{Node, NodeKey, Relation, SecurityGraph}; -use crate::engine::reason::proof::{Link, ProvenChain, QuarantineTarget}; +use crate::engine::reason::proof::{Link, ProvenChain}; /// How a cut edge would be severed by an additive, engine-owned object (ADR-0002). /// Descriptive here — the Actuator port renders these into concrete objects in @@ -301,17 +301,30 @@ pub(crate) fn quarantine_link(chain: &ProvenChain) -> Option { /// `None`-on-unlabeled decline the ledger's `reconcile` uses — so the menu and the ledger /// can never disagree on which downstream nodes are containable. `reconcile` itself is /// untouched — only this pure builder's visibility widens. -pub(crate) fn quarantine_workload_link(target: &QuarantineTarget) -> Option { - if target.labels.is_empty() { +/// +/// Takes the bare `(node, labels)` pair rather than a whole +/// [`QuarantineTarget`](crate::engine::reason::proof::QuarantineTarget) — the only +/// two fields this builder ever reads — so a caller with a workload key + labels but no +/// chain-derived [`QuarantineReason`] can reuse the exact same self-reference shape. The +/// ADR-0040 co-resident default-deny sweep ([`crate::engine::respond::actuator::node_containment::co_resident_denies`]) +/// is that second caller: a pod sharing a contained node's host has no `QuarantineReason` of +/// its own (it isn't necessarily itself remotely-exploitable/actively-exploited), only its +/// co-residency, so forcing a reason here would be a fabricated tag on a field nothing reads +/// for this path. +pub(crate) fn quarantine_workload_link( + node: &NodeKey, + labels: &BTreeMap, +) -> Option { + if labels.is_empty() { return None; } Some(Link { - from: target.node.clone(), - to: target.node.clone(), + from: node.clone(), + to: node.clone(), relation: QUARANTINE_WORKLOAD_RELATION.to_string(), technique: None, - from_labels: target.labels.clone(), - to_labels: target.labels.clone(), + from_labels: labels.clone(), + to_labels: labels.clone(), }) } @@ -370,12 +383,53 @@ pub(crate) fn self_severance(graph: &SecurityGraph, host: &NodeKey) -> bool { let Some(host_idx) = graph.index_of(host) else { return false; }; + scheduled_on_host(graph, host_idx).any(|(_, node)| is_protector_component(node)) +} + +/// The workload nodes with a `Relation::ScheduledOn` edge into `host_idx` — the shared +/// walk [`self_severance`] and [`co_resident_workloads`] both need, factored out once so +/// the two can never quietly diverge on what "scheduled on this host" means. +fn scheduled_on_host( + graph: &SecurityGraph, + host_idx: petgraph::stable_graph::NodeIndex, +) -> impl Iterator { graph .inner() .edges_directed(host_idx, petgraph::Direction::Incoming) .filter(|e| matches!(e.weight().relation, Relation::ScheduledOn)) - .filter_map(|e| graph.node(e.source())) - .any(is_protector_component) + .filter_map(move |e| graph.node(e.source()).map(|node| (e.source(), node))) +} + +/// Every workload scheduled on `host` — the co-resident set a +/// [`ProposedAction::ContainNode`] mitigation's default-deny sweep covers (ADR-0040 §4). +/// Walks the SAME `Relation::ScheduledOn` placement edge [`contain_node_link`] keys the +/// mitigation on, in the opposite (incoming) direction. Deliberately includes the +/// workload(s) whose `boundary_break` triggered the escalation: denying its own pod-scoped +/// network too is strictly additional containment, not a correctness gap, and excluding it +/// would need `ContainNode`'s cut to carry the triggering workload key, which the host-only +/// self-reference deliberately does not ([`contain_node_link`]'s doc). Returns `(node, +/// labels)` pairs sorted by key; an unlabeled pod is still returned here — declining it is +/// [`quarantine_workload_link`]'s job (the same decline every other quarantine candidate +/// gets), not this walk's. +/// +/// `pub(crate)`: the actuator's node-containment renderer +/// ([`crate::engine::respond::actuator::node_containment::co_resident_denies`]) is the +/// consumer — a sibling module under `actuator`, not `respond` itself. +pub(crate) fn co_resident_workloads( + graph: &SecurityGraph, + host: &NodeKey, +) -> Vec<(NodeKey, BTreeMap)> { + let Some(host_idx) = graph.index_of(host) else { + return Vec::new(); + }; + let mut out: Vec<(NodeKey, BTreeMap)> = scheduled_on_host(graph, host_idx) + .filter_map(|(idx, node)| match node { + Node::Workload(w) => graph.key_of(idx).map(|k| (k, w.labels.clone())), + _ => None, + }) + .collect(); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out } /// Label-based identification of protector's own chart-rendered workloads: the agent @@ -671,7 +725,7 @@ impl MitigationLedger { if target.node == chain.entry && entry_additively_contained { continue; } - let Some(cut) = quarantine_workload_link(target) else { + let Some(cut) = quarantine_workload_link(&target.node, &target.labels) else { continue; // no labels — decline rather than widen to a namespace }; desired