Skip to content

Commit 1aa74af

Browse files
committed
Fix evaluator host qualification across reboots
1 parent 3ce8fe7 commit 1aa74af

13 files changed

Lines changed: 277 additions & 36 deletions

competition/EVALUATOR-PROTOCOL.md

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -163,9 +163,15 @@ their resource-boundary dependency, exact device-IRQ placement, and two
163163
root-owned systemd units. The first runs before Docker and fails unless SMT is
164164
off, exactly 12 physical CPUs are online, every online CPU uses the performance
165165
governor, turbo is off, Redis-safe overcommit is enabled, and the live topology
166-
derives the frozen 10-evaluation/2-management split. The dependent IRQ unit
167-
then fails unless every discovered movable device IRQ is routed exactly to the
168-
management set. A reboot must prove both units reapply before Docker. Firewall
166+
derives the frozen 10-evaluation/2-management split. Early boot may expose a
167+
transient numeric SMT state; the control normalizes that state and retries the
168+
`off` transition before failing closed. The dependent IRQ unit then fails
169+
unless every discovered movable device IRQ is routed exactly to the management
170+
set. The qualification binds a stable digest of the minimum IRQ and CPU-set
171+
policy, while the live check separately verifies every currently numbered IRQ
172+
and retains the number-sensitive affinity digest only as diagnostics. Both
173+
units are required Docker dependencies. A reboot must prove both units reapply
174+
before Docker. Firewall
169175
qualification remains separate. Docker namespace qualification uses a live
170176
container `/proc/*/{uid,gid}_map` probe and rejects an identity mapping even if
171177
daemon configuration merely claims user-namespace support.

competition/README.md

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,14 @@ identity until the pending main-branch fixes are merged and pushed.
8181
exact-device-IRQ executable, and both root-owned systemd units. The first
8282
fail-closed unit requires exactly 12 online physical CPUs with SMT off, the
8383
`performance` governor, turbo off, `vm.overcommit_memory=1`, and the derived
84-
10+2 split. The dependent unit routes every movable device IRQ to the two
85-
management CPUs and verifies the resulting affinity. Install a reviewed
84+
10+2 split. It normalizes the kernel's transient numeric SMT state and
85+
retries the `off` transition during early boot. The dependent unit routes
86+
every movable device IRQ to the two management CPUs and verifies the
87+
resulting affinity. Both units are required dependencies of Docker, so a
88+
failed host control prevents the daemon from admitting evaluations. The
89+
qualification digest binds the stable IRQ policy while every heartbeat
90+
revalidates all currently discovered IRQs; transient IRQ numbers are kept
91+
only as diagnostic evidence. Install a reviewed
8692
`docker-daemon.example.json` as `/etc/docker/daemon.json` during a controlled
8793
restart; the host check authenticates its bytes, hardening semantics, and a
8894
live non-identity container UID/GID map. The firewall remains a separate
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
3+
# Shared, side-effect-free helpers for the authoritative host-control command.
4+
# This file is sourced by the root-owned installer target and by its regression
5+
# test; it is not an executable entry point.
6+
7+
urnetwork_smt_state() {
8+
local control_path="$1"
9+
tr -d '\n' <"$control_path" 2>/dev/null || true
10+
}
11+
12+
urnetwork_smt_write() {
13+
local control_path="$1" value="$2"
14+
if [ -w "$control_path" ]; then
15+
printf '%s\n' "$value" >"$control_path"
16+
else
17+
printf '%s\n' "$value" | sudo -n tee "$control_path" >/dev/null
18+
fi
19+
}
20+
21+
urnetwork_disable_smt() {
22+
local control_path="$1" attempts="${2:-10}" retry_delay="${3:-1}"
23+
local attempt state
24+
25+
[[ "$attempts" =~ ^[1-9][0-9]*$ ]] || return 2
26+
[ -e "$control_path" ] || return 1
27+
for ((attempt = 1; attempt <= attempts; attempt++)); do
28+
state="$(urnetwork_smt_state "$control_path")"
29+
case "$state" in
30+
off|forceoff|notsupported)
31+
return 0
32+
;;
33+
on)
34+
;;
35+
*)
36+
# Some kernels expose a transient numeric state while CPU
37+
# hotplug is still settling. Normalizing it to `on` makes the
38+
# subsequent `off` transition deterministic.
39+
urnetwork_smt_write "$control_path" on >/dev/null 2>&1 || true
40+
;;
41+
esac
42+
if urnetwork_smt_write "$control_path" off >/dev/null 2>&1; then
43+
state="$(urnetwork_smt_state "$control_path")"
44+
case "$state" in
45+
off|forceoff|notsupported)
46+
return 0
47+
;;
48+
esac
49+
fi
50+
if [ "$attempt" -lt "$attempts" ] && [ "$retry_delay" != 0 ]; then
51+
sleep "$retry_delay"
52+
fi
53+
done
54+
return 1
55+
}

competition/authoritative-host-controls.service.example

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@ ConditionFileIsExecutable=/usr/local/libexec/urnetwork/authoritative-host-contro
88
Type=oneshot
99
ExecStart=/usr/local/libexec/urnetwork/authoritative-host-controls --apply
1010
RemainAfterExit=yes
11+
Restart=on-failure
12+
RestartSec=2
1113
TimeoutStartSec=60
1214

1315
[Install]
1416
WantedBy=multi-user.target
17+
RequiredBy=containerd.service docker.service

competition/authoritative-host-controls.sh

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,15 @@ umask 077
1010

1111
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
1212
readonly RESOURCE_BOUNDARY="$SCRIPT_DIR/container/resource-boundary.sh"
13+
readonly CONTROL_LIBRARY="$SCRIPT_DIR/authoritative-host-controls-lib.sh"
14+
readonly SMT_CONTROL=/sys/devices/system/cpu/smt/control
15+
16+
[ -r "$CONTROL_LIBRARY" ] || {
17+
printf '[competition-host-controls] ERROR: control library is unavailable\n' >&2
18+
exit 1
19+
}
20+
# shellcheck source=authoritative-host-controls-lib.sh
21+
source "$CONTROL_LIBRARY"
1322

1423
mode="${1:---check}"
1524
[ "$#" -eq 1 ] || {
@@ -29,7 +38,7 @@ die() {
2938
exit 1
3039
}
3140

32-
for command in awk jq lscpu paste sort sysctl; do
41+
for command in awk jq lscpu paste sleep sort sysctl; do
3342
command -v "$command" >/dev/null 2>&1 || die "required command missing: $command"
3443
done
3544
[ -x "$RESOURCE_BOUNDARY" ] || die "resource-boundary helper is unavailable"
@@ -43,11 +52,7 @@ write_root_file() {
4352

4453
if [ "$mode" = --apply ]; then
4554
[ "$(id -u)" -eq 0 ] || command -v sudo >/dev/null 2>&1 || die "sudo is required"
46-
if [ -w /sys/devices/system/cpu/smt/control ]; then
47-
printf 'off\n' > /sys/devices/system/cpu/smt/control
48-
else
49-
write_root_file /sys/devices/system/cpu/smt/control off
50-
fi
55+
urnetwork_disable_smt "$SMT_CONTROL" 10 1 || die "could not disable SMT"
5156

5257
while IFS= read -r cpu; do
5358
governor_path="/sys/devices/system/cpu/cpu$cpu/cpufreq/scaling_governor"
@@ -81,7 +86,7 @@ host_cpu_list="$(lscpu -p=CPU | awk -F, '!/^#/ {print $1}' | paste -sd, -)"
8186
logical_cpu_count="$(lscpu -p=CPU | awk -F, '!/^#/ {count++} END {print count+0}')"
8287
physical_core_count="$(lscpu -p=SOCKET,CORE | awk -F, '!/^#/ {seen[$1 ":" $2]=1} END {print length(seen)+0}')"
8388
threads_per_core="$(lscpu -p=CPU,CORE | awk -F, '!/^#/ {count[$2]++} END {max=0; for (core in count) if (max < count[core]) max=count[core]; print max+0}')"
84-
smt_control="$(tr -d '\n' </sys/devices/system/cpu/smt/control 2>/dev/null || true)"
89+
smt_control="$(tr -d '\n' <"$SMT_CONTROL" 2>/dev/null || true)"
8590
governors="$(for path in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
8691
[ -r "$path" ] || continue
8792
cpu="${path#/sys/devices/system/cpu/cpu}"

competition/authoritative-host-irqs.service.example

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@ ConditionFileIsExecutable=/usr/local/libexec/urnetwork/authoritative-host-irqs
99
Type=oneshot
1010
ExecStart=/usr/local/libexec/urnetwork/authoritative-host-irqs --apply
1111
RemainAfterExit=yes
12+
Restart=on-failure
13+
RestartSec=2
1214
TimeoutStartSec=60
1315

1416
[Install]
1517
WantedBy=multi-user.target
18+
RequiredBy=containerd.service docker.service

competition/authoritative-host-irqs.sh

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,14 @@ boundary="$($RESOURCE_BOUNDARY)" || die 'resource boundary is invalid'
3030
management_cpuset="$(jq -er '.management_cpuset' <<<"$boundary")"
3131
evaluation_cpuset="$(jq -er '.evaluation_cpuset' <<<"$boundary")"
3232
[[ "$management_cpuset" =~ ^[0-9,-]+$ ]] || die 'management CPU set is invalid'
33+
irq_policy="$(jq -cnS \
34+
--arg evaluation_cpuset "$evaluation_cpuset" \
35+
--arg management_cpuset "$management_cpuset" \
36+
--argjson minimum_device_irq "$MIN_DEVICE_IRQ" \
37+
'{schema:1,kind:"sim-latency-irq-placement-policy",
38+
evaluation_cpuset:$evaluation_cpuset,management_cpuset:$management_cpuset,
39+
minimum_device_irq:$minimum_device_irq}')"
40+
irq_policy_sha256="$(printf '%s' "$irq_policy" | sha256sum | awk '{print $1}')"
3341

3442
mapfile -t device_irqs < <(
3543
awk -F: -v minimum="$MIN_DEVICE_IRQ" '
@@ -80,6 +88,7 @@ jq -n \
8088
--arg evaluation_cpuset "$evaluation_cpuset" \
8189
--arg management_cpuset "$management_cpuset" \
8290
--arg irq_affinity_sha256 "$irq_affinity_sha256" \
91+
--arg irq_policy_sha256 "$irq_policy_sha256" \
8392
--argjson discovered_irq_count "${#device_irqs[@]}" \
8493
--argjson verified_irq_count "${#verified_irqs[@]}" \
8594
--argjson failed_irqs "$failed_json" \
@@ -88,6 +97,7 @@ jq -n \
8897
evaluation_cpuset:$evaluation_cpuset,management_cpuset:$management_cpuset,
8998
minimum_device_irq:16,discovered_irq_count:$discovered_irq_count,
9099
verified_irq_count:$verified_irq_count,failed_irqs:$failed_irqs,
91-
irq_affinity_sha256:$irq_affinity_sha256,passed:$passed}'
100+
irq_affinity_sha256:$irq_affinity_sha256,
101+
irq_policy_sha256:$irq_policy_sha256,passed:$passed}'
92102

93103
[ "$passed" = true ]

competition/container/smoke-test.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -182,15 +182,15 @@ else
182182
fi
183183
sudo -n chown -R "$container_host_uid:$container_host_gid" "$smoke_root/input" "$smoke_root/output"
184184
sudo -n chmod 0700 "$smoke_root/input" "$smoke_root/output"
185-
config_local_sha256="$($HASH_LOCAL_MOUNT "$config_local_directory")"
186-
vault_local_sha256="$($HASH_LOCAL_MOUNT "$vault_local_directory")"
185+
config_local_sha256="$(sudo -n "$HASH_LOCAL_MOUNT" "$config_local_directory")"
186+
vault_local_sha256="$(sudo -n "$HASH_LOCAL_MOUNT" "$vault_local_directory")"
187187

188188
verify_local_sources_unchanged() {
189-
[ "$($HASH_LOCAL_MOUNT "$config_local_directory")" = "$config_local_sha256" ] || {
189+
[ "$(sudo -n "$HASH_LOCAL_MOUNT" "$config_local_directory")" = "$config_local_sha256" ] || {
190190
printf 'config/local changed during the smoke test\n' >&2
191191
return 1
192192
}
193-
[ "$($HASH_LOCAL_MOUNT "$vault_local_directory")" = "$vault_local_sha256" ] || {
193+
[ "$(sudo -n "$HASH_LOCAL_MOUNT" "$vault_local_directory")" = "$vault_local_sha256" ] || {
194194
printf 'vault/local changed during the smoke test\n' >&2
195195
return 1
196196
}

competition/container_isolation_test.go

Lines changed: 67 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -660,7 +660,8 @@ func TestAuthoritativeHostControlsAreFailClosed(t *testing.T) {
660660
}
661661
control := string(controlBytes)
662662
for _, required := range []string{
663-
`/sys/devices/system/cpu/smt/control off`,
663+
`readonly SMT_CONTROL=/sys/devices/system/cpu/smt/control`,
664+
`urnetwork_disable_smt "$SMT_CONTROL" 10 1`,
664665
`write_root_file "$governor_path" performance`,
665666
`write_root_file /sys/devices/system/cpu/intel_pstate/no_turbo 1`,
666667
`sysctl -q -w vm.overcommit_memory=1`,
@@ -686,7 +687,10 @@ func TestAuthoritativeHostControlsAreFailClosed(t *testing.T) {
686687
`ConditionFileIsExecutable=/usr/local/libexec/urnetwork/authoritative-host-controls`,
687688
`ExecStart=/usr/local/libexec/urnetwork/authoritative-host-controls --apply`,
688689
`RemainAfterExit=yes`,
690+
`Restart=on-failure`,
691+
`RestartSec=2`,
689692
`TimeoutStartSec=60`,
693+
`RequiredBy=containerd.service docker.service`,
690694
} {
691695
if !strings.Contains(controlUnit, required) {
692696
t.Errorf("host-control unit is missing %q", required)
@@ -709,13 +713,42 @@ func TestAuthoritativeHostControlsAreFailClosed(t *testing.T) {
709713
`printf '%s\n' "$management_cpuset" | sudo -n tee "$affinity_path"`,
710714
`[ "$configured" != "$management_cpuset" ]`,
711715
`[ "${#failed_irqs[@]}" -eq 0 ]`,
716+
`kind:"sim-latency-irq-placement-policy"`,
717+
`irq_policy_sha256`,
712718
`[ "$passed" = true ]`,
713719
} {
714720
if !strings.Contains(irq, required) {
715721
t.Errorf("authoritative IRQ control is missing %q", required)
716722
}
717723
}
718724

725+
hostCheckBytes, err := os.ReadFile("host-self-check.sh")
726+
if err != nil {
727+
t.Fatal(err)
728+
}
729+
hostCheck := string(hostCheckBytes)
730+
for _, required := range []string{
731+
`irq_report="$($IRQ_CONTROL --check`,
732+
`[ "$irq_live_passed" = true ]`,
733+
`[ "$irq_policy_sha256" = "$expected_irq_policy_sha" ]`,
734+
} {
735+
if !strings.Contains(hostCheck, required) {
736+
t.Errorf("host IRQ qualification is missing %q", required)
737+
}
738+
}
739+
factsStart := strings.Index(hostCheck, `facts="$(jq -cnS`)
740+
factsEnd := strings.Index(hostCheck, `qualification_sha256="$(printf`)
741+
if factsStart == -1 || factsEnd <= factsStart {
742+
t.Fatal("host qualification facts block is unavailable")
743+
}
744+
facts := hostCheck[factsStart:factsEnd]
745+
if !strings.Contains(facts, `irq_policy_sha256`) {
746+
t.Fatal("host qualification does not bind the stable IRQ policy")
747+
}
748+
if strings.Contains(facts, `irq_affinity_sha256`) {
749+
t.Fatal("host qualification still binds reboot-unstable IRQ numbers")
750+
}
751+
719752
irqUnitBytes, err := os.ReadFile("authoritative-host-irqs.service.example")
720753
if err != nil {
721754
t.Fatal(err)
@@ -727,6 +760,8 @@ func TestAuthoritativeHostControlsAreFailClosed(t *testing.T) {
727760
`Requires=urnetwork-authoritative-host-controls.service`,
728761
`ConditionFileIsExecutable=/usr/local/libexec/urnetwork/authoritative-host-irqs`,
729762
`ExecStart=/usr/local/libexec/urnetwork/authoritative-host-irqs --apply`,
763+
`Restart=on-failure`,
764+
`RequiredBy=containerd.service docker.service`,
730765
} {
731766
if !strings.Contains(irqUnit, required) {
732767
t.Errorf("IRQ unit is missing %q", required)
@@ -740,11 +775,12 @@ func TestAuthoritativeHostControlsAreFailClosed(t *testing.T) {
740775
installer := string(installerBytes)
741776
for _, required := range []string{
742777
`install -D -o root -g root -m 0555 "$CONTROL_SOURCE" "$CONTROL_TARGET"`,
778+
`install -D -o root -g root -m 0444 "$CONTROL_LIBRARY_SOURCE" "$CONTROL_LIBRARY_TARGET"`,
743779
`install -D -o root -g root -m 0555 "$BOUNDARY_SOURCE" "$BOUNDARY_TARGET"`,
744780
`install -D -o root -g root -m 0555 "$IRQ_SOURCE" "$IRQ_TARGET"`,
745781
`install -D -o root -g root -m 0444 "$UNIT_SOURCE" "$UNIT_TARGET"`,
746782
`install -D -o root -g root -m 0444 "$IRQ_UNIT_SOURCE" "$IRQ_UNIT_TARGET"`,
747-
`systemctl enable "$UNIT_NAME" "$IRQ_UNIT_NAME"`,
783+
`systemctl reenable "$UNIT_NAME" "$IRQ_UNIT_NAME"`,
748784
`sudo -n "$CONTROL_TARGET" --check`,
749785
`sudo -n "$IRQ_TARGET" --check`,
750786
} {
@@ -754,6 +790,35 @@ func TestAuthoritativeHostControlsAreFailClosed(t *testing.T) {
754790
}
755791
}
756792

793+
func TestAuthoritativeHostSMTNormalization(t *testing.T) {
794+
command := exec.Command("bash", "./test-authoritative-host-controls-lib.sh")
795+
output, err := command.CombinedOutput()
796+
if err != nil {
797+
t.Fatalf("SMT normalization regression test failed: %v\n%s", err, output)
798+
}
799+
}
800+
801+
func TestContainerSmokeHashesRemappedLocalSourcesAsRoot(t *testing.T) {
802+
scriptBytes, err := os.ReadFile("container/smoke-test.sh")
803+
if err != nil {
804+
t.Fatal(err)
805+
}
806+
script := string(scriptBytes)
807+
for _, required := range []string{
808+
`sudo -n chown -R "$container_host_uid:$container_host_gid" "$smoke_root/local-source"`,
809+
`config_local_sha256="$(sudo -n "$HASH_LOCAL_MOUNT" "$config_local_directory")"`,
810+
`vault_local_sha256="$(sudo -n "$HASH_LOCAL_MOUNT" "$vault_local_directory")"`,
811+
} {
812+
if !strings.Contains(script, required) {
813+
t.Errorf("remapped local-source smoke boundary is missing %q", required)
814+
}
815+
}
816+
if strings.Contains(script, `config_local_sha256="$($HASH_LOCAL_MOUNT`) ||
817+
strings.Contains(script, `vault_local_sha256="$($HASH_LOCAL_MOUNT`) {
818+
t.Fatal("smoke hashes a remapped local source as the unprivileged caller")
819+
}
820+
}
821+
757822
// Build output is attacker-controlled and must be drained without allowing a
758823
// noisy package initializer to consume the host-memory reserve.
759824
func TestEvaluatorBoundsCandidateBuildLogWhileDrainingIt(t *testing.T) {

competition/host-config.example.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"numa_list": "0",
1919
"governor": "performance",
2020
"turbo_state": "disabled",
21-
"irq_affinity_sha256": "REPLACE_WITH_64_HEX",
21+
"irq_policy_sha256": "REPLACE_WITH_64_HEX",
2222
"docker_daemon_config_sha256": "REPLACE_WITH_64_HEX",
2323
"job_cgroup": "/urnetwork/competition.slice/evaluator.scope",
2424
"artifact_quota_bytes": 34359738368,

0 commit comments

Comments
 (0)