Prevent Experiment lifecycle CI deadline failures

tests/dev/security-gate-cases.sh

@@ -230,7 +230,8 @@ guard-mount tests/guard/cases.sh SUMMARY failures=0
 config-authority tests/agent/config-guard.sh SUMMARY failures=0
 experiment-contract tests/experiment/contract-cases.sh EXPERIMENT CONTRACT PASS
 experiment-authorization tests/experiment/authorization-cases.sh EXPERIMENT AUTHORIZATION PASS
-experiment-local-lifecycle tests/experiment/local-lifecycle-cases.sh EXPERIMENT LOCAL LIFECYCLE PASS
+experiment-local-lifecycle tests/experiment/local-onboarding-cases.sh EXPERIMENT LOCAL LIFECYCLE PASS
+experiment-install-lifecycle tests/experiment/install-lifecycle-cases.sh EXPERIMENT INSTALL LIFECYCLE PASS
 experiment-source-adapters tests/experiment/source-adapter-cases.sh EXPERIMENT SOURCE ADAPTERS PASS
 config-matrix tests/agent/config-matrix.sh SUMMARY failures=0
 allowlist-schema tests/agent/allowlist-cases.sh SUMMARY failures=0

tests/experiment/aggregate-harness-cases.sh

@@ -3,6 +3,8 @@ set -u -o pipefail
 
 repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)"
 lifecycle="$repo_root/tests/experiment/local-lifecycle-cases.sh"
+local_onboarding="$repo_root/tests/experiment/local-onboarding-cases.sh"
+install_lifecycle="$repo_root/tests/experiment/install-lifecycle-cases.sh"
 source_adapters="$repo_root/tests/experiment/source-adapter-cases.sh"
 expected_count=23
 work=""
@@ -25,6 +27,8 @@ fi
 trap 'cleanup_work >/dev/null 2>&1 || true' EXIT
 replica="$work/repo"
 replica_lifecycle="$replica/tests/experiment/local-lifecycle-cases.sh"
+replica_local_onboarding="$replica/tests/experiment/local-onboarding-cases.sh"
+replica_install_lifecycle="$replica/tests/experiment/install-lifecycle-cases.sh"
 replica_source_adapters="$replica/tests/experiment/source-adapter-cases.sh"
 mkdir -p "$replica/tests/experiment" "$replica/tests/install"
 cp "$lifecycle" "$replica_lifecycle"
@@ -293,6 +297,38 @@ run_source_replica() {
   run_selected "$output" "$replica_source_adapters" "$@"
 }
 
+route_output_valid() {
+  local output="$1"
+  local expected_file="$2"
+  local expected_assertions="$3"
+  local marker="$4"
+  local rc="$5"
+
+  [ "$rc" -eq 0 ] &&
+    cmp -s "$expected_file" <(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output") &&
+    [ "$(grep -Fxc \
+      "SUMMARY assertions=$expected_assertions expected=$expected_assertions failures=0 infra=0" \
+      "$output" || true)" -eq 1 ] &&
+    [ "$(awk 'END {print}' "$output")" = "$marker" ] &&
+    awk -v summary="SUMMARY assertions=$expected_assertions expected=$expected_assertions failures=0 infra=0" \
+      -v marker="$marker" \
+      '/^(PASS|FAIL) [A-Z0-9-]+ / {next} $0 == summary {next} $0 == marker {next} {bad=1} END {exit bad}' \
+      "$output"
+}
+
+infrastructure_output_valid() {
+  local output="$1"
+  local expected_assertions="$2"
+  local rc="$3"
+
+  [ "$rc" -eq 125 ] &&
+    [ "$(grep -Fxc \
+      "SUMMARY assertions=0 expected=$expected_assertions failures=0 infra=1" \
+      "$output" || true)" -eq 1 ] &&
+    ! grep -Fq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$output" &&
+    ! grep -Fq 'EXPERIMENT INSTALL LIFECYCLE PASS' "$output"
+}
+
 wait_for_path() {
   local path="$1"
   local attempts=0
@@ -355,10 +391,8 @@ run_replica "$work/baseline.out" env \
 mutant_lifecycle="$replica/tests/experiment/local-lifecycle-hidden-duplicate.sh"
 awk '
   { print }
-  $0 == "subcases=(" { in_subcases=1; next }
-  in_subcases && $0 == ")" {
+  $0 == "subcases=(\"${all_subcases[@]:$selected_start:$selected_count}\")" {
     print "\"$repo_root/tests/install/local-install-cases.sh\" >/dev/null 2>&1"
-    in_subcases=0
   }
 ' "$replica_lifecycle" > "$mutant_lifecycle"
 chmod +x "$mutant_lifecycle"
@@ -387,6 +421,169 @@ run_replica "$work/overlap.out" env \
   AGENT_LAB_AGG_EXEC_LOG="$overlap_executions" \
   AGENT_LAB_AGG_BARRIER_DIR="$overlap_barrier" || overlap_rc=$?
 
+split_routing_contract=0
+split_output_contract=0
+split_mutation_contract=0
+tail_mutation_contract=0
+wrapper_infrastructure_contract=0
+if [ -f "$local_onboarding" ] && [ -f "$install_lifecycle" ]; then
+  cp "$local_onboarding" "$replica_local_onboarding"
+  cp "$install_lifecycle" "$replica_install_lifecycle"
+  chmod +x "$replica_local_onboarding" "$replica_install_lifecycle"
+
+  expected_local_ids="$work/expected-local-ids"
+  expected_install_ids="$work/expected-install-ids"
+  expected_local_executions="$work/expected-local-executions"
+  expected_install_executions="$work/expected-install-executions"
+  printf '%s\n' "${expected_ids[@]:0:86}" > "$expected_local_ids"
+  printf '%s\n' "${expected_ids[@]:86:47}" > "$expected_install_ids"
+  printf '%s\n' \
+    local-install-cases.sh \
+    local-config-cases.sh \
+    local-image-catalog-cases.sh > "$expected_local_executions"
+  printf '%s\n' \
+    install-store-cases.sh \
+    install-state-cases.py \
+    install-integrity-cases.py \
+    install-mutation-cases.py > "$expected_install_executions"
+
+  reset_fixtures
+  split_barrier="$work/split-barrier"
+  local_executions="$work/local-executions"
+  install_executions="$work/install-executions"
+  mkdir "$split_barrier"
+  : > "$local_executions"
+  : > "$install_executions"
+  local_route_rc=0
+  install_route_rc=0
+  env \
+    AGENT_LAB_AGG_EXEC_LOG="$local_executions" \
+    AGENT_LAB_AGG_BARRIER_DIR="$split_barrier" \
+    bash "$replica_local_onboarding" > "$work/local-route.out" 2>&1 &
+  local_route_pid=$!
+  env \
+    AGENT_LAB_AGG_EXEC_LOG="$install_executions" \
+    AGENT_LAB_AGG_BARRIER_DIR="$split_barrier" \
+    bash "$replica_install_lifecycle" > "$work/install-route.out" 2>&1 &
+  install_route_pid=$!
+  wait "$local_route_pid" || local_route_rc=$?
+  wait "$install_route_pid" || install_route_rc=$?
+
+  combined_executions="$work/combined-executions"
+  LC_ALL=C sort "$local_executions" "$install_executions" > "$combined_executions"
+  if [ "$local_route_rc" -eq 0 ] && [ "$install_route_rc" -eq 0 ] &&
+     cmp -s <(LC_ALL=C sort "$expected_local_executions") \
+       <(LC_ALL=C sort "$local_executions") &&
+     cmp -s <(LC_ALL=C sort "$expected_install_executions") \
+       <(LC_ALL=C sort "$install_executions") &&
+     cmp -s <(LC_ALL=C sort "$expected_executions") "$combined_executions"; then
+    split_routing_contract=1
+  fi
+  if route_output_valid \
+       "$work/local-route.out" "$expected_local_ids" 86 \
+       'EXPERIMENT LOCAL LIFECYCLE PASS' "$local_route_rc" &&
+     route_output_valid \
+       "$work/install-route.out" "$expected_install_ids" 47 \
+       'EXPERIMENT INSTALL LIFECYCLE PASS' "$install_route_rc"; then
+    split_output_contract=1
+  fi
+
+  mutant_install_route="$replica/tests/experiment/install-lifecycle-wrong-route.sh"
+  mutation_write_rc=0
+  awk '
+    $0 == "exec bash \"$script_dir/local-lifecycle-cases.sh\" install" {
+      print "exec bash \"$script_dir/local-lifecycle-cases.sh\" local"
+      changes++
+      next
+    }
+    { print }
+    END { if (changes != 1) exit 1 }
+  ' "$replica_install_lifecycle" > "$mutant_install_route" || mutation_write_rc=$?
+  chmod +x "$mutant_install_route"
+  mutant_install_executions="$work/mutant-install-executions"
+  : > "$mutant_install_executions"
+  mutant_install_rc=0
+  env AGENT_LAB_AGG_EXEC_LOG="$mutant_install_executions" \
+    bash "$mutant_install_route" > "$work/mutant-install-route.out" 2>&1 || \
+    mutant_install_rc=$?
+  if [ "$mutation_write_rc" -eq 0 ] && [ "$mutant_install_rc" -eq 0 ] &&
+     ! route_output_valid \
+       "$work/mutant-install-route.out" "$expected_install_ids" 47 \
+       'EXPERIMENT INSTALL LIFECYCLE PASS' "$mutant_install_rc" &&
+     ! cmp -s <(LC_ALL=C sort "$expected_install_executions") \
+       <(LC_ALL=C sort "$mutant_install_executions"); then
+    split_mutation_contract=1
+  fi
+
+  tail_subcase_mutant="$replica/tests/experiment/local-lifecycle-tail-subcase.sh"
+  tail_subcase_write_rc=0
+  awk '
+    $0 == "all_subcases=(" { in_subcases=1 }
+    in_subcases && $0 == ")" {
+      print "  \"$repo_root/tests/install/local-install-cases.sh\" # tail-orphan mutant"
+      changes++
+      in_subcases=0
+    }
+    { print }
+    END { if (changes != 1) exit 1 }
+  ' "$replica_lifecycle" > "$tail_subcase_mutant" || tail_subcase_write_rc=$?
+  chmod +x "$tail_subcase_mutant"
+  reset_fixtures
+  tail_subcase_executions="$work/tail-subcase-executions"
+  : > "$tail_subcase_executions"
+  tail_subcase_rc=0
+  env AGENT_LAB_AGG_EXEC_LOG="$tail_subcase_executions" \
+    bash "$tail_subcase_mutant" > "$work/tail-subcase.out" 2>&1 || \
+    tail_subcase_rc=$?
+
+  tail_id_mutant="$replica/tests/experiment/local-lifecycle-tail-id.sh"
+  tail_id_write_rc=0
+  awk '
+    { print }
+    $0 == "  M-STORE-LIVE-001 M-STORE-UNCERT-001 M-STORE-STAGE-001 > \"$expected\"" {
+      print "printf '\''%s\\n'\'' ORPHAN-134 >> \"$expected\""
+      changes++
+    }
+    END { if (changes != 1) exit 1 }
+  ' "$replica_lifecycle" > "$tail_id_mutant" || tail_id_write_rc=$?
+  chmod +x "$tail_id_mutant"
+  reset_fixtures
+  tail_id_executions="$work/tail-id-executions"
+  : > "$tail_id_executions"
+  tail_id_rc=0
+  env AGENT_LAB_AGG_EXEC_LOG="$tail_id_executions" \
+    bash "$tail_id_mutant" > "$work/tail-id.out" 2>&1 || tail_id_rc=$?
+
+  if [ "$tail_subcase_write_rc" -eq 0 ] &&
+     infrastructure_output_valid "$work/tail-subcase.out" 133 "$tail_subcase_rc" &&
+     [ ! -s "$tail_subcase_executions" ] &&
+     [ "$tail_id_write_rc" -eq 0 ] &&
+     infrastructure_output_valid "$work/tail-id.out" 133 "$tail_id_rc" &&
+     [ ! -s "$tail_id_executions" ]; then
+    tail_mutation_contract=1
+  fi
+
+  saved_lifecycle="$work/local-lifecycle-core.saved"
+  mv "$replica_lifecycle" "$saved_lifecycle"
+  missing_local_core_rc=0
+  missing_install_core_rc=0
+  bash "$replica_local_onboarding" > "$work/missing-local-core.out" 2>&1 || \
+    missing_local_core_rc=$?
+  bash "$replica_install_lifecycle" > "$work/missing-install-core.out" 2>&1 || \
+    missing_install_core_rc=$?
+  mv "$saved_lifecycle" "$replica_lifecycle"
+  if infrastructure_output_valid \
+       "$work/missing-local-core.out" 86 "$missing_local_core_rc" &&
+     grep -Fxq 'INFRA shared lifecycle core unavailable' \
+       "$work/missing-local-core.out" &&
+     infrastructure_output_valid \
+       "$work/missing-install-core.out" 47 "$missing_install_core_rc" &&
+     grep -Fxq 'INFRA shared lifecycle core unavailable' \
+       "$work/missing-install-core.out"; then
+    wrapper_infrastructure_contract=1
+  fi
+fi
+
 if [ "$baseline_rc" -eq 0 ] &&
    cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$baseline_executions") &&
    [ "$mutation_count" -eq 1 ] &&
@@ -398,10 +595,13 @@ if [ "$baseline_rc" -eq 0 ] &&
    cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$overlap_executions") &&
    [ -f "$overlap_barrier/local-install-cases.sh.ready" ] &&
    [ -f "$overlap_barrier/local-image-catalog-cases.sh.ready" ] &&
-   [ -f "$overlap_barrier/install-state-cases.py.ready" ]; then
-  pass AGG-001 "execution ledger and overlap barrier prove exact-once concurrent routing"
+   [ -f "$overlap_barrier/install-state-cases.py.ready" ] &&
+   [ "$split_routing_contract" -eq 1 ] &&
+   [ "$split_mutation_contract" -eq 1 ] &&
+   [ "$tail_mutation_contract" -eq 1 ]; then
+  pass AGG-001 "execution ledgers prove exact routing and reject wrong-route or orphan-tail mutants"
 else
-  fail AGG-001 "execution ledger and overlap barrier prove exact-once concurrent routing"
+  fail AGG-001 "execution ledgers prove exact routing and reject wrong-route or orphan-tail mutants"
 fi
 
 reset_fixtures
@@ -412,10 +612,11 @@ if [ "$success_rc" -eq 0 ] &&
    [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 133 ] &&
    [ "$(grep -Fxc 'SUMMARY assertions=133 expected=133 failures=0 infra=0' "$success_output")" -eq 1 ] &&
    [ "$(tail -n 1 "$success_output")" = 'EXPERIMENT LOCAL LIFECYCLE PASS' ] &&
-   awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=133 expected=133 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then
-  pass AGG-002 "success forwards only assertions then one summary and marker"
+   awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=133 expected=133 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output" &&
+   [ "$split_output_contract" -eq 1 ]; then
+  pass AGG-002 "compatibility and split routes emit exact summaries and markers"
 else
-  fail AGG-002 "success forwards only assertions then one summary and marker"
+  fail AGG-002 "compatibility and split routes emit exact summaries and markers"
 fi
 
 missing_records=()
@@ -531,22 +732,38 @@ chmod +x "$summaryless"
 summaryless_rc=0
 run_replica "$work/summaryless.out" env || summaryless_rc=$?
 if [ "$summaryless_rc" -eq 125 ] &&
+   [ "$wrapper_infrastructure_contract" -eq 1 ] &&
    ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/summaryless.out"; then
-  pass AGG-009 "missing subcase summary maps to one hundred twenty-five"
+  pass AGG-009 "missing summaries or shared core map to infrastructure"
 else
-  fail AGG-009 "missing subcase summary maps to one hundred twenty-five"
+  fail AGG-009 "missing summaries or shared core map to infrastructure"
 fi
 
-lane_assignment_count="$(grep -Ec '^[[:space:]]*(readonly[[:space:]]+)?lane_count=' \
+lane_assignment_count="$(grep -Ec '^[[:space:]]+lane_count=[0-9]+$' \
   "$replica_lifecycle" || true)"
-if [ "$lane_assignment_count" -eq 1 ] &&
-   [ "$(grep -Fxc 'readonly lane_count=3' "$replica_lifecycle")" -eq 1 ] &&
+if [ "$lane_assignment_count" -eq 3 ] &&
+   [ "$(grep -Fxc '    lane_count=3' "$replica_lifecycle")" -eq 1 ] &&
+   [ "$(grep -Fxc '    lane_count=2' "$replica_lifecycle")" -eq 1 ] &&
+   [ "$(grep -Fxc '    lane_count=1' "$replica_lifecycle")" -eq 1 ] &&
+   [ "$(grep -Fxc '    lane_map=(0 1 2 0 1 2 0)' "$replica_lifecycle")" -eq 1 ] &&
+   [ "$(grep -Fxc '    lane_map=(0 0 1)' "$replica_lifecycle")" -eq 1 ] &&
+   [ "$(grep -Fxc '    lane_map=(0 0 0 0)' "$replica_lifecycle")" -eq 1 ] &&
+   [ "$(grep -Fxc 'readonly lane_count' "$replica_lifecycle")" -eq 1 ] &&
+   [ "$(grep -Fxc 'readonly lane_map' "$replica_lifecycle")" -eq 1 ] &&
    [ "$(grep -Fxc 'for ((lane = 0; lane < lane_count; lane++)); do' \
      "$replica_lifecycle")" -eq 1 ] &&
-   [ "$(grep -Fxc '  run_lane "$lane" &' "$replica_lifecycle")" -eq 1 ]; then
-  pass AGG-010 "lifecycle declares one immutable three-lane bound"
+   [ "$(grep -Fxc '    [ "${lane_map[$index]}" -eq "$lane" ] || continue' \
+     "$replica_lifecycle")" -eq 1 ] &&
+   [ "$(grep -Fxc '  run_lane "$lane" &' "$replica_lifecycle")" -eq 1 ] &&
+   [ -f "$replica_local_onboarding" ] &&
+   [ -f "$replica_install_lifecycle" ] &&
+   [ "$(grep -Fxc 'exec bash "$script_dir/local-lifecycle-cases.sh" local' \
+     "$replica_local_onboarding" || true)" -eq 1 ] &&
+   [ "$(grep -Fxc 'exec bash "$script_dir/local-lifecycle-cases.sh" install' \
+     "$replica_install_lifecycle" || true)" -eq 1 ]; then
+  pass AGG-010 "split routes preserve three measured aggregate lanes through one scheduler"
 else
-  fail AGG-010 "lifecycle declares one immutable three-lane bound"
+  fail AGG-010 "split routes preserve three measured aggregate lanes through one scheduler"
 fi
 
 hold_ids=(

tests/experiment/install-lifecycle-cases.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -u -o pipefail
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+if [ ! -f "$script_dir/local-lifecycle-cases.sh" ] ||
+   [ ! -r "$script_dir/local-lifecycle-cases.sh" ]; then
+  printf 'INFRA shared lifecycle core unavailable\n' >&2
+  printf 'SUMMARY assertions=0 expected=47 failures=0 infra=1\n'
+  exit 125
+fi
+exec bash "$script_dir/local-lifecycle-cases.sh" install