diff --git a/deployment/aliyun/polymarket-raw-ops-cutover.sh b/deployment/aliyun/polymarket-raw-ops-cutover.sh index 0ad2ab99..0648b33f 100755 --- a/deployment/aliyun/polymarket-raw-ops-cutover.sh +++ b/deployment/aliyun/polymarket-raw-ops-cutover.sh @@ -952,17 +952,66 @@ verify_bootstrap_rust_runtime() { [[ $cmdline == "$RUST_EXEC " ]] } +verify_contained_bootstrap_recovery() { + local recovery=$1 candidate_sha=$2 source_revision=$3 baseline + local active_state main_pid fragment drop_ins exec_argv restarts invocation binary_sha unit + jq -e --arg candidate "$candidate_sha" --arg source "$source_revision" ' + .mode == "gamma_tagged_500" + and .candidate_probe.schema == "monday.polymarket_gamma_tagged_500_recovery_probe.v1" + and .candidate_probe.candidate_sha256 == $candidate + and .candidate_probe.source_revision == $source + and (.candidate_probe.sha256 | type == "string" and test("^[a-f0-9]{64}$")) + and .baseline.active_state == "inactive" + and .baseline.main_pid == 0 + and .baseline.exec_start == "/opt/monday/bin/polymarket-raw-ops collect-reference --max-trade-polls-per-cycle 200" + and .baseline.fragment_path == "/etc/systemd/system/polymarket-reference-collector.service" + and .baseline.drop_in_paths == [] + and (.baseline.restarts | type == "number" and floor == . and . >= 0) + and (.baseline.invocation_id | type == "string" and test("^[a-f0-9]{32}$")) + and .baseline.binary_path == "/opt/monday/bin/polymarket-raw-ops" + and (.baseline.binary_sha256 | type == "string" and test("^[a-f0-9]{64}$")) + and .baseline.binary_sha256 != $candidate + ' <<<"$recovery" >/dev/null || return 1 + baseline=$(jq -c .baseline <<<"$recovery") || return 1 + active_state=$(systemctl show --property=ActiveState --value "$COLLECTOR_UNIT") || return 1 + main_pid=$(systemctl show --property=MainPID --value "$COLLECTOR_UNIT") || return 1 + [[ $active_state == $(jq -er .active_state <<<"$baseline") && $main_pid == 0 ]] \ + || return 1 + fragment=$(systemctl show --property=FragmentPath --value "$COLLECTOR_UNIT") || return 1 + drop_ins=$(systemctl show --property=DropInPaths --value "$COLLECTOR_UNIT") || return 1 + exec_argv=$(effective_exec_argv "$COLLECTOR_UNIT") || return 1 + restarts=$(systemctl show --property=NRestarts --value "$COLLECTOR_UNIT") || return 1 + invocation=$(systemctl show --property=InvocationID --value "$COLLECTOR_UNIT") || return 1 + [[ $fragment == $(jq -er .fragment_path <<<"$baseline") && -z $drop_ins \ + && $exec_argv == $(jq -er .exec_start <<<"$baseline") \ + && $restarts == $(jq -er .restarts <<<"$baseline") \ + && $invocation == $(jq -er .invocation_id <<<"$baseline") ]] || return 1 + [[ -f $ACTIVE_BINARY && ! -L $ACTIVE_BINARY && -x $ACTIVE_BINARY ]] || return 1 + secure_regular_file "$ACTIVE_BINARY" || return 1 + binary_sha=$(sha256sum "$ACTIVE_BINARY" | awk '{print $1}') || return 1 + [[ $binary_sha == $(jq -er .binary_sha256 <<<"$baseline") ]] || return 1 + for unit in "$REFERENCE_UPLOAD_UNIT" "$REFERENCE_UPLOAD_TIMER" \ + "$MARKET_UPLOAD_UNIT" "$MARKET_UPLOAD_TIMER"; do + [[ $(systemctl show --property=ActiveState --value "$unit") == inactive ]] || return 1 + done +} + snapshot_legacy() { local rollback_dir=$1 baseline_mode=${2:-legacy_python} local baseline_release_path=${3:-} baseline_release_sha=${4:-} candidate_sha=${5:-} + local contained_recovery=${6:-false} local state_json=$rollback_dir/state.json asset enabled active mode snapshot_asset local control_present=false control_assets control_files install -d -m 0750 "$rollback_dir/systemd" "$rollback_dir/bin" \ "$rollback_dir/config" "$rollback_dir/control" secure_root_chain "$rollback_dir" \ || die 'rollback snapshot directory chain is not trusted' + [[ $contained_recovery == true || $contained_recovery == false ]] \ + || die 'rollback snapshot has an invalid recovery state' jq -n --arg baseline_mode "$baseline_mode" --arg candidate_sha "$candidate_sha" \ - '{baseline_mode:$baseline_mode,candidate_sha256:$candidate_sha}' >"$state_json" + --argjson contained_recovery "$contained_recovery" \ + '{baseline_mode:$baseline_mode,candidate_sha256:$candidate_sha, + contained_recovery:$contained_recovery}' >"$state_json" for asset in "${UNIT_ASSETS[@]}"; do secure_regular_file "/etc/systemd/system/$asset" mode=$(stat -c %a -- "/etc/systemd/system/$asset") @@ -1060,6 +1109,7 @@ restore_legacy() ( local expected_manifest_sha started_epoch rollback_pid current_pid restarts rollback_mode local rollback_sha temporary_link previous_health_sha current_health_sha local bootstrap_path bootstrap_sha bootstrap_mode bootstrap_restored bootstrap_active_mode + local contained_recovery local control_dir_present control_files= local rollback_health_policy=$rollback_dir/control/polymarket-legacy-health-policy.jq secure_root_chain "$evidence_dir" || die 'rollback evidence directory is not trusted' @@ -1071,6 +1121,19 @@ restore_legacy() ( ) || die 'rollback snapshot checksum failed' rollback_mode=$(jq -er '.baseline_mode // "legacy_python" | select(. == "legacy_python" or . == "rust_release" or . == "rust_bootstrap")' \ "$rollback_dir/state.json") || die 'rollback snapshot has no valid baseline mode' + contained_recovery=$(jq -er \ + '(.contained_recovery // false) | select(type == "boolean") | tostring' \ + "$rollback_dir/state.json") \ + || die 'rollback snapshot has no valid contained recovery state' + if [[ $contained_recovery == true ]]; then + jq -e --arg collector "$COLLECTOR_UNIT" --arg reference_timer "$REFERENCE_UPLOAD_TIMER" \ + --arg market_timer "$MARKET_UPLOAD_TIMER" ' + .units[$collector].active == false + and .units[$reference_timer].active == false + and .units[$market_timer].active == false + ' "$rollback_dir/state.json" >/dev/null \ + || die 'contained recovery rollback would restart a saved baseline unit' + fi [[ $rollback_mode == rust_release ]] \ && rollback_health_policy=$rollback_dir/control/polymarket-rust-health-policy.jq [[ $rollback_mode == rust_bootstrap ]] || secure_regular_file "$rollback_health_policy" @@ -1093,6 +1156,10 @@ restore_legacy() ( die 'Rust rollback snapshot has no control release' fi + if [[ $contained_recovery == true ]]; then + clear_health_before_restart "$evidence_dir" \ + "pre-contained-recovery-rollback-$(date -u +%Y%m%dT%H%M%SZ)-$$" + fi systemctl stop "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER" systemctl stop "$REFERENCE_UPLOAD_UNIT" "$MARKET_UPLOAD_UNIT" systemctl stop "$COLLECTOR_UNIT" @@ -1100,8 +1167,9 @@ restore_legacy() ( current_health_sha= [[ $rollback_mode == legacy_python || ! -f $HEALTH || -L $HEALTH ]] \ || previous_health_sha=$(sha256sum "$HEALTH" | awk '{print $1}') - [[ $rollback_mode == rust_release ]] \ - || clear_health_before_restart "$evidence_dir" "pre-rollback-$(date -u +%Y%m%dT%H%M%SZ)-$$" + if [[ $contained_recovery == false && $rollback_mode != rust_release ]]; then + clear_health_before_restart "$evidence_dir" "pre-rollback-$(date -u +%Y%m%dT%H%M%SZ)-$$" + fi for asset in "${UNIT_ASSETS[@]}"; do mode=$(jq -r --arg asset "$asset" '.unit_modes[$asset] // "0644"' \ "$rollback_dir/state.json") @@ -1208,6 +1276,29 @@ restore_legacy() ( sync -f /etc/systemd/system sync -f /opt/monday systemctl daemon-reload + if [[ $contained_recovery == true ]]; then + systemctl reset-failed "$COLLECTOR_UNIT" + systemctl stop "$COLLECTOR_UNIT" + for asset in "$COLLECTOR_UNIT" "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER"; do + if jq -e --arg unit "$asset" '.units[$unit].enabled == true' \ + "$rollback_dir/state.json" >/dev/null; then + systemctl enable "$asset" + else + systemctl disable "$asset" + fi + done + for asset in "$COLLECTOR_UNIT" "$REFERENCE_UPLOAD_UNIT" "$REFERENCE_UPLOAD_TIMER" \ + "$MARKET_UPLOAD_UNIT" "$MARKET_UPLOAD_TIMER"; do + [[ $(systemctl show --property=ActiveState --value "$asset") == inactive ]] \ + || die 'contained recovery rollback restarted a collector or uploader' + done + [[ $(systemctl show --property=MainPID --value "$COLLECTOR_UNIT") == 0 ]] \ + || die 'contained recovery rollback left a collector process running' + verify_saved_unit_state "$rollback_dir/state.json" \ + || die 'contained recovery rollback did not restore saved unit state' + printf '%s\n' "$evidence_dir" + return + fi systemctl reset-failed "$COLLECTOR_UNIT" [[ $(systemctl show --property=NRestarts --value "$COLLECTOR_UNIT") == 0 ]] \ || die 'legacy restart counter did not reset before rollback verification' @@ -1585,6 +1676,22 @@ baseline_runtime_stability_required=$(jq -er \ '.baseline_runtime_stability_required | select(type == "boolean") | tostring' \ "$gate_json") \ || die 'shadow gate has no valid baseline runtime stability contract' +recovery_json=$(jq -c '.recovery // null' "$gate_json") \ + || die 'shadow gate has no valid recovery binding' +contained_recovery=false +if [[ $recovery_json != null ]]; then + contained_recovery=true + [[ $baseline_mode == rust_bootstrap \ + && $baseline_runtime_stability_required == false ]] \ + || die 'contained recovery has an invalid baseline runtime contract' + [[ $(jq -er '.baseline_degraded | select(. == true) | tostring' "$gate_json") == true ]] \ + || die 'contained recovery baseline is not explicitly degraded' + verify_contained_bootstrap_recovery "$recovery_json" "$candidate_sha" \ + "$gate_source_revision" \ + || die 'contained recovery baseline identity changed after the shadow gate' + gate_baseline_release_path=$(jq -er '.baseline.binary_path' <<<"$recovery_json") + gate_baseline_release_sha=$(jq -er '.baseline.binary_sha256' <<<"$recovery_json") +else legacy_pid=$(systemctl show --property=MainPID --value "$COLLECTOR_UNIT") [[ $legacy_pid =~ ^[1-9][0-9]*$ ]] \ || die 'cutover requires a verifiable active legacy reference collector PID' @@ -1596,7 +1703,7 @@ gate_legacy_restarts=$(jq -er \ gate_legacy_invocation_id=$(jq -er \ '.legacy_runtime.invocation_id | select(type == "string" and test("^[a-f0-9]{32}$"))' \ "$gate_json") || die 'shadow gate has no valid legacy systemd invocation ID' -if [[ $baseline_mode == legacy_python ]]; then +if [[ $contained_recovery == false && $baseline_mode == legacy_python ]]; then if [[ $baseline_runtime_stability_required == true ]]; then [[ $legacy_pid == "$gate_legacy_pid" ]] \ || die 'legacy collector MainPID changed after the shadow gate' @@ -1658,6 +1765,7 @@ else "$legacy_pid" "$gate_legacy_invocation_id" "$gate_legacy_restarts" \ || die 'bootstrap Rust baseline identity or restart counter changed after the shadow gate' fi +fi verify_cutover_target_preflight "$baseline_mode" "$ACTIVE_BINARY" \ "$CONTROL_DIR" "${RELEASE_MANIFEST##*/}" secure_regular_file \ || die 'production cutover target state would reject promotion' @@ -1672,7 +1780,8 @@ mkdir -m 0750 "$evidence_dir" || die 'cutover evidence directory already exists' secure_root_chain "$evidence_dir" || die 'cutover evidence directory is not trusted' rollback_dir="$evidence_dir/rollback" snapshot_legacy "$rollback_dir" "$baseline_mode" \ - "${gate_baseline_release_path:-}" "${gate_baseline_release_sha:-}" "$candidate_sha" + "${gate_baseline_release_path:-}" "${gate_baseline_release_sha:-}" "$candidate_sha" \ + "$contained_recovery" transition_started=false cutover_succeeded=false @@ -1726,7 +1835,7 @@ elif [[ $baseline_mode == rust_release ]]; then verify_upload_units "$baseline_pinned_upload_env" \ || die 'Rust baseline upload units changed before drain' fi -if [[ $baseline_mode != rust_bootstrap ]]; then +if [[ $contained_recovery == false && $baseline_mode != rust_bootstrap ]]; then systemctl start "$REFERENCE_UPLOAD_UNIT" verify_oneshot_success "$REFERENCE_UPLOAD_UNIT" \ || die 'legacy reference uploader drain did not complete successfully' @@ -1736,36 +1845,42 @@ if [[ $baseline_mode == rust_release ]]; then verify_oneshot_success "$MARKET_UPLOAD_UNIT" \ || die 'Rust market uploader drain did not complete successfully' fi -legacy_stop_cursor=$(journal_cursor "$COLLECTOR_UNIT") \ - || die 'could not capture the legacy collector journal cursor before stop' -[[ $(oss_config_sha256) == "$gate_oss_config_sha" ]] \ - || die 'OSS configuration changed during the legacy uploader drain' -if [[ $baseline_mode == rust_release ]]; then - [[ $(oss_config_sha256 "$baseline_pinned_upload_env") == "$gate_oss_config_sha" ]] \ - || die 'active Rust uploader configuration changed during drain' -fi -if [[ $baseline_mode == legacy_python ]]; then -verify_legacy_runtime "$legacy_pid" "$gate_legacy_restarts" "$gate_legacy_invocation_id" \ - || die 'legacy collector identity or restart counter changed during uploader drain' -elif [[ $baseline_mode == rust_release ]]; then - pre_stop_health_not_before=$(($(date -u +%s) - MAX_HEALTH_SILENCE_SECONDS)) - verify_rust_runtime "$gate_baseline_release_path" "$pre_stop_health_not_before" \ - "$legacy_pid" "$gate_legacy_invocation_id" "$gate_legacy_restarts" \ - "$LEGACY_HEALTH_POLICY" \ - || die 'Rust baseline identity or health changed during uploader drain' +if [[ $contained_recovery == true ]]; then + verify_contained_bootstrap_recovery "$recovery_json" "$candidate_sha" \ + "$gate_source_revision" \ + || die 'contained recovery baseline changed before promotion' else - verify_bootstrap_rust_runtime "$gate_baseline_release_path" "$gate_baseline_release_sha" \ - "$legacy_pid" "$gate_legacy_invocation_id" "$gate_legacy_restarts" \ - || die 'bootstrap Rust baseline identity changed during uploader drain' -fi + legacy_stop_cursor=$(journal_cursor "$COLLECTOR_UNIT") \ + || die 'could not capture the legacy collector journal cursor before stop' + [[ $(oss_config_sha256) == "$gate_oss_config_sha" ]] \ + || die 'OSS configuration changed during the legacy uploader drain' + if [[ $baseline_mode == rust_release ]]; then + [[ $(oss_config_sha256 "$baseline_pinned_upload_env") == "$gate_oss_config_sha" ]] \ + || die 'active Rust uploader configuration changed during drain' + fi + if [[ $baseline_mode == legacy_python ]]; then + verify_legacy_runtime "$legacy_pid" "$gate_legacy_restarts" "$gate_legacy_invocation_id" \ + || die 'legacy collector identity or restart counter changed during uploader drain' + elif [[ $baseline_mode == rust_release ]]; then + pre_stop_health_not_before=$(($(date -u +%s) - MAX_HEALTH_SILENCE_SECONDS)) + verify_rust_runtime "$gate_baseline_release_path" "$pre_stop_health_not_before" \ + "$legacy_pid" "$gate_legacy_invocation_id" "$gate_legacy_restarts" \ + "$LEGACY_HEALTH_POLICY" \ + || die 'Rust baseline identity or health changed during uploader drain' + else + verify_bootstrap_rust_runtime "$gate_baseline_release_path" "$gate_baseline_release_sha" \ + "$legacy_pid" "$gate_legacy_invocation_id" "$gate_legacy_restarts" \ + || die 'bootstrap Rust baseline identity changed during uploader drain' + fi -systemctl stop "$COLLECTOR_UNIT" -verify_no_restart_after_cursor \ - "$COLLECTOR_UNIT" "$legacy_stop_cursor" "$gate_legacy_invocation_id" \ - || die 'legacy collector journal recorded a restart during final stop' -stopped_legacy_restarts=$(systemctl show --property=NRestarts --value "$COLLECTOR_UNIT") -[[ $stopped_legacy_restarts == "$gate_legacy_restarts" ]] \ - || die 'legacy collector restarted between final verification and stop' + systemctl stop "$COLLECTOR_UNIT" + verify_no_restart_after_cursor \ + "$COLLECTOR_UNIT" "$legacy_stop_cursor" "$gate_legacy_invocation_id" \ + || die 'legacy collector journal recorded a restart during final stop' + stopped_legacy_restarts=$(systemctl show --property=NRestarts --value "$COLLECTOR_UNIT") + [[ $stopped_legacy_restarts == "$gate_legacy_restarts" ]] \ + || die 'legacy collector restarted between final verification and stop' +fi legacy_state_handoff_json=null if [[ $baseline_mode == legacy_python ]]; then clear_health_before_restart "$evidence_dir" pre-cutover @@ -1913,6 +2028,7 @@ jq -n \ --arg journal_sha256 "$journal_sha" \ --arg rollback_manifest_sha256 "$rollback_sha" \ --arg rust_invocation_id "$rust_invocation_id" \ + --argjson recovery "$recovery_json" \ --argjson legacy_state_handoff "$legacy_state_handoff_json" \ --argjson main_pid "$main_pid" \ '{schema:$schema,baseline_mode:$baseline_mode,candidate_sha256:$candidate_sha256, @@ -1924,6 +2040,7 @@ jq -n \ gate_json_sha256:$gate_json_sha256, gate_terminal_receipt_sha256:$gate_terminal_receipt_sha256, gate_systemd_invocation_id:$gate_systemd_invocation_id, + recovery:$recovery, legacy_state_handoff:$legacy_state_handoff, completed_at:$completed_at, collector:{main_pid:$main_pid,restarts:0,invocation_id:$rust_invocation_id, diff --git a/deployment/aliyun/polymarket-raw-ops-gate-control.sh b/deployment/aliyun/polymarket-raw-ops-gate-control.sh index 6329384c..5dec70cb 100755 --- a/deployment/aliyun/polymarket-raw-ops-gate-control.sh +++ b/deployment/aliyun/polymarket-raw-ops-gate-control.sh @@ -12,6 +12,16 @@ readonly SCRIPT_DIR readonly CONTROL="$SCRIPT_DIR/${0##*/}" readonly GATE="$SCRIPT_DIR/polymarket-raw-ops-shadow-gate.sh" readonly UNIT_ASSET="$SCRIPT_DIR/$UNIT_TEMPLATE" +readonly COLLECTOR_UNIT=polymarket-reference-collector.service +readonly RUST_PRODUCTION_EXEC='/opt/monday/bin/polymarket-raw-ops collect-reference --max-trade-polls-per-cycle 200' +readonly RUST_ACTIVE_BINARY_PATH=/opt/monday/bin/polymarket-raw-ops +readonly RECOVERY_PROBE_MAX_AGE_SECONDS=900 +readonly -a RECOVERY_UPLOAD_UNITS=( + polymarket-reference-upload.service + polymarket-reference-upload.timer + polymarket-market-tape-upload.service + polymarket-market-tape-upload.timer +) die() { printf 'Polymarket Gate control failed: %s\n' "$*" >&2 @@ -22,6 +32,7 @@ usage() { printf '%s\n' \ "Usage: ${0##*/} install" \ "Usage: ${0##*/} start " \ + " ${0##*/} recover " \ " ${0##*/} status " \ " ${0##*/} cancel " } @@ -52,9 +63,13 @@ INSTALLED_UNIT=$(prefix_path "/etc/systemd/system/$UNIT_TEMPLATE") readonly INSTALLED_UNIT SYSTEMD_UNIT_DIR=$(prefix_path /etc/systemd/system) readonly SYSTEMD_UNIT_DIR +RUST_ACTIVE_BINARY=$(prefix_path "$RUST_ACTIVE_BINARY_PATH") +readonly RUST_ACTIVE_BINARY +RECOVERY_PROBE_ROOT=$(prefix_path /data/monday/evidence/polymarket-candidate-probes) +readonly RECOVERY_PROBE_ROOT readonly CONTROL_LOCK="$RUN_ROOT/control.lock" -for command in awk chmod cmp date flock install jq ln mkdir mv rm sha256sum stat \ +for command in awk chmod cmp date flock install jq ln mkdir mv readlink rm sed sha256sum stat \ sync systemctl tr; do command -v "$command" >/dev/null 2>&1 \ || die "missing required command: $command" @@ -92,6 +107,15 @@ runtime_request_for() { printf '%s/%s.request.json\n' "$RUN_ROOT" "$1"; } invocation_dir_for() { printf '%s/%s/%s\n' "$RECEIPT_ROOT" "$1" "$2"; } systemctl_value() { systemctl show "$1" --property="$2" --value; } +effective_exec_argv() { + local unit=$1 raw argv + raw=$(systemctl_value "$unit" ExecStart) || return 1 + argv=$(sed -nE 's/^.*argv\[\]=([^;]+);.*$/\1/p' <<<"$raw" \ + | sed -E 's/[[:space:]]+$//') + [[ -n $argv ]] || return 1 + printf '%s\n' "$argv" +} + sync_file() { [[ $test_mode == true ]] || sync "$1"; } sync_dir() { [[ $test_mode == true ]] || sync -f "$1"; } @@ -174,7 +198,7 @@ install_gate_unit() { } write_runtime_request() { - local candidate_path=$1 candidate_sha=$2 source_revision=$3 + local candidate_path=$1 candidate_sha=$2 source_revision=$3 recovery=${4:-null} local destination temporary control_sha gate_sha unit_sha destination=$(runtime_request_for "$candidate_sha") [[ ! -L $destination ]] || die 'runtime request is a symlink' @@ -185,11 +209,12 @@ write_runtime_request() { jq -n --arg candidate "$candidate_sha" --arg candidate_path "$candidate_path" \ --arg source "$source_revision" \ --arg control_sha "$control_sha" --arg gate_sha "$gate_sha" \ - --arg unit_sha "$unit_sha" ' + --arg unit_sha "$unit_sha" --argjson recovery "$recovery" ' {schema:"monday.polymarket_gate_request.v1",candidate_sha256:$candidate, candidate_path:$candidate_path, source_revision:$source,control_sha256:$control_sha, gate_sha256:$gate_sha,unit_sha256:$unit_sha} + + (if $recovery == null then {} else {recovery:$recovery} end) ' >"$temporary" chmod 0444 "$temporary" mv "$temporary" "$destination" @@ -612,8 +637,108 @@ cancel_gate() { jq -c . "$receipt" } +recovery_probe() { + local candidate_sha=$1 source_revision=$2 probe=$3 canonical now observed age + local probe_sha probe_json + candidate_sha=$(printf '%s' "$candidate_sha" | tr '[:upper:]' '[:lower:]') + source_revision=$(printf '%s' "$source_revision" | tr '[:upper:]' '[:lower:]') + valid_candidate "$candidate_sha" || die 'candidate SHA-256 is invalid' + valid_source "$source_revision" || die 'source revision is invalid' + canonical=$(readlink -f -- "$probe") || die 'recovery probe cannot be resolved' + [[ $canonical == "$RECOVERY_PROBE_ROOT/$candidate_sha/"* ]] \ + || die 'recovery probe is outside the exact candidate evidence root' + secure_control_file "$canonical" + probe_json=$(jq -cS . "$canonical") || die 'recovery probe is not valid JSON' + jq -e --arg candidate "$candidate_sha" --arg source "$source_revision" ' + .schema == "monday.polymarket_gamma_tagged_500_recovery_probe.v1" + and .candidate_sha256 == $candidate and .source_revision == $source + and (.observed_at | type == "string" + and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")) + and .gamma.tagged_closed == {query:"closed=true&tag_id=21",attempts:3,http_status:500} + and .gamma.untagged_closed == {query:"closed=true",attempts:3,http_status:200} + and .candidate_once.exit_status == 0 + and (.candidate_once.duration_seconds | type == "number" and floor == . and . > 0 and . <= 180) + and .candidate_once.health_updated_at == .observed_at + ' <<<"$probe_json" >/dev/null || die 'recovery probe does not prove the bounded Gamma fallback' + observed=$(jq -er .observed_at <<<"$probe_json") + observed=$(date -u -d "$observed" +%s) || die 'recovery probe timestamp is invalid' + now=$(date -u +%s) + age=$((now - observed)) + ((age >= 0 && age <= RECOVERY_PROBE_MAX_AGE_SECONDS)) \ + || die 'recovery probe is stale or from the future' + probe_sha=$(sha256sum "$canonical" | awk '{print $1}') + jq -c --arg sha "$probe_sha" '. + {sha256:$sha}' <<<"$probe_json" +} + +recovery_baseline() { + local candidate_sha=$1 active_state main_pid fragment drop_ins exec_argv + local restarts invocation binary_sha + active_state=$(systemctl_value "$COLLECTOR_UNIT" ActiveState) \ + || die 'cannot read contained baseline state' + [[ $active_state == inactive ]] \ + || die 'recovery requires the direct bootstrap baseline to be stopped' + main_pid=$(systemctl_value "$COLLECTOR_UNIT" MainPID) \ + || die 'cannot read contained baseline MainPID' + [[ $main_pid == 0 ]] || die 'recovery baseline still has a managed process' + fragment=$(systemctl_value "$COLLECTOR_UNIT" FragmentPath) \ + || die 'cannot read contained baseline fragment' + [[ $fragment == "/etc/systemd/system/$COLLECTOR_UNIT" ]] \ + || die 'recovery baseline unit fragment is not exact' + drop_ins=$(systemctl_value "$COLLECTOR_UNIT" DropInPaths) \ + || die 'cannot read contained baseline drop-ins' + [[ -z $drop_ins ]] || die 'recovery baseline has unexpected unit drop-ins' + exec_argv=$(effective_exec_argv "$COLLECTOR_UNIT") \ + || die 'cannot read contained baseline ExecStart' + [[ $exec_argv == "$RUST_PRODUCTION_EXEC" ]] \ + || die 'recovery baseline ExecStart is not the direct Rust bootstrap' + restarts=$(systemctl_value "$COLLECTOR_UNIT" NRestarts) \ + || die 'cannot read contained baseline restart counter' + [[ $restarts =~ ^[0-9]+$ ]] \ + || die 'contained baseline restart counter is invalid' + invocation=$(systemctl_value "$COLLECTOR_UNIT" InvocationID) \ + || die 'cannot read contained baseline invocation ID' + valid_invocation "$invocation" || die 'contained baseline invocation ID is invalid' + [[ -f $RUST_ACTIVE_BINARY && ! -L $RUST_ACTIVE_BINARY && -x $RUST_ACTIVE_BINARY ]] \ + || die 'recovery baseline is not the direct executable' + secure_control_file "$RUST_ACTIVE_BINARY" + binary_sha=$(sha256sum "$RUST_ACTIVE_BINARY" | awk '{print $1}') + valid_candidate "$binary_sha" || die 'contained baseline digest is invalid' + [[ $candidate_sha != "$binary_sha" ]] \ + || die 'candidate digest matches the contained bootstrap baseline' + jq -cn --arg active_state "$active_state" --arg exec_start "$exec_argv" \ + --arg fragment_path "$fragment" --arg invocation_id "$invocation" \ + --arg binary_path "$RUST_ACTIVE_BINARY_PATH" --arg binary_sha256 "$binary_sha" \ + --argjson main_pid "$main_pid" --argjson restarts "$restarts" \ + '{active_state:$active_state,main_pid:$main_pid,exec_start:$exec_start, + fragment_path:$fragment_path,drop_in_paths:[],restarts:$restarts, + invocation_id:$invocation_id,binary_path:$binary_path,binary_sha256:$binary_sha256}' +} + +verify_recovery_uploaders_stopped() { + local unit active_state + for unit in "${RECOVERY_UPLOAD_UNITS[@]}"; do + active_state=$(systemctl_value "$unit" ActiveState) \ + || die "cannot read recovery uploader state: $unit" + [[ $active_state == inactive ]] \ + || die "recovery requires inactive uploader/timer: $unit" + done +} + +recover_gate() { + local candidate_path=$1 candidate_sha=$2 source_revision=$3 probe=$4 + local probe_json baseline_json recovery_json + candidate_sha=$(printf '%s' "$candidate_sha" | tr '[:upper:]' '[:lower:]') + source_revision=$(printf '%s' "$source_revision" | tr '[:upper:]' '[:lower:]') + probe_json=$(recovery_probe "$candidate_sha" "$source_revision" "$probe") + baseline_json=$(recovery_baseline "$candidate_sha") + verify_recovery_uploaders_stopped + recovery_json=$(jq -cn --argjson baseline "$baseline_json" --argjson probe "$probe_json" \ + '{mode:"gamma_tagged_500",baseline:$baseline,candidate_probe:$probe}') + start_gate "$candidate_path" "$candidate_sha" "$source_revision" "$recovery_json" +} + start_gate() { - local candidate_path=$1 candidate_sha source_revision=$3 unit invocation env_file + local candidate_path=$1 candidate_sha source_revision=$3 recovery=${4:-null} unit invocation env_file local active_state main_pid candidate_sha=$(printf '%s' "$2" | tr '[:upper:]' '[:lower:]') source_revision=$(printf '%s' "$source_revision" | tr '[:upper:]' '[:lower:]') @@ -645,7 +770,7 @@ start_gate() { || die 'cannot read Gate state before start' [[ $active_state == inactive || $active_state == failed ]] \ || die 'a Gate job already owns this candidate' - write_runtime_request "$candidate_path" "$candidate_sha" "$source_revision" + write_runtime_request "$candidate_path" "$candidate_sha" "$source_revision" "$recovery" env_file="$RUN_ROOT/$candidate_sha.env" [[ ! -L $env_file ]] || die 'Gate EnvironmentFile is a symlink' printf 'MONDAY_POLYMARKET_GATE_CONTROL=%s\n' "$CONTROL" >"${env_file}.tmp.$$" @@ -680,6 +805,10 @@ case "$action" in [[ $# -eq 4 ]] || { usage >&2; exit 2; } start_gate "$2" "$3" "$4" ;; + recover) + [[ $# -eq 5 ]] || { usage >&2; exit 2; } + recover_gate "$2" "$3" "$4" "$5" + ;; status) [[ $# -eq 3 ]] || { usage >&2; exit 2; } if ! valid_candidate "$2" || ! valid_invocation "$3"; then diff --git a/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh b/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh index 7b7f88ce..efbefed5 100755 --- a/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh +++ b/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh @@ -442,6 +442,11 @@ legacy_runtime_budget_observation() { } verify_baseline_identity() { + if [[ ${baseline_recovery:-false} == true ]]; then + verify_contained_recovery_baseline "$recovery_json" "$candidate_sha" \ + "$source_revision" + return + fi if [[ $baseline_mode == legacy_python ]]; then verify_legacy_identity "$legacy_pid" "$legacy_restarts" "$legacy_invocation_id" return @@ -471,6 +476,74 @@ verify_baseline_identity() { [[ $(readlink -f -- "/proc/$legacy_pid/exe") == "$baseline_release_path" ]] } +verify_recovery_binding() { + local recovery=$1 candidate=$2 source=$3 + jq -e --arg candidate "$candidate" --arg source "$source" ' + .mode == "gamma_tagged_500" + and .candidate_probe.schema == "monday.polymarket_gamma_tagged_500_recovery_probe.v1" + and .candidate_probe.candidate_sha256 == $candidate + and .candidate_probe.source_revision == $source + and (.candidate_probe.sha256 | type == "string" and test("^[a-f0-9]{64}$")) + and (.candidate_probe.observed_at | type == "string" + and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")) + and .candidate_probe.gamma.tagged_closed == {query:"closed=true&tag_id=21",attempts:3,http_status:500} + and .candidate_probe.gamma.untagged_closed == {query:"closed=true",attempts:3,http_status:200} + and .candidate_probe.candidate_once.exit_status == 0 + and (.candidate_probe.candidate_once.duration_seconds | type == "number" + and floor == . and . > 0 and . <= 180) + and .candidate_probe.candidate_once.health_updated_at == .candidate_probe.observed_at + and .baseline.active_state == "inactive" + and .baseline.main_pid == 0 + and .baseline.exec_start == "/opt/monday/bin/polymarket-raw-ops collect-reference --max-trade-polls-per-cycle 200" + and .baseline.fragment_path == "/etc/systemd/system/polymarket-reference-collector.service" + and .baseline.drop_in_paths == [] + and (.baseline.restarts | type == "number" and floor == . and . >= 0) + and (.baseline.invocation_id | type == "string" and test("^[a-f0-9]{32}$")) + and .baseline.binary_path == "/opt/monday/bin/polymarket-raw-ops" + and (.baseline.binary_sha256 | type == "string" and test("^[a-f0-9]{64}$")) + and .baseline.binary_sha256 != $candidate + ' <<<"$recovery" >/dev/null +} + +verify_recovery_admission() { + local recovery=$1 candidate=$2 source=$3 observed now age + verify_recovery_binding "$recovery" "$candidate" "$source" || return 1 + observed=$(jq -er '.candidate_probe.observed_at' <<<"$recovery") || return 1 + observed=$(date -u -d "$observed" +%s) || return 1 + now=$(date -u +%s) || return 1 + age=$((now - observed)) + ((age >= 0 && age <= 900)) +} + +verify_contained_recovery_baseline() { + local recovery=$1 candidate=$2 source=$3 expected active_state main_pid + local exec_argv fragment drop_ins restarts invocation binary_sha unit + verify_recovery_binding "$recovery" "$candidate" "$source" || return 1 + expected=$(jq -c .baseline <<<"$recovery") || return 1 + active_state=$(systemctl show --property=ActiveState --value "$LEGACY_UNIT") || return 1 + main_pid=$(systemctl show --property=MainPID --value "$LEGACY_UNIT") || return 1 + [[ $active_state == $(jq -er .active_state <<<"$expected") && $main_pid == 0 ]] \ + || return 1 + fragment=$(systemctl show --property=FragmentPath --value "$LEGACY_UNIT") || return 1 + drop_ins=$(systemctl show --property=DropInPaths --value "$LEGACY_UNIT") || return 1 + exec_argv=$(effective_exec_argv "$LEGACY_UNIT") || return 1 + restarts=$(systemctl show --property=NRestarts --value "$LEGACY_UNIT") || return 1 + invocation=$(systemctl show --property=InvocationID --value "$LEGACY_UNIT") || return 1 + [[ $fragment == $(jq -er .fragment_path <<<"$expected") && -z $drop_ins \ + && $exec_argv == $(jq -er .exec_start <<<"$expected") \ + && $restarts == $(jq -er .restarts <<<"$expected") \ + && $invocation == $(jq -er .invocation_id <<<"$expected") ]] || return 1 + [[ -f $RUST_ACTIVE_BINARY && ! -L $RUST_ACTIVE_BINARY && -x $RUST_ACTIVE_BINARY ]] \ + || return 1 + secure_control_file "$RUST_ACTIVE_BINARY" || return 1 + binary_sha=$(sha256sum "$RUST_ACTIVE_BINARY" | awk '{print $1}') || return 1 + [[ $binary_sha == $(jq -er .binary_sha256 <<<"$expected") ]] || return 1 + for unit in polymarket-reference-upload.service polymarket-reference-upload.timer \ + polymarket-market-tape-upload.service polymarket-market-tape-upload.timer; do + [[ $(systemctl show --property=ActiveState --value "$unit") == inactive ]] || return 1 + done +} + verify_cutover_target_preflight() { local baseline_mode=$1 active_binary=$2 control_dir=$3 release_manifest_name=$4 local file_verifier=$5 unit fragment expected_fragment drop_ins asset assets @@ -1259,63 +1332,84 @@ secure_root_chain /run/monday || die 'runtime control directory is not trusted' exec 9>"$LOCK_FILE" flock -n 9 || die 'another Polymarket release operation is running' -baseline_exec=$(effective_exec_argv "$LEGACY_UNIT") || \ - die 'active reference collector has no verifiable ExecStart' baseline_release_path= baseline_release_sha= -case "$baseline_exec" in - "$LEGACY_EXEC") - baseline_mode=legacy_python - baseline_label=Python - ;; - "$RUST_PRODUCTION_EXEC") - if [[ -L $RUST_ACTIVE_BINARY ]]; then - baseline_mode=rust_release - baseline_label='Rust production' - baseline_release_path=$(readlink -f -- "$RUST_ACTIVE_BINARY") || \ - die 'active Rust collector symlink cannot be resolved' - [[ $baseline_release_path =~ ^$RELEASE_ROOT/([a-f0-9]{64})/polymarket-raw-ops$ ]] \ - || die 'active Rust collector does not resolve to an immutable release' - baseline_release_sha=${BASH_REMATCH[1]} - else - baseline_mode=rust_bootstrap - baseline_label='Rust bootstrap' - [[ -f $RUST_ACTIVE_BINARY && ! -L $RUST_ACTIVE_BINARY ]] \ - || die 'active Rust collector is neither an immutable release nor a direct bootstrap binary' - baseline_release_path=$(readlink -f -- "$RUST_ACTIVE_BINARY") || \ - die 'active Rust bootstrap binary cannot be resolved' - [[ $baseline_release_path == "$RUST_ACTIVE_BINARY" ]] \ - || die 'active Rust bootstrap binary is not a direct canonical path' - baseline_release_sha=$(sha256sum "$baseline_release_path" | awk '{print $1}') - [[ $baseline_release_sha =~ ^[a-f0-9]{64}$ ]] \ - || die 'active Rust bootstrap binary digest is invalid' - fi - [[ $candidate_sha != "$baseline_release_sha" ]] || \ - die 'candidate digest matches the active Rust release' - ;; - *) die 'active reference collector ExecStart is not an approved baseline' ;; -esac baseline_health_start_required=false baseline_runtime_stability_required=true baseline_degraded=false -if [[ $baseline_mode == legacy_python ]]; then - baseline_health_start_required=$LEGACY_HEALTH_START_REQUIRED - baseline_runtime_stability_required=$LEGACY_RUNTIME_STABILITY_REQUIRED -elif [[ $baseline_mode == rust_bootstrap ]]; then - ! verify_fresh_baseline_health "$LEGACY_SPOOL/health.json" "$RUST_HEALTH_POLICY" \ - || die 'healthy unregistered Rust baseline must be adopted before a normal release gate' +baseline_recovery=false +recovery_json=$(jq -c '.recovery // null' \ + "$MONDAY_POLYMARKET_GATE_INVOCATION_DIR/request.json") \ + || die 'Gate request has no valid recovery binding' +legacy_pid=0 +legacy_restarts=0 +legacy_invocation_id= +if [[ $recovery_json != null ]]; then + baseline_mode=rust_bootstrap + baseline_label='contained Rust bootstrap' + baseline_recovery=true + baseline_runtime_stability_required=false baseline_degraded=true + baseline_release_path=$(jq -er '.baseline.binary_path' <<<"$recovery_json") + baseline_release_sha=$(jq -er '.baseline.binary_sha256' <<<"$recovery_json") +else + baseline_exec=$(effective_exec_argv "$LEGACY_UNIT") || \ + die 'active reference collector has no verifiable ExecStart' + case "$baseline_exec" in + "$LEGACY_EXEC") + baseline_mode=legacy_python + baseline_label=Python + ;; + "$RUST_PRODUCTION_EXEC") + if [[ -L $RUST_ACTIVE_BINARY ]]; then + baseline_mode=rust_release + baseline_label='Rust production' + baseline_release_path=$(readlink -f -- "$RUST_ACTIVE_BINARY") || \ + die 'active Rust collector symlink cannot be resolved' + [[ $baseline_release_path =~ ^$RELEASE_ROOT/([a-f0-9]{64})/polymarket-raw-ops$ ]] \ + || die 'active Rust collector does not resolve to an immutable release' + baseline_release_sha=${BASH_REMATCH[1]} + else + baseline_mode=rust_bootstrap + baseline_label='Rust bootstrap' + [[ -f $RUST_ACTIVE_BINARY && ! -L $RUST_ACTIVE_BINARY ]] \ + || die 'active Rust collector is neither an immutable release nor a direct bootstrap binary' + baseline_release_path=$(readlink -f -- "$RUST_ACTIVE_BINARY") || \ + die 'active Rust bootstrap binary cannot be resolved' + [[ $baseline_release_path == "$RUST_ACTIVE_BINARY" ]] \ + || die 'active Rust bootstrap binary is not a direct canonical path' + baseline_release_sha=$(sha256sum "$baseline_release_path" | awk '{print $1}') + [[ $baseline_release_sha =~ ^[a-f0-9]{64}$ ]] \ + || die 'active Rust bootstrap binary digest is invalid' + fi + [[ $candidate_sha != "$baseline_release_sha" ]] || \ + die 'candidate digest matches the active Rust release' + ;; + *) die 'active reference collector ExecStart is not an approved baseline' ;; + esac + if [[ $baseline_mode == legacy_python ]]; then + baseline_health_start_required=$LEGACY_HEALTH_START_REQUIRED + baseline_runtime_stability_required=$LEGACY_RUNTIME_STABILITY_REQUIRED + elif [[ $baseline_mode == rust_bootstrap ]]; then + ! verify_fresh_baseline_health "$LEGACY_SPOOL/health.json" "$RUST_HEALTH_POLICY" \ + || die 'healthy unregistered Rust baseline must be adopted before a normal release gate' + baseline_degraded=true + fi + legacy_pid=$(systemctl show --property=MainPID --value "$LEGACY_UNIT") + [[ $legacy_pid =~ ^[1-9][0-9]*$ ]] || die 'active legacy collector has no verifiable MainPID' + legacy_restarts=$(systemctl show --property=NRestarts --value "$LEGACY_UNIT") + [[ $legacy_restarts =~ ^[0-9]+$ ]] \ + || die 'active legacy collector has no verifiable restart counter' + legacy_invocation_id=$(systemctl show --property=InvocationID --value "$LEGACY_UNIT") + [[ $legacy_invocation_id =~ ^[a-f0-9]{32}$ ]] \ + || die 'active legacy collector has no verifiable systemd invocation ID' +fi +if [[ $baseline_recovery == true ]]; then + verify_recovery_admission "$recovery_json" "$candidate_sha" "$source_revision" \ + || die 'contained recovery probe is stale, from the future, or does not bind the candidate' fi -legacy_pid=$(systemctl show --property=MainPID --value "$LEGACY_UNIT") -[[ $legacy_pid =~ ^[1-9][0-9]*$ ]] || die 'active legacy collector has no verifiable MainPID' -legacy_restarts=$(systemctl show --property=NRestarts --value "$LEGACY_UNIT") -[[ $legacy_restarts =~ ^[0-9]+$ ]] \ - || die 'active legacy collector has no verifiable restart counter' -legacy_invocation_id=$(systemctl show --property=InvocationID --value "$LEGACY_UNIT") -[[ $legacy_invocation_id =~ ^[a-f0-9]{32}$ ]] \ - || die 'active legacy collector has no verifiable systemd invocation ID' verify_baseline_identity \ - || die 'active reference collector identity or restart counter is not exact' + || die 'reference collector recovery/baseline identity is not exact' ! baseline_health_requires_continuous_freshness "$baseline_mode" \ || verify_fresh_baseline_health "$LEGACY_SPOOL/health.json" \ || die 'active Rust collector health is not fresh and fail-closed clean' @@ -1835,22 +1929,41 @@ canonical_uploaded_segments=$(jq -er \ verify_baseline_identity \ || die 'baseline collector identity changed while parity or OSS readback was running' -baseline_proc_exe='' -if [[ $baseline_mode == rust_release || $baseline_mode == rust_bootstrap ]]; then - baseline_proc_exe=$(readlink -f -- "/proc/$legacy_pid/exe") \ - || die 'could not capture the production Rust executable identity' -fi verify_current_oss_config -legacy_exec_argv=$(effective_exec_argv "$LEGACY_UNIT") \ - || die 'could not capture the effective legacy ExecStart' -legacy_cmdline=$(proc_cmdline "$legacy_pid") \ - || die 'could not capture the exact legacy command line' -legacy_cmdline_argv=${legacy_cmdline% } -legacy_cmdline_sha=$(printf '%s' "$legacy_cmdline_argv" | sha256sum | awk '{print $1}') -legacy_fragment_path=$(systemctl show --property=FragmentPath --value "$LEGACY_UNIT") -legacy_drop_ins=$(systemctl show --property=DropInPaths --value "$LEGACY_UNIT") -legacy_drop_ins_json=$(jq -cn --arg value "$legacy_drop_ins" \ - '$value | split(" ") | map(select(length > 0))') +legacy_runtime_json=null +recovery_evidence=null +if [[ $baseline_recovery == true ]]; then + recovery_evidence=$recovery_json +else + baseline_proc_exe='' + if [[ $baseline_mode == rust_release || $baseline_mode == rust_bootstrap ]]; then + baseline_proc_exe=$(readlink -f -- "/proc/$legacy_pid/exe") \ + || die 'could not capture the production Rust executable identity' + fi + legacy_exec_argv=$(effective_exec_argv "$LEGACY_UNIT") \ + || die 'could not capture the effective legacy ExecStart' + legacy_cmdline=$(proc_cmdline "$legacy_pid") \ + || die 'could not capture the exact legacy command line' + legacy_cmdline_argv=${legacy_cmdline% } + legacy_cmdline_sha=$(printf '%s' "$legacy_cmdline_argv" | sha256sum | awk '{print $1}') + legacy_fragment_path=$(systemctl show --property=FragmentPath --value "$LEGACY_UNIT") + legacy_drop_ins=$(systemctl show --property=DropInPaths --value "$LEGACY_UNIT") + legacy_drop_ins_json=$(jq -cn --arg value "$legacy_drop_ins" \ + '$value | split(" ") | map(select(length > 0))') + legacy_runtime_json=$(jq -cn --arg exec "$legacy_exec_argv" \ + --arg cmdline "$legacy_cmdline_argv" --arg cmdline_sha "$legacy_cmdline_sha" \ + --arg fragment "$legacy_fragment_path" --argjson drop_ins "$legacy_drop_ins_json" \ + --argjson pid "$legacy_pid" --argjson restarts "$legacy_restarts" \ + --arg invocation "$legacy_invocation_id" --arg path "$baseline_release_path" \ + --arg sha "$baseline_release_sha" --arg proc_exe "$baseline_proc_exe" \ + --arg mode "$baseline_mode" ' + {exec_start:$exec,cmdline:$cmdline,cmdline_sha256:$cmdline_sha, + fragment_path:$fragment,drop_in_paths:$drop_ins,main_pid:$pid, + restarts:$restarts,invocation_id:$invocation} + + (if $mode == "rust_release" or $mode == "rust_bootstrap" then + {release_path:$path,proc_exe:$proc_exe,release_sha256:$sha} + else {} end)') +fi completed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) production_eligible=true [[ $test_only == false ]] || production_eligible=false @@ -1866,16 +1979,7 @@ jq \ --arg started_at "$started_at" \ --arg completed_at "$completed_at" \ --arg baseline_mode "$baseline_mode" \ - --arg legacy_exec "$legacy_exec_argv" \ - --arg legacy_cmdline "$legacy_cmdline_argv" \ - --arg legacy_cmdline_sha256 "$legacy_cmdline_sha" \ - --arg legacy_invocation_id "$legacy_invocation_id" \ - --arg legacy_fragment_path "$legacy_fragment_path" \ - --argjson legacy_drop_in_paths "$legacy_drop_ins_json" \ - --argjson legacy_pid "$legacy_pid" \ - --argjson legacy_restarts "$legacy_restarts" \ - --arg baseline_release_path "$baseline_release_path" \ - --arg baseline_release_sha256 "$baseline_release_sha" --arg baseline_proc_exe "$baseline_proc_exe" \ + --argjson recovery "$recovery_evidence" --argjson legacy_runtime "$legacy_runtime_json" \ --arg shadow_exec "$shadow_exec_argv" \ --arg shadow_cmdline "$shadow_cmdline_argv" \ --arg shadow_invocation_id "$shadow_invocation_id" \ @@ -1943,14 +2047,8 @@ jq \ baseline_health_completion_file_identity: $baseline_health_completion_file_identity, real_market_preflight:$real_market_preflight, - legacy_runtime:({exec_start:$legacy_exec,cmdline:$legacy_cmdline, - cmdline_sha256:$legacy_cmdline_sha256, - fragment_path:$legacy_fragment_path,drop_in_paths:$legacy_drop_in_paths, - main_pid:$legacy_pid,restarts:$legacy_restarts, - invocation_id:$legacy_invocation_id} - + if $baseline_mode == "rust_release" or $baseline_mode == "rust_bootstrap" then - {release_path:$baseline_release_path,proc_exe:$baseline_proc_exe, - release_sha256:$baseline_release_sha256} else {} end), + recovery:$recovery, + legacy_runtime:$legacy_runtime, shadow_runtime:{exec_start:$shadow_exec,cmdline:$shadow_cmdline, fragment_path:$shadow_fragment_path,drop_in_paths:$shadow_drop_in_paths, main_pid:$shadow_pid,restarts:$shadow_restarts, diff --git a/deployment/aliyun/polymarket-shadow-gate-policy.jq b/deployment/aliyun/polymarket-shadow-gate-policy.jq index 41b41d23..44e57974 100644 --- a/deployment/aliyun/polymarket-shadow-gate-policy.jq +++ b/deployment/aliyun/polymarket-shadow-gate-policy.jq @@ -61,6 +61,38 @@ def legacy_health_snapshot($allow_bounded_rate_limits): and .stale_settlement_markets == [] and (.overdue_unresolved_markets | type == "array" and all(.[]; type == "string" and length > 0)); +def contained_bootstrap_recovery($candidate; $source): + .mode == "gamma_tagged_500" + and (.candidate_probe.schema + == "monday.polymarket_gamma_tagged_500_recovery_probe.v1") + and .candidate_probe.candidate_sha256 == $candidate + and .candidate_probe.source_revision == $source + and (.candidate_probe.sha256 | sha256) + and (.candidate_probe.observed_at | utc_iso8601_unix | type == "number") + and .candidate_probe.gamma.tagged_closed + == {query:"closed=true&tag_id=21",attempts:3,http_status:500} + and .candidate_probe.gamma.untagged_closed + == {query:"closed=true",attempts:3,http_status:200} + and .candidate_probe.candidate_once.exit_status == 0 + and (.candidate_probe.candidate_once.duration_seconds | positive_integer and . <= 180) + and .candidate_probe.candidate_once.health_updated_at + == .candidate_probe.observed_at + and .baseline.active_state == "inactive" + and .baseline.main_pid == 0 + and .baseline.exec_start + == "/opt/monday/bin/polymarket-raw-ops collect-reference --max-trade-polls-per-cycle 200" + and .baseline.fragment_path + == "/etc/systemd/system/polymarket-reference-collector.service" + and .baseline.drop_in_paths == [] + and (.baseline.restarts | nonnegative_integer) + and (.baseline.invocation_id | type == "string" and test("^[a-f0-9]{32}$")) + and .baseline.binary_path == "/opt/monday/bin/polymarket-raw-ops" + and (.baseline.binary_sha256 | sha256) + and .baseline.binary_sha256 != $candidate; +def recovery_matches_gate: + . as $gate + | ($gate.recovery | contained_bootstrap_recovery( + $gate.candidate_sha256; $gate.deployment_source_revision)); .schema == "monday.polymarket_shadow_gate.v1" and (.candidate_sha256 | sha256) @@ -139,6 +171,7 @@ and .passed == true and ( ( .baseline_mode == "legacy_python" + and .recovery == null and .baseline_runtime_stability_required == true and ( .baseline_health_start_required == false @@ -196,6 +229,7 @@ and ( or ( .baseline_mode == "rust_release" + and .recovery == null and .baseline_health_start_required == false and .baseline_runtime_stability_required == true and .baseline_health_completion_required == false @@ -219,6 +253,7 @@ and ( or ( .baseline_mode == "rust_bootstrap" + and .recovery == null and .baseline_degraded == true and .baseline_health_start_required == false and .baseline_runtime_stability_required == true @@ -239,6 +274,24 @@ and ( and .legacy_runtime.release_path == "/opt/monday/bin/polymarket-raw-ops" and .legacy_runtime.proc_exe == .legacy_runtime.release_path ) + or + ( + .baseline_mode == "rust_bootstrap" + and .baseline_degraded == true + and .baseline_health_start_required == false + and .baseline_runtime_stability_required == false + and .baseline_health_completion_required == false + and .baseline_health_snapshot == null + and .baseline_health_completion_snapshot == null + and .baseline_health_start_success_unix == null + and .baseline_health_cutoff_unix == null + and .baseline_health_start_written_at_unix == null + and .baseline_health_completion_written_at_unix == null + and .baseline_health_start_file_identity == null + and .baseline_health_completion_file_identity == null + and .legacy_runtime == null + and recovery_matches_gate + ) ) and ( .shadow_runtime.exec_start == ( @@ -290,7 +343,8 @@ and .checks.real_market_segment_preflight == true and (.comparison_mode == "legacy_overlap" or ( .comparison_mode == "rust_self" and (.baseline_mode == "legacy_python" or .baseline_mode == "rust_bootstrap") - and .baseline_runtime_stability_required == true + and (.baseline_runtime_stability_required == true + or recovery_matches_gate) )) and (.metrics.oss_uploaded_segments | positive_integer) and (.metrics.oss_canonical_uploaded_segments | positive_integer) diff --git a/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh b/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh index 2e54226b..29b3c3df 100755 --- a/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh +++ b/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh @@ -121,7 +121,7 @@ supervisor_source=$(printf 'b%.0s' {1..40}) supervisor_invocation=$(printf '1%.0s' {1..32}) mkdir -p "$supervisor_fake_bin" "$supervisor_control_dir" \ "$supervisor_root/etc/systemd/system" \ - "$supervisor_root/run/monday" \ + "$supervisor_root/run/monday" "$supervisor_root/opt/monday/bin" \ "$supervisor_root/data/monday/evidence" cp "$GATE_CONTROL" "$supervisor_control" cp "$GATE_UNIT" "$supervisor_control_dir/${GATE_UNIT##*/}" @@ -138,10 +138,32 @@ printf '#!/usr/bin/env bash\nexit 0\n' >"$supervisor_candidate" chmod 0755 "$supervisor_candidate" supervisor_candidate_sha=$(sha256sum "$supervisor_candidate" | awk '{print $1}') supervisor_unit="polymarket-raw-ops-gate@${supervisor_candidate_sha}.service" +supervisor_baseline="$supervisor_root/opt/monday/bin/polymarket-raw-ops" +printf '#!/usr/bin/env bash\nexit 1\n' >"$supervisor_baseline" +chmod 0755 "$supervisor_baseline" +supervisor_baseline_sha=$(sha256sum "$supervisor_baseline" | awk '{print $1}') +supervisor_probe_root="$supervisor_root/data/monday/evidence/polymarket-candidate-probes/$supervisor_candidate_sha" +mkdir -p "$supervisor_probe_root" +supervisor_probe="$supervisor_probe_root/gamma-tagged-500.json" +jq -n --arg candidate "$supervisor_candidate_sha" --arg source "$supervisor_source" \ + --arg observed_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" ' + {schema:"monday.polymarket_gamma_tagged_500_recovery_probe.v1", + candidate_sha256:$candidate,source_revision:$source,observed_at:$observed_at, + gamma:{tagged_closed:{query:"closed=true&tag_id=21",attempts:3,http_status:500}, + untagged_closed:{query:"closed=true",attempts:3,http_status:200}}, + candidate_once:{exit_status:0,duration_seconds:23,health_updated_at:$observed_at}} +' >"$supervisor_probe" mkdir "$supervisor_state" printf 'inactive\n' >"$supervisor_state/active" printf '%s\n' "$supervisor_invocation" >"$supervisor_state/invocation" printf 'inactive\n' >"$supervisor_state/shadow" +printf 'inactive\n' >"$supervisor_state/baseline-active" +printf '0\n' >"$supervisor_state/baseline-main-pid" +printf '2\n' >"$supervisor_state/baseline-restarts" +printf '%s\n' "$(printf 'a%.0s' {1..32})" >"$supervisor_state/baseline-invocation" +printf '%s\n' '/opt/monday/bin/polymarket-raw-ops collect-reference --max-trade-polls-per-cycle 200' \ + >"$supervisor_state/baseline-exec" +printf 'inactive\n' >"$supervisor_state/uploader-active" : >"$supervisor_calls" : >"$supervisor_gate_calls" @@ -154,6 +176,20 @@ write_state() { printf '%s\n' "$2" >"$FAKE_SYSTEMCTL_STATE/$1"; } printf '%s\n' "$*" >>"$FAKE_SYSTEMCTL_CALLS" case "${1:-}" in + is-active) + [[ ${2:-} == --quiet && $# -eq 3 ]] || exit 2 + unit=${3:-} + case "$unit" in + polymarket-reference-collector.service) + [[ $(read_state baseline-active) == active ]] + ;; + polymarket-reference-upload.service|polymarket-reference-upload.timer|\ + polymarket-market-tape-upload.service|polymarket-market-tape-upload.timer) + [[ $(read_state uploader-active) == active ]] + ;; + *) exit 2 ;; + esac + ;; daemon-reload) [[ $# -eq 1 ]] || exit 2 ;; @@ -198,19 +234,48 @@ case "${1:-}" in ActiveState) if [[ $unit == polymarket-reference-collector-shadow@* ]]; then read_state shadow + elif [[ $unit == polymarket-reference-collector.service ]]; then + read_state baseline-active + elif [[ $unit == polymarket-reference-upload.service \ + || $unit == polymarket-reference-upload.timer \ + || $unit == polymarket-market-tape-upload.service \ + || $unit == polymarket-market-tape-upload.timer ]]; then + read_state uploader-active else read_state active fi ;; - InvocationID) read_state invocation ;; + InvocationID) + if [[ $unit == polymarket-reference-collector.service ]]; then + read_state baseline-invocation + else + read_state invocation + fi + ;; MainPID) if [[ $unit == polymarket-reference-collector-shadow@* ]]; then [[ $(read_state shadow) == active ]] && printf '5252\n' || printf '0\n' + elif [[ $unit == polymarket-reference-collector.service ]]; then + read_state baseline-main-pid else [[ $(read_state active) == active ]] && printf '4242\n' || printf '0\n' fi ;; - FragmentPath) printf '/etc/systemd/system/polymarket-raw-ops-gate@.service\n' ;; + NRestarts) + [[ $unit == polymarket-reference-collector.service ]] || exit 2 + read_state baseline-restarts + ;; + ExecStart) + [[ $unit == polymarket-reference-collector.service ]] || exit 2 + printf 'argv[]=%s;\n' "$(read_state baseline-exec)" + ;; + FragmentPath) + if [[ $unit == polymarket-reference-collector.service ]]; then + printf '/etc/systemd/system/polymarket-reference-collector.service\n' + else + printf '/etc/systemd/system/polymarket-raw-ops-gate@.service\n' + fi + ;; DropInPaths) printf '\n' ;; *) exit 2 ;; esac @@ -365,16 +430,178 @@ ln -s "$supervisor_control_dir/${GATE_UNIT##*/}" "$installed_supervisor_unit" reject gate_control install rm "$installed_supervisor_unit" gate_control install >/dev/null + +# A recovery admission is not a generic gate bypass: it starts only with the +# stopped direct bootstrap identity, a fresh exact candidate probe, and every +# uploader/timer contained. +supervisor_recovery_gate_invocation=$(printf '9%.0s' {1..32}) +set_supervisor_state invocation "$supervisor_recovery_gate_invocation" +set_supervisor_state active inactive +set_supervisor_state baseline-active inactive +set_supervisor_state uploader-active inactive +gate_control recover "$supervisor_candidate" "$supervisor_candidate_sha" \ + "$supervisor_source" "$supervisor_probe" >"$supervisor_tmp/recover-start.json" +jq -e --arg invocation "$supervisor_recovery_gate_invocation" ' + .phase == "running" and .systemd_invocation_id == $invocation +' "$supervisor_tmp/recover-start.json" >/dev/null +supervisor_recovery_request="$supervisor_root/data/monday/evidence/polymarket-gate-jobs/$supervisor_candidate_sha/$supervisor_recovery_gate_invocation/request.json" +jq -e --arg candidate "$supervisor_candidate_sha" --arg source "$supervisor_source" \ + --arg baseline "$supervisor_baseline_sha" ' + .recovery.mode == "gamma_tagged_500" + and .recovery.candidate_probe.candidate_sha256 == $candidate + and .recovery.candidate_probe.source_revision == $source + and .recovery.baseline.binary_sha256 == $baseline + and .recovery.baseline.active_state == "inactive" + and .recovery.baseline.main_pid == 0 +' "$supervisor_recovery_request" >/dev/null +gate_control cancel "$supervisor_candidate_sha" "$supervisor_recovery_gate_invocation" >/dev/null + +set_supervisor_state active inactive +set_supervisor_state baseline-active active +reject gate_control recover "$supervisor_candidate" "$supervisor_candidate_sha" \ + "$supervisor_source" "$supervisor_probe" +set_supervisor_state baseline-active inactive +set_supervisor_state uploader-active active +reject gate_control recover "$supervisor_candidate" "$supervisor_candidate_sha" \ + "$supervisor_source" "$supervisor_probe" +set_supervisor_state uploader-active inactive +set_supervisor_state baseline-exec '/opt/monday/bin/not-polymarket-raw-ops collect-reference' +reject gate_control recover "$supervisor_candidate" "$supervisor_candidate_sha" \ + "$supervisor_source" "$supervisor_probe" +set_supervisor_state baseline-exec '/opt/monday/bin/polymarket-raw-ops collect-reference --max-trade-polls-per-cycle 200' +jq '.observed_at = "1970-01-01T00:00:00Z"' "$supervisor_probe" >"$supervisor_probe.stale" +reject gate_control recover "$supervisor_candidate" "$supervisor_candidate_sha" \ + "$supervisor_source" "$supervisor_probe.stale" +reject gate_control recover "$supervisor_candidate" "$supervisor_candidate_sha" \ + "$supervisor_source" "$supervisor_probe_root/missing.json" +jq --arg wrong_candidate "$(printf 'f%.0s' {1..64})" \ + '.candidate_sha256 = $wrong_candidate' "$supervisor_probe" >"$supervisor_probe.wrong-candidate" +reject gate_control recover "$supervisor_candidate" "$supervisor_candidate_sha" \ + "$supervisor_source" "$supervisor_probe.wrong-candidate" +recovery_mutations=$(grep -E \ + '^(start|stop|restart|enable|disable) (polymarket-reference-collector[.]service|polymarket-reference-upload[.](service|timer)|polymarket-market-tape-upload[.](service|timer))$' \ + "$supervisor_calls" || true) +if [[ -n $recovery_mutations ]]; then + printf 'recovery admission mutated the contained baseline: %s\n' \ + "$recovery_mutations" >&2 + exit 1 +fi + +# A fresh probe admits once; its binding remains valid through the 900-second Gate. +recovery_binding_contract="$supervisor_tmp/recovery-binding-contract.sh" +sed -n '/^verify_recovery_binding() {$/,/^}$/p' "$GATE" >"$recovery_binding_contract" +sed -n '/^verify_recovery_admission() {$/,/^}$/p' "$GATE" >>"$recovery_binding_contract" +sed -n '/^verify_contained_recovery_baseline() {$/,/^}$/p' "$GATE" >>"$recovery_binding_contract" +( + set -euo pipefail + RUST_ACTIVE_BINARY="$supervisor_baseline" LEGACY_UNIT=polymarket-reference-collector.service + supervisor_baseline_invocation=$(<"$supervisor_state/baseline-invocation") + recovery_active_state=inactive recovery_fragment=/etc/systemd/system/polymarket-reference-collector.service + recovery_drop_ins='' recovery_exec='/opt/monday/bin/polymarket-raw-ops collect-reference --max-trade-polls-per-cycle 200' + recovery_restarts=2 recovery_invocation=$supervisor_baseline_invocation recovery_binary_sha=$supervisor_baseline_sha + recovery_binary_secure=true + recovery=$(jq -c .recovery "$supervisor_recovery_request") + recovery_observed_epoch=1000 recovery_now=$recovery_observed_epoch + date() { + [[ $1 == -u ]] || { command date "$@"; return; } + case "$2" in -d) printf '%s\n' "$recovery_observed_epoch" ;; +%s) printf '%s\n' "$recovery_now" ;; *) command date "$@" ;; esac + } + systemctl() { + case "$*" in + *ActiveState*) printf '%s\n' "$recovery_active_state" ;; *MainPID*) printf '0\n' ;; + *FragmentPath*) printf '%s\n' "$recovery_fragment" ;; + *DropInPaths*) printf '%s\n' "$recovery_drop_ins" ;; *NRestarts*) printf '%s\n' "$recovery_restarts" ;; + *InvocationID*) printf '%s\n' "$recovery_invocation" ;; *) return 1 ;; + esac + } + effective_exec_argv() { printf '%s\n' "$recovery_exec"; } + secure_control_file() { [[ $recovery_binary_secure == true ]]; } + sha256sum() { printf '%s %s\n' "$recovery_binary_sha" "$1"; } + # shellcheck source=/dev/null + source "$recovery_binding_contract" + verify_recovery_admission "$recovery" "$supervisor_candidate_sha" "$supervisor_source" + recovery_now=$((recovery_observed_epoch + 901)) + if verify_recovery_admission "$recovery" "$supervisor_candidate_sha" "$supervisor_source"; then + printf 'recovery admission accepted a stale probe\n' >&2; exit 1 + fi + verify_contained_recovery_baseline "$recovery" "$supervisor_candidate_sha" "$supervisor_source" || { + printf 'recovery identity check aged a valid Gate probe\n' >&2; exit 1 + } + recovery_binary_secure=false + if verify_contained_recovery_baseline "$recovery" "$supervisor_candidate_sha" "$supervisor_source"; then + printf 'recovery identity check accepted an insecure baseline binary\n' >&2; exit 1 + fi + recovery_binary_secure=true + for recovery_drift in active fragment drop_ins exec restarts invocation binary; do + recovery_active_state=inactive recovery_fragment=/etc/systemd/system/polymarket-reference-collector.service + recovery_drop_ins='' recovery_exec='/opt/monday/bin/polymarket-raw-ops collect-reference --max-trade-polls-per-cycle 200' + recovery_restarts=2 recovery_invocation=$supervisor_baseline_invocation recovery_binary_sha=$supervisor_baseline_sha + case "$recovery_drift" in + active) recovery_active_state=active ;; fragment) recovery_fragment=/tmp/wrong.service ;; + drop_ins) recovery_drop_ins=/etc/systemd/system/recovery.conf ;; + exec) recovery_exec='/opt/monday/bin/not-polymarket-raw-ops collect-reference' ;; + restarts) recovery_restarts=3 ;; invocation) recovery_invocation=$(printf 'b%.0s' {1..32}) ;; + binary) recovery_binary_sha=$(printf 'c%.0s' {1..64}) ;; + esac + if verify_contained_recovery_baseline "$recovery" "$supervisor_candidate_sha" "$supervisor_source"; then + printf 'recovery identity check accepted %s drift\n' "$recovery_drift" >&2; exit 1 + fi + done +) +: >"$supervisor_calls" + +# Normal Gate requests intentionally omit recovery metadata. Both consumers +# must preserve literal null rather than turning that optional value into a +# jq failure. +normal_optional_gate_contract="$supervisor_tmp/normal-optional-gate.sh" +sed -n '/^recovery_json=$(jq -c.*\.recovery \/\/ null/,/^legacy_pid=0$/p' \ + "$GATE" >"$normal_optional_gate_contract" +normal_optional_cutover_contract="$supervisor_tmp/normal-optional-cutover.sh" +sed -n '/^recovery_json=$(jq -c.*\.recovery \/\/ null/,/^contained_recovery=false$/p' \ + "$CUTOVER" >"$normal_optional_cutover_contract" +[[ -s $normal_optional_gate_contract && -s $normal_optional_cutover_contract ]] || { + printf 'optional recovery consumers are missing\n' >&2 + exit 1 +} +normal_optional_dir="$supervisor_tmp/normal-optional" +mkdir "$normal_optional_dir" +printf '%s\n' '{"schema":"monday.polymarket_gate_request.v1"}' \ + >"$normal_optional_dir/request.json" +if ! ( + set -euo pipefail + MONDAY_POLYMARKET_GATE_INVOCATION_DIR="$normal_optional_dir" + die() { exit 77; } + # shellcheck source=/dev/null + source "$normal_optional_gate_contract" + [[ $recovery_json == null && $legacy_pid == 0 ]] +); then + printf 'normal Gate request rejected an absent recovery binding\n' >&2 + exit 1 +fi +if ! ( + set -euo pipefail + gate_json="$normal_optional_dir/request.json" + die() { exit 77; } + # shellcheck source=/dev/null + source "$normal_optional_cutover_contract" + [[ $recovery_json == null && $contained_recovery == false ]] +); then + printf 'normal cutover rejected an absent recovery binding\n' >&2 + exit 1 +fi + +supervisor_starts_before_normal=$(grep -Fxc "start $supervisor_unit" "$supervisor_calls" || true) start_supervisor "$supervisor_invocation" >"$supervisor_tmp/start.json" jq -e --arg unit "$supervisor_unit" --arg invocation "$supervisor_invocation" ' .unit == $unit and .systemd_invocation_id == $invocation and .phase == "running" and .terminal_state == null' \ "$supervisor_tmp/start.json" >/dev/null -[[ $(grep -Fxc "start $supervisor_unit" "$supervisor_calls") == 1 ]] +supervisor_starts_after_normal=$(grep -Fxc "start $supervisor_unit" "$supervisor_calls") +[[ $supervisor_starts_after_normal -eq $((supervisor_starts_before_normal + 1)) ]] assert_running_status "$supervisor_invocation" reject gate_control start "$supervisor_candidate" "$supervisor_candidate_sha" \ "$supervisor_source" -[[ $(grep -Fxc "start $supervisor_unit" "$supervisor_calls") == 1 ]] +[[ $(grep -Fxc "start $supervisor_unit" "$supervisor_calls") == "$supervisor_starts_after_normal" ]] if env "${gate_control_env[@]}" INVOCATION_ID="$supervisor_invocation" \ FAKE_GATE_EXIT=17 "$supervisor_control" run "$supervisor_candidate_sha" \ >/dev/null 2>&1; then @@ -1403,8 +1630,8 @@ exercise_rust_bootstrap_identity() ( exercise_rust_bootstrap_identity bootstrap_baseline_selection="$tmp_dir/bootstrap-baseline-selection.sh" -sed -n '/^baseline_exec=$(effective_exec_argv/,/^baseline_health_start_required=false$/p' \ - "$GATE" >"$bootstrap_baseline_selection" +sed -n '/^ baseline_exec=$(effective_exec_argv/,/^ esac$/p' "$GATE" \ + | sed 's/^ //' >"$bootstrap_baseline_selection" ( set -euo pipefail LEGACY_UNIT=polymarket-reference-collector.service @@ -1424,8 +1651,8 @@ sed -n '/^baseline_exec=$(effective_exec_argv/,/^baseline_health_start_required= ) bootstrap_health_admission="$tmp_dir/bootstrap-health-admission.sh" -sed -n '/^baseline_health_start_required=false$/,/^fi$/p' "$GATE" \ - >"$bootstrap_health_admission" +sed -n '/^ elif \[\[ \$baseline_mode == rust_bootstrap \]\]; then$/,/^ fi$/p' \ + "$GATE" | sed '1d; $d; s/^ //' >"$bootstrap_health_admission" ( set -euo pipefail baseline_mode=rust_bootstrap @@ -1789,7 +2016,7 @@ cutover_state_handoff_line=$(grep -n \ cutover_apply_handoff_line=$(grep -n \ '^apply_legacy_state_handoff "$LEGACY_STATE" "$evidence_dir"' \ "$CUTOVER" | cut -d: -f1) -cutover_stop_collector_line=$(grep -n '^systemctl stop "$COLLECTOR_UNIT"$' \ +cutover_stop_collector_line=$(grep -n '^[[:space:]]*systemctl stop "$COLLECTOR_UNIT"$' \ "$CUTOVER" | tail -1 | cut -d: -f1) cutover_start_rust_line=$(grep -n '^systemctl restart "$COLLECTOR_UNIT"$' \ "$CUTOVER" | tail -1 | cut -d: -f1) @@ -3814,6 +4041,65 @@ jq -e -f "$POLICY" "$tmp_dir/rust-bootstrap-gate.json" >/dev/null || { printf 'gate policy rejected a bounded Rust bootstrap recovery\n' >&2 exit 1 } +jq ' + .baseline_mode = "rust_bootstrap" + | .baseline_degraded = true + | .baseline_health_start_required = false + | .baseline_runtime_stability_required = false + | .baseline_health_completion_required = false + | .baseline_health_snapshot = null + | .baseline_health_completion_snapshot = null + | .baseline_health_start_success_unix = null + | .baseline_health_cutoff_unix = null + | .baseline_health_start_written_at_unix = null + | .baseline_health_completion_written_at_unix = null + | .baseline_health_start_file_identity = null + | .baseline_health_completion_file_identity = null + | .legacy_runtime = null + | .recovery = { + mode:"gamma_tagged_500", + baseline:{ + active_state:"inactive",main_pid:0, + exec_start:"/opt/monday/bin/polymarket-raw-ops collect-reference --max-trade-polls-per-cycle 200", + fragment_path:"/etc/systemd/system/polymarket-reference-collector.service", + drop_in_paths:[],restarts:2,invocation_id:("2" * 32), + binary_path:"/opt/monday/bin/polymarket-raw-ops", + binary_sha256:(if .candidate_sha256 == ("0" * 64) then ("1" * 64) else ("0" * 64) end) + }, + candidate_probe:{ + schema:"monday.polymarket_gamma_tagged_500_recovery_probe.v1", + candidate_sha256:.candidate_sha256, + source_revision:.deployment_source_revision,sha256:("3" * 64), + observed_at:"2026-07-15T00:00:01Z", + gamma:{ + tagged_closed:{query:"closed=true&tag_id=21",attempts:3,http_status:500}, + untagged_closed:{query:"closed=true",attempts:3,http_status:200} + }, + candidate_once:{exit_status:0,duration_seconds:23, + health_updated_at:"2026-07-15T00:00:01Z"} + } + } +' "$tmp_dir/rust-bootstrap-gate.json" >"$tmp_dir/contained-recovery-gate.json" +jq -e -f "$POLICY" "$tmp_dir/contained-recovery-gate.json" >/dev/null || { + printf 'gate policy rejected a verified contained bootstrap recovery\n' >&2 + exit 1 +} +while IFS='|' read -r name filter; do + jq "$filter" "$tmp_dir/contained-recovery-gate.json" \ + >"$tmp_dir/contained-recovery-$name.json" + if jq -e -f "$POLICY" "$tmp_dir/contained-recovery-$name.json" >/dev/null; then + printf 'contained recovery policy accepted %s drift\n' "$name" >&2 + exit 1 + fi +done <<'EOF' +candidate_binding|.recovery.candidate_probe.candidate_sha256 = (if .candidate_sha256 == ("0" * 64) then ("1" * 64) else ("0" * 64) end) +source_binding|.recovery.candidate_probe.source_revision = (if .deployment_source_revision == ("0" * 40) then ("1" * 40) else ("0" * 40) end) +baseline_active|.recovery.baseline.active_state = "active" +runtime_stability|.baseline_runtime_stability_required = true +missing_recovery|.recovery = null +baseline_is_candidate|.recovery.baseline.binary_sha256 = .candidate_sha256 +legacy_runtime_present|.legacy_runtime = {exec_start:"/opt/monday/bin/polymarket-raw-ops collect-reference"} +EOF jq '.metrics.legacy_metadata_count = 1' "$tmp_dir/rust-bootstrap-gate.json" \ >"$tmp_dir/rust-bootstrap-metadata-only-gate.json" jq -e -f "$POLICY" "$tmp_dir/rust-bootstrap-metadata-only-gate.json" >/dev/null || { @@ -4677,6 +4963,31 @@ grep -Fq 'NRestarts' "$CUTOVER" [[ $(grep -Fc '[[ $invocation_id == "$expected_invocation_id" ]]' "$CUTOVER") -eq 3 ]] grep -Fq 'invocation_id:$rust_invocation_id' "$CUTOVER" grep -Fq 'verify_shadow_identity' "$GATE" +grep -Fq 'verify_contained_bootstrap_recovery "$recovery_json" "$candidate_sha"' "$CUTOVER" +grep -Fq 'contained recovery rollback would restart a saved baseline unit' "$CUTOVER" +grep -Fq 'contained recovery rollback restarted a collector or uploader' "$CUTOVER" +contained_recovery_guard_line=$(grep -nF \ + 'contained recovery rollback would restart a saved baseline unit' "$CUTOVER" \ + | head -1 | cut -d: -f1 || true) +contained_recovery_health_label_line=$(grep -nF \ + '"pre-contained-recovery-rollback-' \ + "$CUTOVER" | head -1 | cut -d: -f1 || true) +restore_stop_line=$(grep -nF \ + 'systemctl stop "$REFERENCE_UPLOAD_TIMER" "$MARKET_UPLOAD_TIMER"' "$CUTOVER" \ + | head -1 | cut -d: -f1 || true) +[[ $contained_recovery_guard_line =~ ^[1-9][0-9]*$ \ + && $contained_recovery_health_label_line =~ ^[1-9][0-9]*$ \ + && $restore_stop_line =~ ^[1-9][0-9]*$ \ + && $contained_recovery_guard_line -lt $contained_recovery_health_label_line \ + && $contained_recovery_health_label_line -lt $restore_stop_line ]] || { + printf 'contained recovery rollback mutates before validating or clearing health\n' >&2 + exit 1 +} +[[ $(sed -n "$((contained_recovery_health_label_line - 1))p" "$CUTOVER") \ + == *'clear_health_before_restart "$evidence_dir"'* ]] || { + printf 'contained recovery rollback does not clear the candidate health file\n' >&2 + exit 1 +} grep -Fq 'health_advanced=true' "$CUTOVER" grep -Fq 'updated_epoch >= started_epoch' "$CUTOVER" grep -Fq 'gate_legacy_pid' "$CUTOVER" @@ -4716,9 +5027,10 @@ release_move_line=$(grep -n '^ mv "$staging" "$release_dir"$' "$GATE" \ exit 1 } -cutover_stop_line=$(grep -n '^systemctl stop "$COLLECTOR_UNIT"$' "$CUTOVER" | cut -d: -f1) +cutover_stop_line=$(grep -n '^[[:space:]]*systemctl stop "$COLLECTOR_UNIT"$' \ + "$CUTOVER" | tail -1 | cut -d: -f1) cutover_legacy_promotion="$tmp_dir/cutover-legacy-promotion.sh" -sed -n '/^# Cutover depends on the current gate bundle/,/^systemctl stop "$COLLECTOR_UNIT"$/p' \ +sed -n '/^# Cutover depends on the current gate bundle/,/^[[:space:]]*systemctl stop "$COLLECTOR_UNIT"$/p' \ "$CUTOVER" >"$cutover_legacy_promotion" cutover_legacy_promotion_joined="$tmp_dir/cutover-legacy-promotion-joined.sh" join_shell_continuations "$cutover_legacy_promotion" \ @@ -4752,21 +5064,21 @@ if grep -Fq 'verify_fresh_legacy_runtime' "$cutover_legacy_rollback"; then fi legacy_drain_line=$(grep -n '^[[:space:]]*verify_oneshot_success "$REFERENCE_UPLOAD_UNIT"' "$CUTOVER" \ | head -1 | cut -d: -f1) -legacy_cursor_line=$(grep -n '^legacy_stop_cursor=$(journal_cursor "$COLLECTOR_UNIT")' \ +legacy_cursor_line=$(grep -n '^[[:space:]]*legacy_stop_cursor=$(journal_cursor "$COLLECTOR_UNIT")' \ "$CUTOVER" | cut -d: -f1) legacy_final_runtime_line=$(grep -n \ - '^verify_legacy_runtime "$legacy_pid" "$gate_legacy_restarts" "$gate_legacy_invocation_id"' \ + '^[[:space:]]*verify_legacy_runtime "$legacy_pid" "$gate_legacy_restarts" "$gate_legacy_invocation_id"' \ "$CUTOVER" \ | tail -1 | cut -d: -f1) legacy_final_oss_line=$(grep -n \ 'OSS configuration changed during the legacy uploader drain' "$CUTOVER" \ | cut -d: -f1) -legacy_journal_guard_line=$(grep -n '^verify_no_restart_after_cursor' "$CUTOVER" \ +legacy_journal_guard_line=$(grep -n '^[[:space:]]*verify_no_restart_after_cursor' "$CUTOVER" \ | tail -1 | cut -d: -f1) -legacy_stopped_counter_line=$(grep -n '^stopped_legacy_restarts=' "$CUTOVER" \ +legacy_stopped_counter_line=$(grep -n '^[[:space:]]*stopped_legacy_restarts=' "$CUTOVER" \ | cut -d: -f1) legacy_stopped_equality_line=$(grep -n \ - '^\[\[ \$stopped_legacy_restarts == "\$gate_legacy_restarts" \]\]' "$CUTOVER" \ + '^[[:space:]]*\[\[ \$stopped_legacy_restarts == "\$gate_legacy_restarts" \]\]' "$CUTOVER" \ | cut -d: -f1) cutover_clear_line=$(grep -n '^clear_health_before_restart "$evidence_dir" pre-cutover$' \ "$CUTOVER" | cut -d: -f1)