diff --git a/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh b/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh index 9cb62508c..ad2a5b9e7 100755 --- a/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh +++ b/deployment/aliyun/polymarket-raw-ops-shadow-gate.sh @@ -7,27 +7,21 @@ export TZ=UTC readonly REQUIRED_DURATION_SECONDS=900 # The verifier subtracts a 600-second trade maturity lag and requires a -# non-empty event window, so the deployment tail must be strictly longer. +# non-empty event window, so 601 seconds of the observation are retained. readonly PARITY_TAIL_SECONDS=601 -readonly MINIMUM_GATE_SECONDS=$((REQUIRED_DURATION_SECONDS + PARITY_TAIL_SECONDS)) +readonly MINIMUM_GATE_SECONDS=$REQUIRED_DURATION_SECONDS readonly MAX_ACCEPTED_CYCLE_SECONDS=180 readonly INITIAL_HEALTH_GRACE_SECONDS=60 readonly HEALTH_SETTLE_SECONDS=$((MAX_ACCEPTED_CYCLE_SECONDS + INITIAL_HEALTH_GRACE_SECONDS)) readonly MAX_HEALTH_SILENCE_SECONDS=240 -# Legacy full-catalog cycles observed 35–58 minutes; bound admission at 65. -readonly LEGACY_HEALTH_START_WAIT_SECONDS=3900 readonly LEGACY_HEALTH_COMPLETION_REQUIRED=false -readonly LEGACY_HEALTH_START_REQUIRED=false -readonly LEGACY_RUNTIME_STABILITY_REQUIRED=false -# One real-segment upload can spend one 300-second compression timeout, fifteen -# candidate OSS operations, four independent Gate readbacks, and 300 seconds -# of local processing reserve. -readonly REAL_MARKET_PREFLIGHT_BUDGET_SECONDS=6300 +readonly LEGACY_HEALTH_START_REQUIRED=true +readonly LEGACY_RUNTIME_STABILITY_REQUIRED=true +readonly REAL_MARKET_PREFLIGHT_BUDGET_SECONDS=300 readonly LEGACY_RUNTIME_MAX_SECONDS=21600 readonly LEGACY_RUNTIME_RESERVE_SECONDS=60 readonly SAMPLE_SECONDS=30 readonly PARITY_CUTOFF_LAG_SECONDS=60 -readonly SETTLEMENT_EVENT_LOOKBACK_SECONDS=900 readonly LEGACY_UNIT=polymarket-reference-collector.service readonly LEGACY_EXEC='/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py' readonly RUST_PRODUCTION_EXEC='/opt/monday/bin/polymarket-raw-ops collect-reference' @@ -76,7 +70,7 @@ usage() { printf '%s\n' \ 'Usage: polymarket-raw-ops-shadow-gate.sh ' \ '' \ - 'A production-eligible gate observes for 900 seconds plus a 601-second current-hour parity tail.' + 'A production-eligible gate observes for 900 seconds total, including a 601-second parity interval.' } valid_parity_window() { @@ -86,12 +80,9 @@ valid_parity_window() { } bounded_parity_window_start() { - local gate_started_at=$1 common_cutoff=$2 allow_short=$3 lookback_seconds=$4 - local parity_started_at settlement_safe_started_at - parity_started_at=$((common_cutoff - common_cutoff % 3600)) - settlement_safe_started_at=$((gate_started_at + lookback_seconds)) - ((parity_started_at >= settlement_safe_started_at)) \ - || parity_started_at=$settlement_safe_started_at + local gate_started_at=$1 common_cutoff=$2 allow_short=$3 + local parity_started_at + parity_started_at=$gate_started_at if [[ $allow_short == true ]] && ((parity_started_at >= common_cutoff)); then ((common_cutoff > 0)) || return 1 parity_started_at=$((common_cutoff - 1)) @@ -522,28 +513,6 @@ fresh_legacy_health_observation() { file_identity:$file_identity}' } -wait_for_fresh_legacy_health_observation() { - local health=$1 policy=$2 observation deadline remaining - deadline=$((SECONDS + LEGACY_HEALTH_START_WAIT_SECONDS)) - while true; do - ((SECONDS < deadline)) || return 1 - verify_baseline_identity || return 1 - if observation=$(fresh_legacy_health_observation "$health" "$policy"); then - ((SECONDS < deadline)) || return 1 - verify_baseline_identity || return 1 - ((SECONDS < deadline)) || return 1 - printf '%s\n' "$observation" - return 0 - fi - remaining=$((deadline - SECONDS)) - ((remaining > 0)) || return 1 - if ((remaining > 5)); then - remaining=5 - fi - sleep "$remaining" - done -} - verify_fresh_baseline_health() { fresh_baseline_health_snapshot "$@" >/dev/null } @@ -942,7 +911,7 @@ real_market_segment_preflight() { } run_budgeted_real_market_preflight() { - local legacy_runtime_budget_required observation + local legacy_runtime_budget_required observation preflight_output if [[ $baseline_mode == legacy_python \ && $LEGACY_RUNTIME_STABILITY_REQUIRED == true ]]; then verify_baseline_identity || { @@ -950,8 +919,10 @@ run_budgeted_real_market_preflight() { return 1 } legacy_runtime_budget_required=$((REAL_MARKET_PREFLIGHT_BUDGET_SECONDS \ - + LEGACY_HEALTH_START_WAIT_SECONDS + gate_seconds \ - + PARITY_CUTOFF_LAG_SECONDS + LEGACY_RUNTIME_RESERVE_SECONDS)) + + gate_seconds \ + + PARITY_CUTOFF_LAG_SECONDS \ + + zstd_timeout_seconds + oss_copy_timeout_seconds \ + + LEGACY_RUNTIME_RESERVE_SECONDS)) if observation=$(legacy_runtime_budget_observation \ "$legacy_runtime_budget_required"); then : @@ -968,7 +939,7 @@ run_budgeted_real_market_preflight() { return 1 } fi - timeout --signal=KILL "$REAL_MARKET_PREFLIGHT_BUDGET_SECONDS" env \ + preflight_output=$(timeout --signal=KILL "$REAL_MARKET_PREFLIGHT_BUDGET_SECONDS" env \ "candidate_sha=$candidate_sha" "run_id=$run_id" \ "release_binary=$release_binary" "oss_bucket=$oss_bucket" \ "oss_endpoint=$oss_endpoint" "oss_region=$oss_region" \ @@ -979,7 +950,15 @@ run_budgeted_real_market_preflight() { "deployment_bundle_sha=$deployment_bundle_sha" \ "release_manifest_sha=$release_manifest_sha" \ "control_archive_sha=$control_archive_sha" \ - "$0" --real-market-preflight-worker "$@" + "$0" --real-market-preflight-worker "$@") || return 1 + if [[ $baseline_mode == legacy_python \ + && $LEGACY_RUNTIME_STABILITY_REQUIRED == true ]]; then + verify_baseline_identity || { + printf 'legacy baseline identity changed during real preflight\n' >&2 + return 1 + } + fi + printf '%s\n' "$preflight_output" } install_pinned_upload_env() { @@ -1233,6 +1212,8 @@ verify_cutover_target_preflight "$baseline_mode" "$RUST_ACTIVE_BINARY" \ gate_seconds=${MONDAY_POLYMARKET_GATE_SECONDS:-$MINIMUM_GATE_SECONDS} [[ $gate_seconds =~ ^[1-9][0-9]*$ ]] || die 'gate duration must be a positive integer' +((gate_seconds <= MINIMUM_GATE_SECONDS)) \ + || die 'production gate duration must be exactly 900 seconds' test_only=false if ((gate_seconds < MINIMUM_GATE_SECONDS)); then [[ ${MONDAY_ALLOW_SHORT_GATE_FOR_TESTS:-0} == 1 ]] \ @@ -1390,9 +1371,36 @@ baseline_health_completion_written_at_unix=null baseline_health_completion_file_identity=null baseline_health_start_success_unix=null baseline_health_cutoff_unix=null -started_at_unix=$(date -u +%s) -started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) -start_uptime=$SECONDS +if [[ $baseline_mode == legacy_python \ + && $LEGACY_HEALTH_START_REQUIRED == true ]]; then + verify_baseline_identity \ + || die 'baseline identity changed before legacy health admission' + baseline_health_observation=$(fresh_legacy_health_observation \ + "$LEGACY_SPOOL/health.json" \ + "$release_control_dir/${LEGACY_HEALTH_POLICY##*/}") \ + || die 'active legacy collector health is not fresh and fail-closed clean' + baseline_health_snapshot=$(jq -c '.health' <<<"$baseline_health_observation") \ + || die 'admitted legacy collector health observation is invalid' + baseline_health_start_written_at_unix=$(jq -er '.written_at_unix' \ + <<<"$baseline_health_observation") \ + || die 'admitted legacy collector health has no write time' + baseline_health_start_file_identity=$(jq -ce '.file_identity + | select(type == "string" and length > 0)' \ + <<<"$baseline_health_observation") \ + || die 'admitted legacy collector health has no file identity' + baseline_health_started_at=$(jq -er '.updated_at' <<<"$baseline_health_snapshot") \ + || die 'admitted legacy collector health has no updated_at' + baseline_health_start_success_at=$(jq -er '.last_success_at' \ + <<<"$baseline_health_snapshot") \ + || die 'admitted legacy collector health has no last_success_at' + baseline_health_start_success_unix=$(date -u -d \ + "$baseline_health_start_success_at" +%s) \ + || die 'admitted legacy collector last_success_at is invalid' + ((baseline_health_start_success_unix <= baseline_health_start_written_at_unix)) \ + || die 'admitted legacy collector success is after its completed write' + verify_baseline_identity \ + || die 'baseline identity changed during legacy health admission' +fi systemctl start "$shadow_unit" shadow_invocation_id=$(systemctl show --property=InvocationID --value "$shadow_unit") [[ $shadow_invocation_id =~ ^[a-f0-9]{32}$ ]] \ @@ -1413,6 +1421,16 @@ read -r memory_events_start_high memory_events_start_max memory_events_start_oom && $memory_events_start_oom_group_kill == 0 ]] \ || die 'Rust shadow reached MemoryHigh, MemoryMax, or OOM before the gate baseline' memory_events_end=$memory_events_start +verify_baseline_identity \ + || die 'baseline identity changed while the Rust shadow was starting' +started_at_unix=$(date -u +%s) +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +start_uptime=$SECONDS +observation_deadline=$gate_seconds +if [[ $test_only == true ]] \ + && ((observation_deadline < HEALTH_SETTLE_SECONDS)); then + observation_deadline=$HEALTH_SETTLE_SECONDS +fi last_health= last_health_change=$start_uptime @@ -1563,8 +1581,7 @@ while :; do common_cutoff=$((common_cutoff - PARITY_CUTOFF_LAG_SECONDS)) fi parity_window_started_at=$(bounded_parity_window_start \ - "$started_at_unix" "$common_cutoff" "$test_only" \ - "$SETTLEMENT_EVENT_LOOKBACK_SECONDS") \ + "$started_at_unix" "$common_cutoff" "$test_only") \ || die 'could not derive a bounded parity window start' fi fi @@ -1576,21 +1593,11 @@ while :; do die 'legacy collector did not complete a clean post-start cycle during the gate' fi - if ((elapsed >= gate_seconds)) && [[ -n $common_cutoff ]] \ - && [[ $legacy_health_decision == advance ]]; then - if valid_parity_window "$parity_window_started_at" "$common_cutoff"; then - if [[ $test_only == true ]] \ - || ((common_cutoff - parity_window_started_at >= PARITY_TAIL_SECONDS)); then - break - fi - elif [[ $test_only == true ]]; then - die 'short test gate could not derive an ordered parity window' - fi - fi + ((elapsed < observation_deadline)) || break sleep_for=$SAMPLE_SECONDS - if ((elapsed < gate_seconds)); then - remaining=$((gate_seconds - elapsed)) + if ((elapsed < observation_deadline)); then + remaining=$((observation_deadline - elapsed)) ((remaining < sleep_for)) && sleep_for=$remaining fi sleep "$sleep_for" @@ -1675,8 +1682,7 @@ parity_args=( --ended-at-unix "$common_cutoff" --output "$parity_json" ) -if [[ $baseline_mode == legacy_python \ - && $LEGACY_RUNTIME_STABILITY_REQUIRED == false ]]; then +if [[ $baseline_mode == legacy_python ]]; then parity_args+=(--allow-empty-legacy) fi "$release_binary" "${parity_args[@]}" \ @@ -1703,15 +1709,8 @@ canonical_uploaded_segments=$(jq -er \ '.canonical_uploaded_segments | select(type == "number" and floor == . and . > 0)' \ <<<"$upload_json") || die 'shadow uploader did not verify a canonical closed segment' -if [[ $baseline_mode == legacy_python \ - && $LEGACY_RUNTIME_STABILITY_REQUIRED == false ]]; then - legacy_pid=$(systemctl show --property=MainPID --value "$LEGACY_UNIT") - legacy_restarts=$(systemctl show --property=NRestarts --value "$LEGACY_UNIT") - legacy_invocation_id=$(systemctl show --property=InvocationID --value "$LEGACY_UNIT") - verify_legacy_identity "$legacy_pid" "$legacy_restarts" "$legacy_invocation_id" -else - verify_baseline_identity -fi || die 'baseline collector identity changed while parity or OSS readback was running' +verify_baseline_identity \ + || die 'baseline collector identity changed while parity or OSS readback was running' baseline_proc_exe='' [[ $baseline_mode != rust_release ]] || baseline_proc_exe=$(readlink -f -- "/proc/$legacy_pid/exe") || die 'could not capture the production Rust executable identity' verify_current_oss_config diff --git a/deployment/aliyun/polymarket-shadow-gate-policy.jq b/deployment/aliyun/polymarket-shadow-gate-policy.jq index 46dbaa13e..94fa48cfa 100644 --- a/deployment/aliyun/polymarket-shadow-gate-policy.jq +++ b/deployment/aliyun/polymarket-shadow-gate-policy.jq @@ -111,12 +111,17 @@ and ((.real_market_preflight.started_at | utc_iso8601_unix) <= (.real_market_preflight.completed_at | utc_iso8601_unix)) and ((.real_market_preflight.completed_at | utc_iso8601_unix) <= (.started_at | utc_iso8601_unix)) -and (.duration_seconds | positive_integer and . >= 1501) +and (.duration_seconds | positive_integer and . == 900) and (.started_at | utc_iso8601_unix | type == "number") and (.parity_window_started_at_unix | positive_integer) and (.parity_window_ended_at_unix | positive_integer) and (.parity_window_ended_at_unix - .parity_window_started_at_unix >= 601) and (.completed_at | utc_iso8601_unix | type == "number") +and ((.completed_at | utc_iso8601_unix) + - (.started_at | utc_iso8601_unix) >= .duration_seconds) +and (.parity_window_started_at_unix >= (.started_at | utc_iso8601_unix)) +and (.parity_window_ended_at_unix + <= ((.started_at | utc_iso8601_unix) + .duration_seconds)) and .production_eligible == true and .passed == true and ( @@ -172,25 +177,6 @@ and ( and ([has("release_path"),has("release_sha256"),has("proc_exe")] | any | not)) ) or - ( - .baseline_mode == "legacy_python" - and .baseline_health_start_required == false - and .baseline_runtime_stability_required == false - and .baseline_health_completion_required == false - and .baseline_health_snapshot == null - and .baseline_health_completion_snapshot == null - and .baseline_health_start_success_unix == null - and .baseline_health_cutoff_unix == null - and .baseline_health_start_written_at_unix == null - and .baseline_health_completion_written_at_unix == null - and .baseline_health_start_file_identity == null - and .baseline_health_completion_file_identity == null - and (.legacy_runtime | - runtime_identity("/usr/bin/python3 /opt/monday/bin/polymarket_reference_collector.py"; - "dffeb118d105e9312898460249f514eb982c20433cd20840ffb2107c64bbca4a") - and ([has("release_path"),has("release_sha256"),has("proc_exe")] | any | not)) - ) - or ( .baseline_mode == "rust_release" and .baseline_health_start_required == false @@ -262,7 +248,8 @@ and .checks.real_market_segment_preflight == true and (.comparison_mode == "legacy_overlap" or ( .comparison_mode == "rust_self" and .baseline_mode == "legacy_python" - and .baseline_runtime_stability_required == false + and .baseline_health_start_required == true + and .baseline_runtime_stability_required == true )) and (.metrics.oss_uploaded_segments | positive_integer) and (.metrics.oss_canonical_uploaded_segments | positive_integer) diff --git a/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh b/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh index 62a250687..2455bbb80 100755 --- a/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh +++ b/deployment/aliyun/test-polymarket-raw-ops-control-plane.sh @@ -946,8 +946,8 @@ jq -e '.status == "passed" "$good_case/evidence/real-market-preflight.json" >/dev/null export PATH=$original_path -# A short test gate must not continue after the settlement-safe clamp places -# the start at or beyond the common cutoff. +# The live parity interval begins with the Rust shadow; settlement maturity is +# proven inside the bounded observation rather than appended after it. parity_window_verifier="$tmp_dir/valid-parity-window.sh" sed -n '/^valid_parity_window() {$/,/^}$/p' "$GATE" >"$parity_window_verifier" sed -n '/^bounded_parity_window_start() {$/,/^}$/p' "$GATE" \ @@ -962,12 +962,16 @@ if valid_parity_window 1900 1800 || valid_parity_window 1800 1800; then printf 'parity-window verifier accepted a clamped start at/after cutoff\n' >&2 exit 1 fi -[[ $(bounded_parity_window_start 100 1000 true 900) == 999 ]] || { +[[ $(bounded_parity_window_start 1000 1000 true) == 999 ]] || { printf 'short test gate did not cap parity start below the cutoff\n' >&2 exit 1 } -[[ $(bounded_parity_window_start 100 1900 false 900) == 1000 ]] || { - printf 'production gate did not preserve its settlement-safe start\n' >&2 +[[ $(bounded_parity_window_start 100 1000 false) == 100 ]] || { + printf 'production gate appended settlement maturity after shadow start\n' >&2 + exit 1 +} +[[ $(bounded_parity_window_start 3000 3840 false) == 3000 ]] || { + printf 'production gate moved parity start across an hour boundary\n' >&2 exit 1 } @@ -2492,8 +2496,8 @@ jq \ upload_summary:{uploaded_segments:1,canonical_uploaded_segments:1, pending_segments:0,failed_segments:[],last_error:null} }, - duration_seconds:1501, - started_at:"1970-01-01T00:02:00Z", + duration_seconds:900, + started_at:"1970-01-01T00:01:40Z", parity_window_started_at_unix:100, parity_window_ended_at_unix:1000, completed_at:"1970-01-01T01:12:01Z", @@ -2518,7 +2522,7 @@ jq \ }, baseline_health_start_success_unix:100, baseline_health_cutoff_unix:1000, - baseline_health_start_written_at_unix:110, + baseline_health_start_written_at_unix:100, baseline_health_completion_written_at_unix:1301, baseline_health_start_file_identity:"1:10", baseline_health_completion_file_identity:"1:11", @@ -2570,23 +2574,25 @@ jq '.baseline_health_start_required = false | .baseline_health_start_written_at_unix = null | .baseline_health_start_file_identity = null' \ "$tmp_dir/expedited-legacy-gate.json" \ - >"$tmp_dir/python-nonblocking-legacy-gate.json" -jq -e -f "$POLICY" "$tmp_dir/python-nonblocking-legacy-gate.json" >/dev/null || { - printf 'gate policy rejected approved legacy-nonblocking evidence\n' >&2 + >"$tmp_dir/forged-nonblocking-legacy-gate.json" +if jq -e -f "$POLICY" \ + "$tmp_dir/forged-nonblocking-legacy-gate.json" >/dev/null; then + printf 'gate policy accepted a legacy baseline without health or identity checks\n' >&2 exit 1 -} +fi jq '.comparison_mode = "rust_self" | .metrics.legacy_trade_count = 0 | .metrics.legacy_metadata_count = 0 | .metrics.legacy_settlement_count = 0' \ - "$tmp_dir/python-nonblocking-legacy-gate.json" \ + "$tmp_dir/expedited-legacy-gate.json" \ >"$tmp_dir/rust-self-gate.json" jq -e -f "$POLICY" "$tmp_dir/rust-self-gate.json" >/dev/null || { printf 'gate policy rejected valid Rust-self parity evidence\n' >&2 exit 1 } for mutation in \ - '.baseline_runtime_stability_required = true' \ + '.baseline_health_start_required = false' \ + '.baseline_runtime_stability_required = false' \ '.metrics.legacy_trade_count = 1' \ '.metrics.rust_settlement_count = 0' \ '.checks.settlement_parity = false'; do @@ -2597,33 +2603,20 @@ for mutation in \ exit 1 fi done -for field in baseline_health_start_required baseline_runtime_stability_required; do - [[ $(jq -er --arg field "$field" \ - '.[$field] | select(type == "boolean") | tostring' \ - "$tmp_dir/python-nonblocking-legacy-gate.json") == false ]] || { - printf 'cutover boolean parser rejected a false gate contract\n' >&2 - exit 1 - } - grep -Fq ".$field | select(type == \"boolean\") | tostring" "$CUTOVER" || { - printf 'cutover does not preserve a false gate contract boolean\n' >&2 - exit 1 - } -done for mutation in \ 'del(.baseline_health_start_required)' \ - '.baseline_health_start_required = true' \ - '.baseline_runtime_stability_required = true' \ + '.baseline_health_start_required = false' \ + '.baseline_runtime_stability_required = false' \ '.checks.metadata_parity = false'; do - jq "$mutation" "$tmp_dir/python-nonblocking-legacy-gate.json" \ - >"$tmp_dir/forged-python-nonblocking-legacy-gate.json" + jq "$mutation" "$tmp_dir/expedited-legacy-gate.json" \ + >"$tmp_dir/forged-expedited-legacy-gate.json" if jq -e -f "$POLICY" \ - "$tmp_dir/forged-python-nonblocking-legacy-gate.json" >/dev/null; then - printf 'gate policy accepted forged legacy-nonblocking evidence\n' >&2 + "$tmp_dir/forged-expedited-legacy-gate.json" >/dev/null; then + printf 'gate policy accepted forged legacy baseline evidence\n' >&2 exit 1 fi done -if grep -Fq 'baseline_health_observation=$(wait_for_fresh_legacy_health_observation' \ - "$GATE"; then +if grep -Fq 'wait_for_fresh_legacy_health_observation' "$GATE"; then printf 'production Gate still waits for a legacy health publication\n' >&2 exit 1 fi @@ -2698,15 +2691,6 @@ if jq -e -f "$POLICY" "$tmp_dir/error-market-preflight.json" >/dev/null; then printf 'gate policy accepted a preflight with a terminal error\n' >&2 exit 1 fi -jq '.started_at = "1970-01-01T00:16:40Z" - | .baseline_health_start_written_at_unix = 900 - | .baseline_health_completion_written_at_unix = 1000' \ - "$tmp_dir/gate.json" >"$tmp_dir/same-second-baseline-health-write.json" -jq -e -f "$POLICY" "$tmp_dir/same-second-baseline-health-write.json" \ - >/dev/null || { - printf 'gate policy rejected a distinct atomic health write in the Gate start second\n' >&2 - exit 1 -} jq '.baseline_health_completion_file_identity = .baseline_health_start_file_identity' "$tmp_dir/gate.json" \ >"$tmp_dir/reused-baseline-health-file.json" @@ -2947,9 +2931,26 @@ if jq -e -f "$POLICY" "$tmp_dir/unbound-settlement-end.json" >/dev/null; then printf 'gate policy accepted an unbound settlement end window\n' >&2 exit 1 fi -jq '.duration_seconds = 1500' "$tmp_dir/gate.json" >"$tmp_dir/short.json" +jq '.duration_seconds = 899' "$tmp_dir/gate.json" >"$tmp_dir/short.json" if jq -e -f "$POLICY" "$tmp_dir/short.json" >/dev/null; then - printf 'gate policy accepted a shadow shorter than 15 minutes plus its maturity tail\n' >&2 + printf 'gate policy accepted a shadow shorter than 15 minutes total\n' >&2 + exit 1 +fi +jq '.duration_seconds = 901' "$tmp_dir/gate.json" >"$tmp_dir/long.json" +if jq -e -f "$POLICY" "$tmp_dir/long.json" >/dev/null; then + printf 'gate policy accepted a shadow longer than exactly 15 minutes\n' >&2 + exit 1 +fi +jq '.started_at = "1970-01-01T00:01:41Z"' \ + "$tmp_dir/expedited-legacy-gate.json" >"$tmp_dir/pre-shadow-parity.json" +if jq -e -f "$POLICY" "$tmp_dir/pre-shadow-parity.json" >/dev/null; then + printf 'gate policy accepted parity beginning before the formal Gate\n' >&2 + exit 1 +fi +jq '.completed_at = "1970-01-01T00:16:39Z"' \ + "$tmp_dir/expedited-legacy-gate.json" >"$tmp_dir/unelapsed-gate.json" +if jq -e -f "$POLICY" "$tmp_dir/unelapsed-gate.json" >/dev/null; then + printf 'gate policy accepted 900 seconds without 900 elapsed wall-clock seconds\n' >&2 exit 1 fi jq '.production_eligible = false' "$tmp_dir/gate.json" >"$tmp_dir/test-only.json" @@ -3204,7 +3205,6 @@ sed -n \ -e '/^readonly REQUIRED_DURATION_SECONDS=/p' \ -e '/^readonly PARITY_TAIL_SECONDS=/p' \ -e '/^readonly MINIMUM_GATE_SECONDS=/p' \ - -e '/^readonly LEGACY_HEALTH_START_WAIT_SECONDS=/p' \ -e '/^readonly LEGACY_RUNTIME_STABILITY_REQUIRED=/p' \ -e '/^readonly LEGACY_RUNTIME_MAX_SECONDS=/p' \ -e '/^readonly LEGACY_RUNTIME_RESERVE_SECONDS=/p' \ @@ -3236,8 +3236,8 @@ grep -Fq '[[ $zstd_timeout_seconds == 300 && $oss_copy_timeout_seconds == 300 ]] ( # shellcheck source=/dev/null source "$legacy_runtime_budget_contract" - [[ $REAL_MARKET_PREFLIGHT_BUDGET_SECONDS -eq 6300 \ - && $LEGACY_RUNTIME_STABILITY_REQUIRED == false \ + [[ $REAL_MARKET_PREFLIGHT_BUDGET_SECONDS -eq 300 \ + && $LEGACY_RUNTIME_STABILITY_REQUIRED == true \ && $LEGACY_RUNTIME_MAX_SECONDS -eq 21600 \ && $LEGACY_RUNTIME_RESERVE_SECONDS -eq 60 \ && $PARITY_CUTOFF_LAG_SECONDS -eq 60 ]] || { @@ -3252,9 +3252,17 @@ grep -Fq '[[ $zstd_timeout_seconds == 300 && $oss_copy_timeout_seconds == 300 ]] esac } monotonic_uptime_seconds() { printf '20906\n'; } + zstd_timeout_seconds=300 + oss_copy_timeout_seconds=300 required=$((REAL_MARKET_PREFLIGHT_BUDGET_SECONDS \ - + LEGACY_HEALTH_START_WAIT_SECONDS + MINIMUM_GATE_SECONDS \ - + PARITY_CUTOFF_LAG_SECONDS + LEGACY_RUNTIME_RESERVE_SECONDS)) + + MINIMUM_GATE_SECONDS \ + + PARITY_CUTOFF_LAG_SECONDS \ + + zstd_timeout_seconds + oss_copy_timeout_seconds \ + + LEGACY_RUNTIME_RESERVE_SECONDS)) + [[ $required -eq 1920 ]] || { + printf 'Gate runtime budget does not cover bounded post-gate uploads\n' >&2 + exit 1 + } if observation=$(legacy_runtime_budget_observation "$required"); then printf 'Gate accepted 695 seconds of remaining runtime for a %s-second gate\n' \ "$required" >&2 @@ -3292,13 +3300,11 @@ grep -Fq '[[ $zstd_timeout_seconds == 300 && $oss_copy_timeout_seconds == 300 ]] zstd_timeout_seconds=300 oss_copy_timeout_seconds=300 oss_config_sha=unused source_revision=unused deployment_bundle_sha=unused release_manifest_sha=unused control_archive_sha=unused - preflight_calls=0 identity_checks=0 - timeout_args= + timeout_log="$tmp_dir/runtime-budget-timeout.args" verify_baseline_identity() { identity_checks=$((identity_checks + 1)); } timeout() { - preflight_calls=$((preflight_calls + 1)) - timeout_args=$* + printf '%s\n' "$*" >"$timeout_log" printf '{}\n' } legacy_runtime_budget_observation() { @@ -3306,13 +3312,17 @@ grep -Fq '[[ $zstd_timeout_seconds == 300 && $oss_copy_timeout_seconds == 300 ]] return 1 } admission_error="$tmp_dir/runtime-budget-admission.err" - run_budgeted_real_market_preflight source spool download evidence \ - 2>"$admission_error" >/dev/null || { - printf 'Gate still blocks real preflight on legacy runtime budget\n' >&2 + if run_budgeted_real_market_preflight source spool download evidence \ + 2>"$admission_error" >/dev/null; then + printf 'Gate admitted a legacy identity that cannot survive the Gate\n' >&2 + exit 1 + fi + grep -Fq "remaining=695 required=$required" "$admission_error" || { + printf 'Gate admission omitted bounded post-gate work from its runtime budget\n' >&2 exit 1 } - [[ $preflight_calls -eq 1 && $identity_checks -eq 0 ]] || { - printf 'Gate still consults legacy identity or runtime before real preflight\n' >&2 + [[ ! -e $timeout_log && $identity_checks -eq 1 ]] || { + printf 'Gate did not fail closed on an unstable legacy identity before preflight\n' >&2 exit 1 } legacy_runtime_budget_observation() { @@ -3322,11 +3332,11 @@ grep -Fq '[[ $zstd_timeout_seconds == 300 && $oss_copy_timeout_seconds == 300 ]] printf 'Gate rejected a sufficient runtime budget at the admission seam\n' >&2 exit 1 } - [[ $preflight_calls -eq 2 && $identity_checks -eq 0 ]] || { - printf 'Gate made legacy identity a real-preflight prerequisite\n' >&2 + [[ -s $timeout_log && $identity_checks -eq 4 ]] || { + printf 'Gate did not bind legacy identity around real preflight\n' >&2 exit 1 } - [[ $timeout_args == '--signal=KILL 6300 env '* ]] || { + [[ $(<"$timeout_log") == '--signal=KILL 300 env '* ]] || { printf 'Gate real preflight does not have an exact hard deadline\n' >&2 exit 1 } @@ -3380,19 +3390,13 @@ grep -Fq 'run_before_deadline "$preflight_deadline" runuser' "$GATE" || { legacy_health_observer="$tmp_dir/legacy-health-observer.sh" sed -n \ -e '/^readonly MAX_HEALTH_SILENCE_SECONDS=/p' \ - -e '/^readonly LEGACY_HEALTH_START_WAIT_SECONDS=/p' \ -e '/^legacy_health_publication_after_gate() {$/,/^}$/p' \ - -e '/^fresh_legacy_health_observation() {$/,/^}$/p' \ - -e '/^wait_for_fresh_legacy_health_observation() {$/,/^}$/p' "$GATE" \ + -e '/^fresh_legacy_health_observation() {$/,/^}$/p' "$GATE" \ >"$legacy_health_observer" [[ -s $legacy_health_observer ]] || { printf 'Gate has no completed-write freshness verifier for legacy health\n' >&2 exit 1 } -grep -Fq 'wait_for_fresh_legacy_health_observation() {' "$GATE" || { - printf 'Gate has no bounded legacy health admission wait\n' >&2 - exit 1 -} ( if command -v gdate >/dev/null 2>&1; then date() { command gdate "$@"; } @@ -3402,56 +3406,6 @@ grep -Fq 'wait_for_fresh_legacy_health_observation() {' "$GATE" || { fi # shellcheck source=/dev/null source "$legacy_health_observer" - [[ $LEGACY_HEALTH_START_WAIT_SECONDS -eq 3900 ]] || { - printf 'Gate legacy health admission wait is not bounded to 3900 seconds\n' >&2 - exit 1 - } - verify_baseline_identity() { return 0; } - cp "$tmp_dir/legacy-health.json" "$tmp_dir/waiting-legacy-health.json" - touch -d '1970-01-01T00:00:00Z' "$tmp_dir/waiting-legacy-health.json" - cp "$tmp_dir/legacy-health.json" "$tmp_dir/pending-legacy-health.json" - printf '0\n' >"$tmp_dir/legacy-health-wait-sleeps" - sleep() { - local calls - calls=$(($(<"$tmp_dir/legacy-health-wait-sleeps") + 1)) - printf '%s\n' "$calls" >"$tmp_dir/legacy-health-wait-sleeps" - if ((calls == 1)); then - mv "$tmp_dir/pending-legacy-health.json" "$tmp_dir/waiting-legacy-health.json" - fi - } - wait_observation=$(wait_for_fresh_legacy_health_observation \ - "$tmp_dir/waiting-legacy-health.json" "$LEGACY_HEALTH_POLICY") || { - printf 'Gate did not wait for the next completed legacy health publication\n' >&2 - exit 1 - } - [[ $(<"$tmp_dir/legacy-health-wait-sleeps") -eq 1 ]] || { - printf 'Gate legacy health admission did not wait exactly once for completion\n' >&2 - exit 1 - } - jq -e '.health.last_success_at == "2026-07-15T00:00:01Z"' \ - <<<"$wait_observation" >/dev/null - cp "$tmp_dir/legacy-health.json" "$tmp_dir/timed-out-legacy-health.json" - touch -d '1970-01-01T00:00:00Z' "$tmp_dir/timed-out-legacy-health.json" - SECONDS=0 - sleep() { SECONDS=$((SECONDS + LEGACY_HEALTH_START_WAIT_SECONDS)); } - if wait_for_fresh_legacy_health_observation \ - "$tmp_dir/timed-out-legacy-health.json" "$LEGACY_HEALTH_POLICY" >/dev/null; then - printf 'Gate accepted legacy health after its bounded admission wait expired\n' >&2 - exit 1 - fi - cp "$tmp_dir/legacy-health.json" "$tmp_dir/late-legacy-health.json" - touch -d '1970-01-01T00:00:00Z' "$tmp_dir/late-legacy-health.json" - cp "$tmp_dir/legacy-health.json" "$tmp_dir/late-pending-legacy-health.json" - SECONDS=0 - sleep() { - mv "$tmp_dir/late-pending-legacy-health.json" "$tmp_dir/late-legacy-health.json" - SECONDS=$LEGACY_HEALTH_START_WAIT_SECONDS - } - if wait_for_fresh_legacy_health_observation \ - "$tmp_dir/late-legacy-health.json" "$LEGACY_HEALTH_POLICY" >/dev/null; then - printf 'Gate accepted legacy health published after its bounded admission wait expired\n' >&2 - exit 1 - fi legacy_health_publication_after_gate 120 start-file 120 completion-file || { printf 'Gate rejected a distinct atomic health write in the Gate start second\n' >&2 exit 1 @@ -3517,15 +3471,31 @@ grep -Fq 'wait_for_fresh_legacy_health_observation() {' "$GATE" || { ) ) daemon_reload_line=$(grep -nF 'systemctl daemon-reload' "$GATE" | tail -1 | cut -d: -f1) +preflight_line=$(grep -nF \ + 'real_market_preflight_json=$(run_budgeted_real_market_preflight' \ + "$GATE" | cut -d: -f1) +start_snapshot_line=$(grep -nF \ + 'baseline_health_observation=$(fresh_legacy_health_observation' \ + "$GATE" | cut -d: -f1) +start_identity_before_line=$(grep -nF \ + "baseline identity changed before legacy health admission" \ + "$GATE" | cut -d: -f1) +start_identity_after_line=$(grep -nF \ + "baseline identity changed during legacy health admission" \ + "$GATE" | cut -d: -f1) completion_snapshot_line=$(grep -nF \ 'baseline_health_completion_observation=$(fresh_legacy_health_observation' \ "$GATE" | cut -d: -f1) gate_start_line=$(grep -nF 'started_at_unix=$(date -u +%s)' "$GATE" | cut -d: -f1) shadow_start_line=$(grep -nF 'systemctl start "$shadow_unit"' "$GATE" | cut -d: -f1) -if ! ((daemon_reload_line < gate_start_line \ - && gate_start_line < shadow_start_line \ +if ! ((daemon_reload_line < preflight_line \ + && preflight_line < start_identity_before_line \ + && start_identity_before_line < start_snapshot_line \ + && start_snapshot_line < start_identity_after_line \ + && start_identity_after_line < shadow_start_line \ + && shadow_start_line < gate_start_line \ && shadow_start_line < completion_snapshot_line)); then - printf 'legacy health completion seam is not after shadow startup\n' >&2 + printf 'legacy health admission or completion seam is ordered incorrectly\n' >&2 exit 1 fi [[ $(legacy_health_sample_state \ @@ -3641,8 +3611,11 @@ done grep -Fq 'readonly REQUIRED_DURATION_SECONDS=900' "$GATE" grep -Fq 'readonly PARITY_TAIL_SECONDS=601' "$GATE" +grep -Fq 'readonly MINIMUM_GATE_SECONDS=$REQUIRED_DURATION_SECONDS' "$GATE" +grep -Fq 'production gate duration must be exactly 900 seconds' "$GATE" grep -Fq 'readonly LEGACY_HEALTH_COMPLETION_REQUIRED=false' "$GATE" -grep -Fq 'readonly SETTLEMENT_EVENT_LOOKBACK_SECONDS=900' "$GATE" +grep -Fq 'readonly LEGACY_HEALTH_START_REQUIRED=true' "$GATE" +grep -Fq 'readonly LEGACY_RUNTIME_STABILITY_REQUIRED=true' "$GATE" grep -Fq 'bounded_parity_window_start' "$GATE" grep -Fq 'readonly MAX_ACCEPTED_CYCLE_SECONDS=180' "$GATE" grep -Fq 'readonly INITIAL_HEALTH_GRACE_SECONDS=60' "$GATE" @@ -3691,8 +3664,10 @@ if ((legacy_transition_line >= health_settle_line)); then exit 1 fi grep -Fq 'if [[ $legacy_health_decision == advance ]]; then' "$GATE" -grep -Fq '&& [[ $legacy_health_decision == advance ]]; then' \ - "$GATE" +grep -Fq 'observation_deadline=$gate_seconds' "$GATE" +grep -Fq '&& ((observation_deadline < HEALTH_SETTLE_SECONDS)); then' "$GATE" +grep -Fq 'observation_deadline=$HEALTH_SETTLE_SECONDS' "$GATE" +grep -Fq '((elapsed < observation_deadline)) || break' "$GATE" grep -Fq 'if ((elapsed >= HEALTH_SETTLE_SECONDS)); then' "$GATE" if grep -Fq 'if ((elapsed >= HEALTH_SETTLE_SECONDS)) || [[ $test_only == true ]]; then' "$GATE"; then printf 'short shadow gate bypasses the initial health settle window\n' >&2 @@ -3700,7 +3675,10 @@ if grep -Fq 'if ((elapsed >= HEALTH_SETTLE_SECONDS)) || [[ $test_only == true ]] fi grep -Fq 'verify-shadow-parity' "$GATE" grep -Fq 'parity_args+=(--allow-empty-legacy)' "$GATE" -grep -Fq '&& $LEGACY_RUNTIME_STABILITY_REQUIRED == false ]]; then' "$GATE" +if grep -Fq '&& $LEGACY_RUNTIME_STABILITY_REQUIRED == false ]]; then' "$GATE"; then + printf 'Gate retains a legacy identity bypass after parity or OSS readback\n' >&2 + exit 1 +fi [[ ! -e "$SCRIPT_DIR/verify-polymarket-shadow-parity.py" ]] if grep -Fq 'python3 "$PARITY_VERIFIER"' "$GATE"; then printf 'shadow gate still invokes the retired legacy parity verifier\n' >&2 diff --git a/rust_hft/tools/collector/src/polymarket_parity.rs b/rust_hft/tools/collector/src/polymarket_parity.rs index 1c90ee162..22d90b3b0 100644 --- a/rust_hft/tools/collector/src/polymarket_parity.rs +++ b/rust_hft/tools/collector/src/polymarket_parity.rs @@ -956,7 +956,13 @@ fn compare(config: &ShadowParityConfig) -> Result { config.ended_at_unix, trade_event_window_end, )?; - let rust_self_mode = config.allow_empty_legacy && legacy_rows.is_empty(); + let legacy_has_comparison_rows = legacy_rows.iter().any(|row| { + matches!( + row.update.get("kind").and_then(Value::as_str), + Some("polymarket_trade" | "market_settlement") + ) + }); + let rust_self_mode = config.allow_empty_legacy && !legacy_has_comparison_rows; let comparison_mode = if rust_self_mode { "rust_self" } else { @@ -1469,6 +1475,33 @@ mod tests { ); } + #[test] + fn metadata_only_legacy_window_uses_explicit_rust_self_mode() { + let (_root, mut config) = fixture(); + let legacy_metadata = EXPECTED_SYMBOLS + .iter() + .map(|symbol| metadata(symbol)) + .collect::>(); + write_tape( + &config.legacy_spool.join(ACTIVE_TAPE), + &legacy_metadata, + "1970-01-01T00:03:20Z", + ); + config.allow_empty_legacy = true; + + let evidence = compare(&config).unwrap(); + assert_eq!(evidence["passed"], true); + assert_eq!(evidence["comparison_mode"], "rust_self"); + assert!( + evidence["metrics"]["legacy_metadata_count"] + .as_u64() + .unwrap() + > 0 + ); + assert_eq!(evidence["metrics"]["legacy_trade_count"], 0); + assert_eq!(evidence["metrics"]["legacy_settlement_count"], 0); + } + #[test] fn rust_self_mode_rejects_missing_rust_settlement() { let (_root, mut config) = fixture();