diff --git a/deployment/aliyun/host-rust-lob-shadow-gate.sh b/deployment/aliyun/host-rust-lob-shadow-gate.sh index 7e4821256..78292a070 100755 --- a/deployment/aliyun/host-rust-lob-shadow-gate.sh +++ b/deployment/aliyun/host-rust-lob-shadow-gate.sh @@ -27,7 +27,27 @@ usage() { '' \ 'Production gates always observe at least 3600 seconds.' \ 'Tests may set MONDAY_GATE_TEST_SECONDS only with' \ - 'MONDAY_ALLOW_SHORT_GATE_FOR_TESTS=1; test evidence cannot pass cutover.' + 'MONDAY_ALLOW_SHORT_GATE_FOR_TESTS=1; test evidence cannot pass cutover.' \ + 'Test-only health settling may use MONDAY_TEST_HEALTH_SETTLE_SECONDS only' \ + 'with MONDAY_ALLOW_SHORT_GATE_FOR_TESTS=1 and a value below 2400 seconds;' \ + 'otherwise the policy check fails.' +} + +resolve_health_settle_seconds() { + health_settle_seconds=$HEALTH_SETTLE_SECONDS + if [[ -n ${MONDAY_TEST_HEALTH_SETTLE_SECONDS:-} ]]; then + [[ $test_only == true ]] \ + || die 'short health settles require a test-only gate' + [[ ${MONDAY_ALLOW_SHORT_GATE_FOR_TESTS:-0} == 1 ]] \ + || die 'short health settles require MONDAY_ALLOW_SHORT_GATE_FOR_TESTS=1' + [[ ${MONDAY_TEST_HEALTH_SETTLE_SECONDS} =~ ^[1-9][0-9]*$ ]] \ + || die 'test health settle duration must be a positive integer' + (( ${#MONDAY_TEST_HEALTH_SETTLE_SECONDS} <= ${#HEALTH_SETTLE_SECONDS} )) \ + || die 'test health settle duration is too large' + ((MONDAY_TEST_HEALTH_SETTLE_SECONDS < HEALTH_SETTLE_SECONDS)) \ + || die 'test health settle duration must be shorter than the formal settle duration' + health_settle_seconds=$MONDAY_TEST_HEALTH_SETTLE_SECONDS + fi } [[ ${EUID} -eq 0 ]] || die 'must run as root' @@ -110,6 +130,7 @@ if ((gate_seconds < REQUIRED_DURATION_SECONDS)); then || die 'short gates require MONDAY_ALLOW_SHORT_GATE_FOR_TESTS=1' test_only=true fi +resolve_health_settle_seconds env_value() { local file=$1 @@ -230,12 +251,14 @@ jq -n \ --arg deployment_bundle_sha256 "$deployment_bundle_sha256" \ --arg deployment_source_revision "$deployment_source_revision" \ --argjson requested_duration_seconds "$gate_seconds" \ + --argjson health_settle_seconds "$health_settle_seconds" \ --argjson test_only "$test_only" \ '{schema:$schema,run_id:$run_id,created_at:$created_at, candidate_sha256:$candidate_sha256, deployment_bundle_sha256:$deployment_bundle_sha256, deployment_source_revision:$deployment_source_revision, - requested_duration_seconds:$requested_duration_seconds,test_only:$test_only}' \ + requested_duration_seconds:$requested_duration_seconds, + health_settle_seconds:$health_settle_seconds,test_only:$test_only}' \ >"$evidence_dir/run.json" chmod 0640 "$evidence_dir/run.json" @@ -429,7 +452,7 @@ health_catalog_sha256() { | sha256sum | awk '{print $1}' } -settle_deadline=$(( $(monotonic_seconds) + HEALTH_SETTLE_SECONDS )) +settle_deadline=$(( $(monotonic_seconds) + health_settle_seconds )) while ! health_passes spot || ! health_passes usdm; do (( $(monotonic_seconds) < settle_deadline )) \ || die 'shadow health did not reach the fail-closed gate before the settle deadline' @@ -590,7 +613,7 @@ manifest_uris() { local listing=$2 local prefix line token max_age_seconds prefix="oss://${oss_bucket[$market]}/lake/raw/venue=binance/market=${market}/dataset=${dataset[$market]}/shard=${shard_id[$market]}/" - max_age_seconds=$((gate_seconds + HEALTH_SETTLE_SECONDS + 3600)) + max_age_seconds=$((gate_seconds + health_settle_seconds + 3600)) run_oss "$market" ls "$prefix" --recursive --short-format \ --max-age "${max_age_seconds}s" >"$listing" while IFS= read -r line; do @@ -919,6 +942,7 @@ jq -n \ --arg finished_at "$gate_finished_at" \ --argjson required_duration_seconds "$REQUIRED_DURATION_SECONDS" \ --argjson requested_duration_seconds "$gate_seconds" \ + --argjson health_settle_seconds "$health_settle_seconds" \ --argjson duration_seconds "$duration_seconds" \ --argjson test_only "$test_only" \ --argjson checks_passed true \ @@ -931,6 +955,7 @@ jq -n \ started_at:$started_at,finished_at:$finished_at, required_duration_seconds:$required_duration_seconds, requested_duration_seconds:$requested_duration_seconds, + health_settle_seconds:$health_settle_seconds, duration_seconds:$duration_seconds, test_only:$test_only,checks_passed:$checks_passed, production_eligible:$production_eligible,passed:$passed,markets:$markets}' \ diff --git a/deployment/aliyun/test-rust-lob-control-plane.sh b/deployment/aliyun/test-rust-lob-control-plane.sh index 23e95e05c..941d39440 100755 --- a/deployment/aliyun/test-rust-lob-control-plane.sh +++ b/deployment/aliyun/test-rust-lob-control-plane.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Dynamically sourced production functions consume fixture globals and mocks. -# shellcheck disable=SC2016,SC2034,SC2317,SC2329 +# shellcheck disable=SC1090,SC2016,SC2034,SC2154,SC2317,SC2329 set -euo pipefail SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd) @@ -45,7 +45,23 @@ grep -Fq 'fewer than two replay-safe complete OSS manifests' "$GATE" grep -Fq 'replay-unsafe manifest before a later replay-safe manifest' "$LIB" grep -Fq 'install -d -m 0750 -o "$SERVICE_USER" -g "$SERVICE_USER" "$segment_dir"' "$GATE" grep -Fq 'manifest_sha256:$manifest_sha256' "$GATE" +grep -Fq 'readonly HEALTH_SETTLE_SECONDS=2400' "$GATE" grep -Fq 'readonly MAX_HEALTH_SILENCE_SECONDS=120' "$GATE" +grep -Fq 'MONDAY_TEST_HEALTH_SETTLE_SECONDS' "$GATE" +grep -Fq 'short health settles require a test-only gate' "$GATE" +grep -Fq 'test health settle duration is too large' "$GATE" +grep -Fq 'MONDAY_TEST_HEALTH_SETTLE_SECONDS < HEALTH_SETTLE_SECONDS' "$GATE" +grep -Fq 'health_settle_seconds=$HEALTH_SETTLE_SECONDS' "$GATE" +grep -Fq 'settle_deadline=$(( $(monotonic_seconds) + health_settle_seconds ))' "$GATE" +grep -Fq 'max_age_seconds=$((gate_seconds + health_settle_seconds + 3600))' "$GATE" +[[ $(grep -Fc -- '--argjson health_settle_seconds "$health_settle_seconds"' "$GATE") -eq 2 ]] || { + printf 'run and final gate evidence do not both record the effective health settle duration\n' >&2 + exit 1 +} +[[ $(grep -Fc 'health_settle_seconds:$health_settle_seconds' "$GATE") -eq 2 ]] || { + printf 'run and final gate evidence do not both expose the effective health settle duration\n' >&2 + exit 1 +} grep -Fq 'and .all_symbols_bridged == true' "$GATE" grep -Fq 'and .bridged_count == .symbol_count' "$GATE" grep -Fq 'and .snapshot_only_symbols == []' "$GATE" @@ -100,6 +116,43 @@ done tmp_dir=$(mktemp -d) trap 'rm -rf "$tmp_dir"' EXIT +health_settle_body="$tmp_dir/resolve-health-settle.sh" +sed -n '/^resolve_health_settle_seconds()/,/^}/p' "$GATE" >"$health_settle_body" +resolve_health_settle() ( + HEALTH_SETTLE_SECONDS=2400 + gate_seconds=$1 + test_only=$2 + MONDAY_ALLOW_SHORT_GATE_FOR_TESTS=$3 + MONDAY_TEST_HEALTH_SETTLE_SECONDS=$4 + die() { printf '%s\n' "$*" >&2; exit 1; } + # shellcheck disable=SC1090 + . "$health_settle_body" + resolve_health_settle_seconds + printf '%s\n' "$health_settle_seconds" +) +[[ $(resolve_health_settle 1800 true 1 60) == 60 ]] || { + printf 'authorized short health settle was not applied\n' >&2 + exit 1 +} +[[ $(resolve_health_settle 1800 true 1 '') == 2400 ]] || { + printf 'test-only gate without an override did not keep the formal settle\n' >&2 + exit 1 +} +for fixture in \ + '3600 false 1 60' \ + '1800 true 0 60' \ + '1800 true 1 invalid' \ + '1800 true 1 2400' \ + '1800 true 1 2401' \ + "1800 true 1 $(printf '9%.0s' {1..100})"; do + read -r fixture_gate fixture_test fixture_auth fixture_value <<<"$fixture" + if resolve_health_settle "$fixture_gate" "$fixture_test" "$fixture_auth" \ + "$fixture_value" >/dev/null 2>&1; then + printf 'invalid short health settle fixture was accepted: %s\n' "$fixture" >&2 + exit 1 + fi +done + safe_candidates="$tmp_dir/safe-candidates.tsv" unsafe_candidates="$tmp_dir/unsafe-candidates.tsv" printf '100\t200\tsafe-1\n200\t300\tsafe-2\n' >"$safe_candidates"