Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 68 additions & 6 deletions deployment/aliyun/host-rust-lob-shadow-gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -611,7 +611,7 @@ verify_oss_round_trips() {
local actual_digest bytes agg_trade_count manifest_agg_trade_count gap_ns
local previous_end_ns=0
local round_trips='[]'
local -a strict_verifier_args=()
local -a strict_verifier_args=(--require-lob-continuity)

manifest_uris "$market" "$listing" >"$uris"
: >"$candidates"
Expand Down Expand Up @@ -640,11 +640,47 @@ verify_oss_round_trips() {
start_ns=$(jq -er '.start_received_at_ns' "$manifest")
end_ns=$(jq -er '.end_received_at_ns' "$manifest")
((end_ns < gate_started_ns)) && continue
jq -e \
jq -e --arg session_id "${observed_session[$market]}" \
'.schema == "binance.market_tape.v1"
and .trade_summary_contract == "binance.aggregate_trade_summary.v1"
and (.trade_summaries | type) == "object"
and (.trade_summaries | length) > 0
and .lob_continuity.contract == "binance.lob_continuity.v1"
and .lob_continuity.capture_session_id == $session_id
Comment on lines +648 to +649

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Pass session_id to the LOB manifest jq check

For every manifest whose end time reaches the gate window, this second jq invocation references $session_id without defining it. The --arg session_id on the earlier discovery invocation is local to that separate process, so jq exits with '$session_id is not defined' and the production shadow gate always takes the failure path before verifying any OSS segments. Add the argument to this invocation or combine the checks.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 2bf102f: the session argument is now bound on the same jq invocation that evaluates the LOB continuity contract, with a focused regression assertion.

and (.lob_continuity.reconnect_boundary | type) == "boolean"
and .lob_continuity.sequence_gaps == 0
and .lob_continuity.source_time_rollbacks == 0
and .lob_continuity.declared_symbol_count == (.symbols | length)
and .lob_continuity.covered_symbol_count == (.symbols | length)
and .lob_continuity.missing_symbols == []
and (.lob_continuity.symbols | type) == "object"
and (.lob_continuity.symbols | length) == (.symbols | length)
and all(.lob_continuity.symbols[];
.snapshot_seed_count > 0
and .diff_count > 0
and .checkpoint_count > 0
and (.first_update_id | type) == "number"
and (.last_update_id | type) == "number"
and .last_update_id >= .first_update_id
and (.first_source_time_ms | type) == "number"
and (.last_source_time_ms | type) == "number"
and .last_source_time_ms >= .first_source_time_ms
and (.first_received_at_ns | type) == "number"
and (.last_received_at_ns | type) == "number"
and .last_received_at_ns >= .first_received_at_ns
and (.min_source_latency_ms | type) == "number"
and (.max_source_latency_ms | type) == "number"
and .min_source_latency_ms >= -1000
and .max_source_latency_ms <= 30000
and .max_source_latency_ms >= .min_source_latency_ms
and (.min_bid_levels | type) == "number"
and (.max_bid_levels | type) == "number"
and .min_bid_levels > 0
and .max_bid_levels >= .min_bid_levels
and (.min_ask_levels | type) == "number"
and (.max_ask_levels | type) == "number"
and .min_ask_levels > 0
and .max_ask_levels >= .min_ask_levels)
and (.event_types.agg_trade | type) == "number"
and .event_types.agg_trade == (.event_types.agg_trade | floor)
and .event_types.agg_trade > 0' \
Expand Down Expand Up @@ -740,16 +776,36 @@ verify_oss_round_trips() {
--argjson gap_from_previous_ns "$gap_ns" \
--argjson bytes "$bytes" \
--argjson agg_trade_count "$agg_trade_count" \
'{manifest_uri:$manifest_uri,data_uri:$data_uri,success_uri:$success_uri,sha256:$sha256,
--slurpfile manifest "$manifest_path" \
'($manifest[0].lob_continuity) as $lob_continuity
| {manifest_uri:$manifest_uri,data_uri:$data_uri,success_uri:$success_uri,sha256:$sha256,
manifest_sha256:$manifest_sha256,gap_from_previous_ns:$gap_from_previous_ns,
start_received_at_ns:$start_received_at_ns,end_received_at_ns:$end_received_at_ns,bytes:$bytes,
agg_trade_count:$agg_trade_count}')
agg_trade_count:$agg_trade_count,
lob_capture_session_id:$lob_continuity.capture_session_id,
lob_reconnect_boundary:$lob_continuity.reconnect_boundary,
lob_sequence_gaps:$lob_continuity.sequence_gaps,
lob_source_time_rollbacks:$lob_continuity.source_time_rollbacks,
lob_declared_symbol_count:$lob_continuity.declared_symbol_count,
lob_covered_symbol_count:$lob_continuity.covered_symbol_count,
lob_min_source_latency_ms:([$lob_continuity.symbols[].min_source_latency_ms] | min),
lob_max_source_latency_ms:([$lob_continuity.symbols[].max_source_latency_ms] | max),
lob_min_bid_levels:([$lob_continuity.symbols[].min_bid_levels] | min),
lob_min_ask_levels:([$lob_continuity.symbols[].min_ask_levels] | min)}')
round_trips=$(jq -cn --argjson values "$round_trips" --argjson value "$round_trip" \
'$values + [$value]')
done < <(sort -n -k1,1 "$candidates")

"$candidate_binary" "${strict_verifier_args[@]}" >/dev/null \
|| die "$market strict aggregate-trade summary readback failed"
|| die "$market strict aggregate-trade and LOB continuity readback failed"

jq -e --arg session_id "${observed_session[$market]}" '
(map(select(.lob_reconnect_boundary)) | length) == 1
and .[0].lob_reconnect_boundary == true
and all(.[1:][].lob_reconnect_boundary; . == false)
and all(.[].lob_capture_session_id; . == $session_id)' \
<<<"$round_trips" >/dev/null \
|| die "$market LOB evidence crosses a capture-session boundary"

printf '%s\n' "$round_trips"
}
Expand Down Expand Up @@ -794,6 +850,12 @@ for market in "${markets[@]}"; do
memory_peak_bytes:$memory_peak_bytes,memory_max_bytes:$memory_max_bytes,
health_sha256:$health_sha256,
strict_trade_summary_readback:true,
strict_lob_continuity_readback:true,
lob_reconnect_boundaries:([$oss_round_trips[].lob_reconnect_boundary] | map(select(.)) | length),
min_lob_source_latency_ms:([$oss_round_trips[].lob_min_source_latency_ms] | min),
max_lob_source_latency_ms:([$oss_round_trips[].lob_max_source_latency_ms] | max),
min_lob_bid_levels:([$oss_round_trips[].lob_min_bid_levels] | min),
min_lob_ask_levels:([$oss_round_trips[].lob_min_ask_levels] | min),
max_segment_gap_ns:([$oss_round_trips[].gap_from_previous_ns] | max),
oss_roundtrips:($oss_round_trips | length),
agg_trade_segments:($oss_round_trips | length),
Expand All @@ -815,7 +877,7 @@ if [[ $test_only == true ]] || ((duration_seconds < REQUIRED_DURATION_SECONDS));
fi

jq -n \
--arg schema monday.rust_lob_shadow_gate.v2 \
--arg schema monday.rust_lob_shadow_gate.v3 \
--arg candidate_sha256 "$candidate_sha" \
--arg candidate_binary "$candidate_binary" \
--arg deployment_bundle_sha256 "$deployment_bundle_sha256" \
Expand Down
77 changes: 74 additions & 3 deletions deployment/aliyun/rust-lob-shadow-gate-policy.jq
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
.schema == "monday.rust_lob_shadow_gate.v2"
. as $gate
| .schema == "monday.rust_lob_shadow_gate.v3"
and .candidate_sha256 == $candidate_sha256
and .deployment_bundle_sha256 == $deployment_bundle_sha256
and .deployment_source_revision == $deployment_source_revision
Expand Down Expand Up @@ -37,6 +38,18 @@ and (.markets.spot.agg_trade_count | type) == "number"
and .markets.spot.agg_trade_count == (.markets.spot.agg_trade_count | floor)
and .markets.spot.agg_trade_count > 0
and .markets.spot.strict_trade_summary_readback == true
and .markets.spot.strict_lob_continuity_readback == true
and (.markets.spot.lob_reconnect_boundaries | type) == "number"
and .markets.spot.lob_reconnect_boundaries == 1
and (.markets.spot.min_lob_source_latency_ms | type) == "number"
and .markets.spot.min_lob_source_latency_ms >= -1000
and (.markets.spot.max_lob_source_latency_ms | type) == "number"
and .markets.spot.max_lob_source_latency_ms <= 30000
and .markets.spot.max_lob_source_latency_ms >= .markets.spot.min_lob_source_latency_ms
and (.markets.spot.min_lob_bid_levels | type) == "number"
and .markets.spot.min_lob_bid_levels > 0
and (.markets.spot.min_lob_ask_levels | type) == "number"
and .markets.spot.min_lob_ask_levels > 0
and (.markets.spot.max_segment_gap_ns | type) == "number"
and .markets.spot.max_segment_gap_ns == (.markets.spot.max_segment_gap_ns | floor)
and .markets.spot.max_segment_gap_ns >= 0
Expand All @@ -59,9 +72,32 @@ and all(.markets.spot.oss_roundtrip_evidence[];
and .end_received_at_ns >= .start_received_at_ns
and (.agg_trade_count | type) == "number"
and .agg_trade_count == (.agg_trade_count | floor)
and .agg_trade_count > 0)
and .agg_trade_count > 0
and .lob_capture_session_id == $gate.markets.spot.session_id
and (.lob_reconnect_boundary | type) == "boolean"
and .lob_sequence_gaps == 0
and .lob_source_time_rollbacks == 0
and .lob_declared_symbol_count == $gate.markets.spot.symbol_count
and .lob_covered_symbol_count == $gate.markets.spot.symbol_count
and (.lob_min_source_latency_ms | type) == "number"
and .lob_min_source_latency_ms >= -1000
and (.lob_max_source_latency_ms | type) == "number"
and .lob_max_source_latency_ms <= 30000
and .lob_max_source_latency_ms >= .lob_min_source_latency_ms
and (.lob_min_bid_levels | type) == "number"
and .lob_min_bid_levels > 0
and (.lob_min_ask_levels | type) == "number"
and .lob_min_ask_levels > 0)
and (.markets.spot.oss_roundtrip_evidence as $round_trips
| $round_trips[0].gap_from_previous_ns == 0
and $round_trips[0].lob_reconnect_boundary == true
and all($round_trips[1:][].lob_reconnect_boundary; . == false)
and .markets.spot.lob_reconnect_boundaries
== ([$round_trips[].lob_reconnect_boundary] | map(select(.)) | length)
and .markets.spot.min_lob_source_latency_ms == ([$round_trips[].lob_min_source_latency_ms] | min)
and .markets.spot.max_lob_source_latency_ms == ([$round_trips[].lob_max_source_latency_ms] | max)
and .markets.spot.min_lob_bid_levels == ([$round_trips[].lob_min_bid_levels] | min)
and .markets.spot.min_lob_ask_levels == ([$round_trips[].lob_min_ask_levels] | min)
and .markets.spot.max_segment_gap_ns == ([$round_trips[].gap_from_previous_ns] | max)
and all(range(1; ($round_trips | length));
$round_trips[.].start_received_at_ns >= $round_trips[. - 1].end_received_at_ns
Expand Down Expand Up @@ -96,6 +132,18 @@ and (.markets.usdm.agg_trade_count | type) == "number"
and .markets.usdm.agg_trade_count == (.markets.usdm.agg_trade_count | floor)
and .markets.usdm.agg_trade_count > 0
and .markets.usdm.strict_trade_summary_readback == true
and .markets.usdm.strict_lob_continuity_readback == true
and (.markets.usdm.lob_reconnect_boundaries | type) == "number"
and .markets.usdm.lob_reconnect_boundaries == 1
and (.markets.usdm.min_lob_source_latency_ms | type) == "number"
and .markets.usdm.min_lob_source_latency_ms >= -1000
and (.markets.usdm.max_lob_source_latency_ms | type) == "number"
and .markets.usdm.max_lob_source_latency_ms <= 30000
and .markets.usdm.max_lob_source_latency_ms >= .markets.usdm.min_lob_source_latency_ms
and (.markets.usdm.min_lob_bid_levels | type) == "number"
and .markets.usdm.min_lob_bid_levels > 0
and (.markets.usdm.min_lob_ask_levels | type) == "number"
and .markets.usdm.min_lob_ask_levels > 0
and (.markets.usdm.max_segment_gap_ns | type) == "number"
and .markets.usdm.max_segment_gap_ns == (.markets.usdm.max_segment_gap_ns | floor)
and .markets.usdm.max_segment_gap_ns >= 0
Expand All @@ -118,9 +166,32 @@ and all(.markets.usdm.oss_roundtrip_evidence[];
and .end_received_at_ns >= .start_received_at_ns
and (.agg_trade_count | type) == "number"
and .agg_trade_count == (.agg_trade_count | floor)
and .agg_trade_count > 0)
and .agg_trade_count > 0
and .lob_capture_session_id == $gate.markets.usdm.session_id
and (.lob_reconnect_boundary | type) == "boolean"
and .lob_sequence_gaps == 0
and .lob_source_time_rollbacks == 0
and .lob_declared_symbol_count == $gate.markets.usdm.symbol_count
and .lob_covered_symbol_count == $gate.markets.usdm.symbol_count
and (.lob_min_source_latency_ms | type) == "number"
and .lob_min_source_latency_ms >= -1000
and (.lob_max_source_latency_ms | type) == "number"
and .lob_max_source_latency_ms <= 30000
and .lob_max_source_latency_ms >= .lob_min_source_latency_ms
and (.lob_min_bid_levels | type) == "number"
and .lob_min_bid_levels > 0
and (.lob_min_ask_levels | type) == "number"
and .lob_min_ask_levels > 0)
and (.markets.usdm.oss_roundtrip_evidence as $round_trips
| $round_trips[0].gap_from_previous_ns == 0
and $round_trips[0].lob_reconnect_boundary == true
and all($round_trips[1:][].lob_reconnect_boundary; . == false)
and .markets.usdm.lob_reconnect_boundaries
== ([$round_trips[].lob_reconnect_boundary] | map(select(.)) | length)
and .markets.usdm.min_lob_source_latency_ms == ([$round_trips[].lob_min_source_latency_ms] | min)
and .markets.usdm.max_lob_source_latency_ms == ([$round_trips[].lob_max_source_latency_ms] | max)
and .markets.usdm.min_lob_bid_levels == ([$round_trips[].lob_min_bid_levels] | min)
and .markets.usdm.min_lob_ask_levels == ([$round_trips[].lob_min_ask_levels] | min)
and .markets.usdm.max_segment_gap_ns == ([$round_trips[].gap_from_previous_ns] | max)
and all(range(1; ($round_trips | length));
$round_trips[.].start_received_at_ns >= $round_trips[. - 1].end_received_at_ns
Expand Down
45 changes: 41 additions & 4 deletions deployment/aliyun/test-rust-lob-control-plane.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,15 @@ grep -Fq 'strict_verifier_args+=' "$GATE"
grep -Fq -- '--verify-segment' "$GATE"
grep -Fq -- '--segment-content-sha256' "$GATE"
grep -Fq -- '--segment-manifest-sha256' "$GATE"
grep -Fq -- '--require-lob-continuity' "$GATE"
grep -Fq '"$candidate_binary" "${strict_verifier_args[@]}"' "$GATE"
grep -Fq '.lob_continuity.contract == "binance.lob_continuity.v1"' "$GATE"
grep -Fq 'jq -e --arg session_id "${observed_session[$market]}"' "$GATE"
grep -Fq -- '--slurpfile manifest "$manifest_path"' "$GATE"
if grep -Fq -- '--argjson lob_continuity' "$GATE"; then
printf 'shadow gate passes the full-catalog LOB summary through argv\n' >&2
exit 1
fi
grep -Fq 'manifest changed between discovery and readback' "$GATE"
grep -Fq 'manifest_sha256:$manifest_sha256' "$GATE"

Expand Down Expand Up @@ -69,22 +77,40 @@ market_json=$(jq -cn \
session_id:"session-1",oss_roundtrips:2,
agg_trade_segments:2,agg_trade_count:2,
strict_trade_summary_readback:true,
strict_lob_continuity_readback:true,lob_reconnect_boundaries:1,
min_lob_source_latency_ms:0,max_lob_source_latency_ms:0,
min_lob_bid_levels:1,min_lob_ask_levels:1,
max_segment_gap_ns:0,
oss_roundtrip_evidence:[
{success_uri:"oss://bucket/part-1.jsonl.zst._SUCCESS",sha256:$catalog,manifest_sha256:$catalog,
gap_from_previous_ns:0,start_received_at_ns:100,end_received_at_ns:200,agg_trade_count:1},
gap_from_previous_ns:0,start_received_at_ns:100,end_received_at_ns:200,agg_trade_count:1,
lob_capture_session_id:"session-1",lob_reconnect_boundary:true,lob_sequence_gaps:0,
lob_source_time_rollbacks:0,lob_declared_symbol_count:1200,lob_covered_symbol_count:1200,
lob_min_source_latency_ms:0,lob_max_source_latency_ms:0,
lob_min_bid_levels:1,lob_min_ask_levels:1},
{success_uri:"oss://bucket/part-2.jsonl.zst._SUCCESS",sha256:$catalog,manifest_sha256:$catalog,
gap_from_previous_ns:0,start_received_at_ns:200,end_received_at_ns:300,agg_trade_count:1}
gap_from_previous_ns:0,start_received_at_ns:200,end_received_at_ns:300,agg_trade_count:1,
lob_capture_session_id:"session-1",lob_reconnect_boundary:false,lob_sequence_gaps:0,
lob_source_time_rollbacks:0,lob_declared_symbol_count:1200,lob_covered_symbol_count:1200,
lob_min_source_latency_ms:0,lob_max_source_latency_ms:0,
lob_min_bid_levels:1,lob_min_ask_levels:1}
]}')
usdm_market=$(jq -c '
.symbol_count = 500
| .snapshot_ready_count = 500
| .oss_roundtrip_evidence |= map(
.lob_declared_symbol_count = 500 | .lob_covered_symbol_count = 500)' \
<<<"$market_json")
jq -n \
--arg artifact "$artifact" \
--arg bundle "$bundle" \
--arg source "$source_revision" \
--argjson market "$market_json" \
'{schema:"monday.rust_lob_shadow_gate.v2",candidate_sha256:$artifact,
--argjson usdm_market "$usdm_market" \
'{schema:"monday.rust_lob_shadow_gate.v3",candidate_sha256:$artifact,
deployment_bundle_sha256:$bundle,deployment_source_revision:$source,
passed:true,production_eligible:true,checks_passed:true,duration_seconds:3600,
markets:{spot:$market,usdm:($market + {symbol_count:500,snapshot_ready_count:500})}}' \
markets:{spot:$market,usdm:$usdm_market}}' \
>"$tmp_dir/gate.json"

jq -e \
Expand All @@ -103,6 +129,17 @@ if jq -e \
exit 1
fi

jq '.markets.spot.oss_roundtrip_evidence[1].lob_capture_session_id = "session-2"' \
"$tmp_dir/gate.json" >"$tmp_dir/mixed-lob-session.json"
if jq -e \
--arg candidate_sha256 "$artifact" \
--arg deployment_bundle_sha256 "$bundle" \
--arg deployment_source_revision "$source_revision" \
-f "$POLICY" "$tmp_dir/mixed-lob-session.json" >/dev/null; then
printf 'gate policy accepted LOB evidence across a reconnect boundary\n' >&2
exit 1
fi

jq '.markets.spot.oss_roundtrip_evidence[1] |=
(.start_received_at_ns = 90000000300
| .end_received_at_ns = 90000000400
Expand Down
30 changes: 25 additions & 5 deletions deployment/aliyun/test-rust-lob-release-adoption.sh
Original file line number Diff line number Diff line change
Expand Up @@ -143,26 +143,46 @@ setup_fixture() {
catalog_sha256:"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
session_id:"shadow-session",oss_roundtrips:2,
agg_trade_segments:2,agg_trade_count:2,
strict_trade_summary_readback:true,max_segment_gap_ns:0,
strict_trade_summary_readback:true,strict_lob_continuity_readback:true,
lob_reconnect_boundaries:1,
min_lob_source_latency_ms:0,max_lob_source_latency_ms:0,
min_lob_bid_levels:1,min_lob_ask_levels:1,max_segment_gap_ns:0,
oss_roundtrip_evidence:[
{success_uri:"oss://bucket/part-1.jsonl.zst._SUCCESS",
sha256:"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
manifest_sha256:"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
gap_from_previous_ns:0,start_received_at_ns:100,end_received_at_ns:200,agg_trade_count:1},
gap_from_previous_ns:0,start_received_at_ns:100,end_received_at_ns:200,agg_trade_count:1,
lob_capture_session_id:"shadow-session",lob_reconnect_boundary:true,
lob_sequence_gaps:0,lob_source_time_rollbacks:0,
lob_declared_symbol_count:1200,lob_covered_symbol_count:1200,
lob_min_source_latency_ms:0,lob_max_source_latency_ms:0,
lob_min_bid_levels:1,lob_min_ask_levels:1},
{success_uri:"oss://bucket/part-2.jsonl.zst._SUCCESS",
sha256:"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
manifest_sha256:"9999999999999999999999999999999999999999999999999999999999999999",
gap_from_previous_ns:0,start_received_at_ns:200,end_received_at_ns:300,agg_trade_count:1}
gap_from_previous_ns:0,start_received_at_ns:200,end_received_at_ns:300,agg_trade_count:1,
lob_capture_session_id:"shadow-session",lob_reconnect_boundary:false,
lob_sequence_gaps:0,lob_source_time_rollbacks:0,
lob_declared_symbol_count:1200,lob_covered_symbol_count:1200,
lob_min_source_latency_ms:0,lob_max_source_latency_ms:0,
lob_min_bid_levels:1,lob_min_ask_levels:1}
]}')
usdm_market=$(jq -c '
.symbol_count = 500
| .snapshot_ready_count = 500
| .oss_roundtrip_evidence |= map(
.lob_declared_symbol_count = 500 | .lob_covered_symbol_count = 500)' \
<<<"$market")
jq -n \
--arg artifact "$CANDIDATE_SHA256" \
--arg bundle "$DEPLOYMENT_BUNDLE_SHA256" \
--arg source "$DEPLOYMENT_SOURCE_REVISION" \
--argjson market "$market" \
'{schema:"monday.rust_lob_shadow_gate.v2",candidate_sha256:$artifact,
--argjson usdm_market "$usdm_market" \
'{schema:"monday.rust_lob_shadow_gate.v3",candidate_sha256:$artifact,
deployment_bundle_sha256:$bundle,deployment_source_revision:$source,
passed:true,production_eligible:true,checks_passed:true,duration_seconds:3600,
markets:{spot:$market,usdm:($market + {symbol_count:500,snapshot_ready_count:500})}}' \
markets:{spot:$market,usdm:$usdm_market}}' \
>"$gate_dir/gate.json"
(cd "$gate_dir" && sha256sum gate.json >PASSED.sha256)
now_ns=$(($(date +%s) * 1000000000))
Expand Down
Loading
Loading