diff --git a/deployment/aliyun/host-rust-lob-shadow-gate.sh b/deployment/aliyun/host-rust-lob-shadow-gate.sh index aaba8f914..a0a604df4 100755 --- a/deployment/aliyun/host-rust-lob-shadow-gate.sh +++ b/deployment/aliyun/host-rust-lob-shadow-gate.sh @@ -611,7 +611,7 @@ verify_oss_round_trips() { local actual_digest bytes agg_trade_count manifest_agg_trade_count gap_ns local previous_end_ns=0 local round_trips='[]' - local -a strict_verifier_args=() + local -a strict_verifier_args=(--require-lob-continuity) manifest_uris "$market" "$listing" >"$uris" : >"$candidates" @@ -640,11 +640,47 @@ verify_oss_round_trips() { start_ns=$(jq -er '.start_received_at_ns' "$manifest") end_ns=$(jq -er '.end_received_at_ns' "$manifest") ((end_ns < gate_started_ns)) && continue - jq -e \ + jq -e --arg session_id "${observed_session[$market]}" \ '.schema == "binance.market_tape.v1" and .trade_summary_contract == "binance.aggregate_trade_summary.v1" and (.trade_summaries | type) == "object" and (.trade_summaries | length) > 0 + and .lob_continuity.contract == "binance.lob_continuity.v1" + and .lob_continuity.capture_session_id == $session_id + and (.lob_continuity.reconnect_boundary | type) == "boolean" + and .lob_continuity.sequence_gaps == 0 + and .lob_continuity.source_time_rollbacks == 0 + and .lob_continuity.declared_symbol_count == (.symbols | length) + and .lob_continuity.covered_symbol_count == (.symbols | length) + and .lob_continuity.missing_symbols == [] + and (.lob_continuity.symbols | type) == "object" + and (.lob_continuity.symbols | length) == (.symbols | length) + and all(.lob_continuity.symbols[]; + .snapshot_seed_count > 0 + and .diff_count > 0 + and .checkpoint_count > 0 + and (.first_update_id | type) == "number" + and (.last_update_id | type) == "number" + and .last_update_id >= .first_update_id + and (.first_source_time_ms | type) == "number" + and (.last_source_time_ms | type) == "number" + and .last_source_time_ms >= .first_source_time_ms + and (.first_received_at_ns | type) == "number" + and (.last_received_at_ns | type) == "number" + and .last_received_at_ns >= .first_received_at_ns + and (.min_source_latency_ms | type) == "number" + and (.max_source_latency_ms | type) == "number" + and .min_source_latency_ms >= -1000 + and .max_source_latency_ms <= 30000 + and .max_source_latency_ms >= .min_source_latency_ms + and (.min_bid_levels | type) == "number" + and (.max_bid_levels | type) == "number" + and .min_bid_levels > 0 + and .max_bid_levels >= .min_bid_levels + and (.min_ask_levels | type) == "number" + and (.max_ask_levels | type) == "number" + and .min_ask_levels > 0 + and .max_ask_levels >= .min_ask_levels) and (.event_types.agg_trade | type) == "number" and .event_types.agg_trade == (.event_types.agg_trade | floor) and .event_types.agg_trade > 0' \ @@ -740,16 +776,36 @@ verify_oss_round_trips() { --argjson gap_from_previous_ns "$gap_ns" \ --argjson bytes "$bytes" \ --argjson agg_trade_count "$agg_trade_count" \ - '{manifest_uri:$manifest_uri,data_uri:$data_uri,success_uri:$success_uri,sha256:$sha256, + --slurpfile manifest "$manifest_path" \ + '($manifest[0].lob_continuity) as $lob_continuity + | {manifest_uri:$manifest_uri,data_uri:$data_uri,success_uri:$success_uri,sha256:$sha256, manifest_sha256:$manifest_sha256,gap_from_previous_ns:$gap_from_previous_ns, start_received_at_ns:$start_received_at_ns,end_received_at_ns:$end_received_at_ns,bytes:$bytes, - agg_trade_count:$agg_trade_count}') + agg_trade_count:$agg_trade_count, + lob_capture_session_id:$lob_continuity.capture_session_id, + lob_reconnect_boundary:$lob_continuity.reconnect_boundary, + lob_sequence_gaps:$lob_continuity.sequence_gaps, + lob_source_time_rollbacks:$lob_continuity.source_time_rollbacks, + lob_declared_symbol_count:$lob_continuity.declared_symbol_count, + lob_covered_symbol_count:$lob_continuity.covered_symbol_count, + lob_min_source_latency_ms:([$lob_continuity.symbols[].min_source_latency_ms] | min), + lob_max_source_latency_ms:([$lob_continuity.symbols[].max_source_latency_ms] | max), + lob_min_bid_levels:([$lob_continuity.symbols[].min_bid_levels] | min), + lob_min_ask_levels:([$lob_continuity.symbols[].min_ask_levels] | min)}') round_trips=$(jq -cn --argjson values "$round_trips" --argjson value "$round_trip" \ '$values + [$value]') done < <(sort -n -k1,1 "$candidates") "$candidate_binary" "${strict_verifier_args[@]}" >/dev/null \ - || die "$market strict aggregate-trade summary readback failed" + || die "$market strict aggregate-trade and LOB continuity readback failed" + + jq -e --arg session_id "${observed_session[$market]}" ' + (map(select(.lob_reconnect_boundary)) | length) == 1 + and .[0].lob_reconnect_boundary == true + and all(.[1:][].lob_reconnect_boundary; . == false) + and all(.[].lob_capture_session_id; . == $session_id)' \ + <<<"$round_trips" >/dev/null \ + || die "$market LOB evidence crosses a capture-session boundary" printf '%s\n' "$round_trips" } @@ -794,6 +850,12 @@ for market in "${markets[@]}"; do memory_peak_bytes:$memory_peak_bytes,memory_max_bytes:$memory_max_bytes, health_sha256:$health_sha256, strict_trade_summary_readback:true, + strict_lob_continuity_readback:true, + lob_reconnect_boundaries:([$oss_round_trips[].lob_reconnect_boundary] | map(select(.)) | length), + min_lob_source_latency_ms:([$oss_round_trips[].lob_min_source_latency_ms] | min), + max_lob_source_latency_ms:([$oss_round_trips[].lob_max_source_latency_ms] | max), + min_lob_bid_levels:([$oss_round_trips[].lob_min_bid_levels] | min), + min_lob_ask_levels:([$oss_round_trips[].lob_min_ask_levels] | min), max_segment_gap_ns:([$oss_round_trips[].gap_from_previous_ns] | max), oss_roundtrips:($oss_round_trips | length), agg_trade_segments:($oss_round_trips | length), @@ -815,7 +877,7 @@ if [[ $test_only == true ]] || ((duration_seconds < REQUIRED_DURATION_SECONDS)); fi jq -n \ - --arg schema monday.rust_lob_shadow_gate.v2 \ + --arg schema monday.rust_lob_shadow_gate.v3 \ --arg candidate_sha256 "$candidate_sha" \ --arg candidate_binary "$candidate_binary" \ --arg deployment_bundle_sha256 "$deployment_bundle_sha256" \ diff --git a/deployment/aliyun/rust-lob-shadow-gate-policy.jq b/deployment/aliyun/rust-lob-shadow-gate-policy.jq index 57bead25b..6f203d7ce 100644 --- a/deployment/aliyun/rust-lob-shadow-gate-policy.jq +++ b/deployment/aliyun/rust-lob-shadow-gate-policy.jq @@ -1,4 +1,5 @@ -.schema == "monday.rust_lob_shadow_gate.v2" +. as $gate +| .schema == "monday.rust_lob_shadow_gate.v3" and .candidate_sha256 == $candidate_sha256 and .deployment_bundle_sha256 == $deployment_bundle_sha256 and .deployment_source_revision == $deployment_source_revision @@ -37,6 +38,18 @@ and (.markets.spot.agg_trade_count | type) == "number" and .markets.spot.agg_trade_count == (.markets.spot.agg_trade_count | floor) and .markets.spot.agg_trade_count > 0 and .markets.spot.strict_trade_summary_readback == true +and .markets.spot.strict_lob_continuity_readback == true +and (.markets.spot.lob_reconnect_boundaries | type) == "number" +and .markets.spot.lob_reconnect_boundaries == 1 +and (.markets.spot.min_lob_source_latency_ms | type) == "number" +and .markets.spot.min_lob_source_latency_ms >= -1000 +and (.markets.spot.max_lob_source_latency_ms | type) == "number" +and .markets.spot.max_lob_source_latency_ms <= 30000 +and .markets.spot.max_lob_source_latency_ms >= .markets.spot.min_lob_source_latency_ms +and (.markets.spot.min_lob_bid_levels | type) == "number" +and .markets.spot.min_lob_bid_levels > 0 +and (.markets.spot.min_lob_ask_levels | type) == "number" +and .markets.spot.min_lob_ask_levels > 0 and (.markets.spot.max_segment_gap_ns | type) == "number" and .markets.spot.max_segment_gap_ns == (.markets.spot.max_segment_gap_ns | floor) and .markets.spot.max_segment_gap_ns >= 0 @@ -59,9 +72,32 @@ and all(.markets.spot.oss_roundtrip_evidence[]; and .end_received_at_ns >= .start_received_at_ns and (.agg_trade_count | type) == "number" and .agg_trade_count == (.agg_trade_count | floor) - and .agg_trade_count > 0) + and .agg_trade_count > 0 + and .lob_capture_session_id == $gate.markets.spot.session_id + and (.lob_reconnect_boundary | type) == "boolean" + and .lob_sequence_gaps == 0 + and .lob_source_time_rollbacks == 0 + and .lob_declared_symbol_count == $gate.markets.spot.symbol_count + and .lob_covered_symbol_count == $gate.markets.spot.symbol_count + and (.lob_min_source_latency_ms | type) == "number" + and .lob_min_source_latency_ms >= -1000 + and (.lob_max_source_latency_ms | type) == "number" + and .lob_max_source_latency_ms <= 30000 + and .lob_max_source_latency_ms >= .lob_min_source_latency_ms + and (.lob_min_bid_levels | type) == "number" + and .lob_min_bid_levels > 0 + and (.lob_min_ask_levels | type) == "number" + and .lob_min_ask_levels > 0) and (.markets.spot.oss_roundtrip_evidence as $round_trips | $round_trips[0].gap_from_previous_ns == 0 + and $round_trips[0].lob_reconnect_boundary == true + and all($round_trips[1:][].lob_reconnect_boundary; . == false) + and .markets.spot.lob_reconnect_boundaries + == ([$round_trips[].lob_reconnect_boundary] | map(select(.)) | length) + and .markets.spot.min_lob_source_latency_ms == ([$round_trips[].lob_min_source_latency_ms] | min) + and .markets.spot.max_lob_source_latency_ms == ([$round_trips[].lob_max_source_latency_ms] | max) + and .markets.spot.min_lob_bid_levels == ([$round_trips[].lob_min_bid_levels] | min) + and .markets.spot.min_lob_ask_levels == ([$round_trips[].lob_min_ask_levels] | min) and .markets.spot.max_segment_gap_ns == ([$round_trips[].gap_from_previous_ns] | max) and all(range(1; ($round_trips | length)); $round_trips[.].start_received_at_ns >= $round_trips[. - 1].end_received_at_ns @@ -96,6 +132,18 @@ and (.markets.usdm.agg_trade_count | type) == "number" and .markets.usdm.agg_trade_count == (.markets.usdm.agg_trade_count | floor) and .markets.usdm.agg_trade_count > 0 and .markets.usdm.strict_trade_summary_readback == true +and .markets.usdm.strict_lob_continuity_readback == true +and (.markets.usdm.lob_reconnect_boundaries | type) == "number" +and .markets.usdm.lob_reconnect_boundaries == 1 +and (.markets.usdm.min_lob_source_latency_ms | type) == "number" +and .markets.usdm.min_lob_source_latency_ms >= -1000 +and (.markets.usdm.max_lob_source_latency_ms | type) == "number" +and .markets.usdm.max_lob_source_latency_ms <= 30000 +and .markets.usdm.max_lob_source_latency_ms >= .markets.usdm.min_lob_source_latency_ms +and (.markets.usdm.min_lob_bid_levels | type) == "number" +and .markets.usdm.min_lob_bid_levels > 0 +and (.markets.usdm.min_lob_ask_levels | type) == "number" +and .markets.usdm.min_lob_ask_levels > 0 and (.markets.usdm.max_segment_gap_ns | type) == "number" and .markets.usdm.max_segment_gap_ns == (.markets.usdm.max_segment_gap_ns | floor) and .markets.usdm.max_segment_gap_ns >= 0 @@ -118,9 +166,32 @@ and all(.markets.usdm.oss_roundtrip_evidence[]; and .end_received_at_ns >= .start_received_at_ns and (.agg_trade_count | type) == "number" and .agg_trade_count == (.agg_trade_count | floor) - and .agg_trade_count > 0) + and .agg_trade_count > 0 + and .lob_capture_session_id == $gate.markets.usdm.session_id + and (.lob_reconnect_boundary | type) == "boolean" + and .lob_sequence_gaps == 0 + and .lob_source_time_rollbacks == 0 + and .lob_declared_symbol_count == $gate.markets.usdm.symbol_count + and .lob_covered_symbol_count == $gate.markets.usdm.symbol_count + and (.lob_min_source_latency_ms | type) == "number" + and .lob_min_source_latency_ms >= -1000 + and (.lob_max_source_latency_ms | type) == "number" + and .lob_max_source_latency_ms <= 30000 + and .lob_max_source_latency_ms >= .lob_min_source_latency_ms + and (.lob_min_bid_levels | type) == "number" + and .lob_min_bid_levels > 0 + and (.lob_min_ask_levels | type) == "number" + and .lob_min_ask_levels > 0) and (.markets.usdm.oss_roundtrip_evidence as $round_trips | $round_trips[0].gap_from_previous_ns == 0 + and $round_trips[0].lob_reconnect_boundary == true + and all($round_trips[1:][].lob_reconnect_boundary; . == false) + and .markets.usdm.lob_reconnect_boundaries + == ([$round_trips[].lob_reconnect_boundary] | map(select(.)) | length) + and .markets.usdm.min_lob_source_latency_ms == ([$round_trips[].lob_min_source_latency_ms] | min) + and .markets.usdm.max_lob_source_latency_ms == ([$round_trips[].lob_max_source_latency_ms] | max) + and .markets.usdm.min_lob_bid_levels == ([$round_trips[].lob_min_bid_levels] | min) + and .markets.usdm.min_lob_ask_levels == ([$round_trips[].lob_min_ask_levels] | min) and .markets.usdm.max_segment_gap_ns == ([$round_trips[].gap_from_previous_ns] | max) and all(range(1; ($round_trips | length)); $round_trips[.].start_received_at_ns >= $round_trips[. - 1].end_received_at_ns diff --git a/deployment/aliyun/test-rust-lob-control-plane.sh b/deployment/aliyun/test-rust-lob-control-plane.sh index 90a5e7c94..4b71b52d7 100755 --- a/deployment/aliyun/test-rust-lob-control-plane.sh +++ b/deployment/aliyun/test-rust-lob-control-plane.sh @@ -25,7 +25,15 @@ grep -Fq 'strict_verifier_args+=' "$GATE" grep -Fq -- '--verify-segment' "$GATE" grep -Fq -- '--segment-content-sha256' "$GATE" grep -Fq -- '--segment-manifest-sha256' "$GATE" +grep -Fq -- '--require-lob-continuity' "$GATE" grep -Fq '"$candidate_binary" "${strict_verifier_args[@]}"' "$GATE" +grep -Fq '.lob_continuity.contract == "binance.lob_continuity.v1"' "$GATE" +grep -Fq 'jq -e --arg session_id "${observed_session[$market]}"' "$GATE" +grep -Fq -- '--slurpfile manifest "$manifest_path"' "$GATE" +if grep -Fq -- '--argjson lob_continuity' "$GATE"; then + printf 'shadow gate passes the full-catalog LOB summary through argv\n' >&2 + exit 1 +fi grep -Fq 'manifest changed between discovery and readback' "$GATE" grep -Fq 'manifest_sha256:$manifest_sha256' "$GATE" @@ -69,22 +77,40 @@ market_json=$(jq -cn \ session_id:"session-1",oss_roundtrips:2, agg_trade_segments:2,agg_trade_count:2, strict_trade_summary_readback:true, + strict_lob_continuity_readback:true,lob_reconnect_boundaries:1, + min_lob_source_latency_ms:0,max_lob_source_latency_ms:0, + min_lob_bid_levels:1,min_lob_ask_levels:1, max_segment_gap_ns:0, oss_roundtrip_evidence:[ {success_uri:"oss://bucket/part-1.jsonl.zst._SUCCESS",sha256:$catalog,manifest_sha256:$catalog, - gap_from_previous_ns:0,start_received_at_ns:100,end_received_at_ns:200,agg_trade_count:1}, + gap_from_previous_ns:0,start_received_at_ns:100,end_received_at_ns:200,agg_trade_count:1, + lob_capture_session_id:"session-1",lob_reconnect_boundary:true,lob_sequence_gaps:0, + lob_source_time_rollbacks:0,lob_declared_symbol_count:1200,lob_covered_symbol_count:1200, + lob_min_source_latency_ms:0,lob_max_source_latency_ms:0, + lob_min_bid_levels:1,lob_min_ask_levels:1}, {success_uri:"oss://bucket/part-2.jsonl.zst._SUCCESS",sha256:$catalog,manifest_sha256:$catalog, - gap_from_previous_ns:0,start_received_at_ns:200,end_received_at_ns:300,agg_trade_count:1} + gap_from_previous_ns:0,start_received_at_ns:200,end_received_at_ns:300,agg_trade_count:1, + lob_capture_session_id:"session-1",lob_reconnect_boundary:false,lob_sequence_gaps:0, + lob_source_time_rollbacks:0,lob_declared_symbol_count:1200,lob_covered_symbol_count:1200, + lob_min_source_latency_ms:0,lob_max_source_latency_ms:0, + lob_min_bid_levels:1,lob_min_ask_levels:1} ]}') +usdm_market=$(jq -c ' + .symbol_count = 500 + | .snapshot_ready_count = 500 + | .oss_roundtrip_evidence |= map( + .lob_declared_symbol_count = 500 | .lob_covered_symbol_count = 500)' \ + <<<"$market_json") jq -n \ --arg artifact "$artifact" \ --arg bundle "$bundle" \ --arg source "$source_revision" \ --argjson market "$market_json" \ - '{schema:"monday.rust_lob_shadow_gate.v2",candidate_sha256:$artifact, + --argjson usdm_market "$usdm_market" \ + '{schema:"monday.rust_lob_shadow_gate.v3",candidate_sha256:$artifact, deployment_bundle_sha256:$bundle,deployment_source_revision:$source, passed:true,production_eligible:true,checks_passed:true,duration_seconds:3600, - markets:{spot:$market,usdm:($market + {symbol_count:500,snapshot_ready_count:500})}}' \ + markets:{spot:$market,usdm:$usdm_market}}' \ >"$tmp_dir/gate.json" jq -e \ @@ -103,6 +129,17 @@ if jq -e \ exit 1 fi +jq '.markets.spot.oss_roundtrip_evidence[1].lob_capture_session_id = "session-2"' \ + "$tmp_dir/gate.json" >"$tmp_dir/mixed-lob-session.json" +if jq -e \ + --arg candidate_sha256 "$artifact" \ + --arg deployment_bundle_sha256 "$bundle" \ + --arg deployment_source_revision "$source_revision" \ + -f "$POLICY" "$tmp_dir/mixed-lob-session.json" >/dev/null; then + printf 'gate policy accepted LOB evidence across a reconnect boundary\n' >&2 + exit 1 +fi + jq '.markets.spot.oss_roundtrip_evidence[1] |= (.start_received_at_ns = 90000000300 | .end_received_at_ns = 90000000400 diff --git a/deployment/aliyun/test-rust-lob-release-adoption.sh b/deployment/aliyun/test-rust-lob-release-adoption.sh index 713e91cd0..fa2645c00 100755 --- a/deployment/aliyun/test-rust-lob-release-adoption.sh +++ b/deployment/aliyun/test-rust-lob-release-adoption.sh @@ -143,26 +143,46 @@ setup_fixture() { catalog_sha256:"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", session_id:"shadow-session",oss_roundtrips:2, agg_trade_segments:2,agg_trade_count:2, - strict_trade_summary_readback:true,max_segment_gap_ns:0, + strict_trade_summary_readback:true,strict_lob_continuity_readback:true, + lob_reconnect_boundaries:1, + min_lob_source_latency_ms:0,max_lob_source_latency_ms:0, + min_lob_bid_levels:1,min_lob_ask_levels:1,max_segment_gap_ns:0, oss_roundtrip_evidence:[ {success_uri:"oss://bucket/part-1.jsonl.zst._SUCCESS", sha256:"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", manifest_sha256:"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - gap_from_previous_ns:0,start_received_at_ns:100,end_received_at_ns:200,agg_trade_count:1}, + gap_from_previous_ns:0,start_received_at_ns:100,end_received_at_ns:200,agg_trade_count:1, + lob_capture_session_id:"shadow-session",lob_reconnect_boundary:true, + lob_sequence_gaps:0,lob_source_time_rollbacks:0, + lob_declared_symbol_count:1200,lob_covered_symbol_count:1200, + lob_min_source_latency_ms:0,lob_max_source_latency_ms:0, + lob_min_bid_levels:1,lob_min_ask_levels:1}, {success_uri:"oss://bucket/part-2.jsonl.zst._SUCCESS", sha256:"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", manifest_sha256:"9999999999999999999999999999999999999999999999999999999999999999", - gap_from_previous_ns:0,start_received_at_ns:200,end_received_at_ns:300,agg_trade_count:1} + gap_from_previous_ns:0,start_received_at_ns:200,end_received_at_ns:300,agg_trade_count:1, + lob_capture_session_id:"shadow-session",lob_reconnect_boundary:false, + lob_sequence_gaps:0,lob_source_time_rollbacks:0, + lob_declared_symbol_count:1200,lob_covered_symbol_count:1200, + lob_min_source_latency_ms:0,lob_max_source_latency_ms:0, + lob_min_bid_levels:1,lob_min_ask_levels:1} ]}') + usdm_market=$(jq -c ' + .symbol_count = 500 + | .snapshot_ready_count = 500 + | .oss_roundtrip_evidence |= map( + .lob_declared_symbol_count = 500 | .lob_covered_symbol_count = 500)' \ + <<<"$market") jq -n \ --arg artifact "$CANDIDATE_SHA256" \ --arg bundle "$DEPLOYMENT_BUNDLE_SHA256" \ --arg source "$DEPLOYMENT_SOURCE_REVISION" \ --argjson market "$market" \ - '{schema:"monday.rust_lob_shadow_gate.v2",candidate_sha256:$artifact, + --argjson usdm_market "$usdm_market" \ + '{schema:"monday.rust_lob_shadow_gate.v3",candidate_sha256:$artifact, deployment_bundle_sha256:$bundle,deployment_source_revision:$source, passed:true,production_eligible:true,checks_passed:true,duration_seconds:3600, - markets:{spot:$market,usdm:($market + {symbol_count:500,snapshot_ready_count:500})}}' \ + markets:{spot:$market,usdm:$usdm_market}}' \ >"$gate_dir/gate.json" (cd "$gate_dir" && sha256sum gate.json >PASSED.sha256) now_ns=$(($(date +%s) * 1000000000)) diff --git a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs index 4bfa734bf..4951e3813 100644 --- a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs +++ b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs @@ -5,8 +5,10 @@ use data::binance_market_tape::{ DepthSourceClockSequenceValidator, }; use data::binance_market_tape_artifact::{ - seal_binance_market_tape_triplet, verify_binance_market_tape_with_required_trade_summaries, - BinanceMarketTapeTriplet, BinanceMarketTapeTrustAnchor, + seal_binance_market_tape_triplet, + verify_binance_market_tape_with_required_trade_and_lob_summaries, + verify_binance_market_tape_with_required_trade_summaries, BinanceMarketTapeTriplet, + BinanceMarketTapeTrustAnchor, }; use futures::StreamExt; use hft_collector::lob_archiver::{ @@ -51,6 +53,9 @@ struct Args { #[arg(long, requires = "verify_segment")] segment_manifest_sha256: Vec, + + #[arg(long, requires = "verify_segment")] + require_lob_continuity: bool, } #[derive(Debug, Clone)] @@ -636,7 +641,11 @@ fn verify_segments(args: &Args) -> anyhow::Result<()> { let trust = BinanceMarketTapeTrustAnchor::from_lower_hex(content_sha256, manifest_sha256)?; sealed.push(seal_binance_market_tape_triplet(&triplet, &trust)?); } - let verified = verify_binance_market_tape_with_required_trade_summaries(sealed)?; + let verified = if args.require_lob_continuity { + verify_binance_market_tape_with_required_trade_and_lob_summaries(sealed)? + } else { + verify_binance_market_tape_with_required_trade_summaries(sealed)? + }; println!( "strict market-tape verification: ok ({} segments)", verified.segments().len() @@ -2033,6 +2042,25 @@ mod tests { .is_err()); } + #[test] + fn strict_lob_continuity_is_an_explicit_segment_verifier_mode() { + let args = Args::try_parse_from([ + "binance-lob-archiver", + "--verify-segment", + "/tmp/part.jsonl.zst", + "--segment-content-sha256", + &"a".repeat(64), + "--segment-manifest-sha256", + &"b".repeat(64), + "--require-lob-continuity", + ]) + .unwrap(); + assert!(args.require_lob_continuity); + assert!( + Args::try_parse_from(["binance-lob-archiver", "--require-lob-continuity",]).is_err() + ); + } + #[tokio::test] async fn upload_only_rejects_a_missing_spool() { let spool_dir =