diff --git a/benchmarks/ws-head-latency-base.yml b/benchmarks/ws-head-latency-base.yml index 052d70a3..dbe4f812 100644 --- a/benchmarks/ws-head-latency-base.yml +++ b/benchmarks/ws-head-latency-base.yml @@ -62,6 +62,15 @@ prometheus: window: 24h freshness_metric: ws_head_blocks_seen_total +rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="base"}[24h]))) + +dimensions: + region: + - { value: all, label: All regions } + - { value: us-east, label: US-East } + - { value: eu-west, label: EU-West } + - { value: sgp, label: Singapore } + faq: - q: "What is the fastest free Base RPC over WebSocket right now?" a: "{{best_name}} currently wins most per-block races on Base with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Base block, roughly every 2 seconds, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive Base bot to any single provider." @@ -87,6 +96,16 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="base"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="base"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="base"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="base", region="sgp"}[1h]))) - slug: drpc name: dRPC @@ -100,3 +119,13 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="drpc", chain="base"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="base"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="drpc", chain="base"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="base", region="sgp"}[1h]))) diff --git a/benchmarks/ws-head-latency-solana.yml b/benchmarks/ws-head-latency-solana.yml index c906903b..cc9ee17c 100644 --- a/benchmarks/ws-head-latency-solana.yml +++ b/benchmarks/ws-head-latency-solana.yml @@ -63,6 +63,15 @@ prometheus: window: 24h freshness_metric: ws_head_blocks_seen_total +rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="solana"}[24h]))) + +dimensions: + region: + - { value: all, label: All regions } + - { value: us-east, label: US-East } + - { value: eu-west, label: EU-West } + - { value: sgp, label: Singapore } + faq: - q: "What is the fastest free Solana RPC over WebSocket right now?" a: "{{best_name}} currently wins most per-slot races on Solana with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Solana slot, roughly every 400ms, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive Solana keeper or MEV integration to any single provider." @@ -90,6 +99,16 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="solana"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="solana"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="solana"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="solana", region="sgp"}[1h]))) - slug: solana-labs name: Solana Foundation @@ -103,3 +122,13 @@ providers: success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="solana-labs", chain="solana"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="solana"}[24h]))), 1) sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="solana-labs", chain="solana"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="solana-labs", chain="solana", region="sgp"}[1h]))) diff --git a/benchmarks/ws-head-latency.yml b/benchmarks/ws-head-latency.yml new file mode 100644 index 00000000..7b66ea78 --- /dev/null +++ b/benchmarks/ws-head-latency.yml @@ -0,0 +1,169 @@ +# OpenChainBench. Bench № 081 + +slug: ws-head-latency +number: "081" +title: Fastest WebSocket newHeads push, live block-push lag across RPC providers +seo_title: "Fastest WebSocket RPC newHeads push 2026" +seo_description: "{{best_name}} pushes Ethereum newHeads first. Live WebSocket block-push lag for PublicNode, dRPC and Tenderly, measured per block against the earliest arrival in the cohort." +subtitle: Per-block WebSocket push lag in milliseconds versus the earliest provider to deliver the same Ethereum head, measured continuously from one eu-west vantage point. + +category: RPCs +status: live +metric: Block push lag +unit: ms +higher_is_better: false + +seo_intro: | + Every trading bot, MEV searcher, liquidation keeper and live dashboard + that subscribes to `eth_subscribe("newHeads")` cares about one number + first: how many milliseconds after the network produces a block does my + RPC provider actually push it to me? Marketing pages say "real-time + WebSocket streaming"; this page says who delivers block N first and by + how much the others trail. The harness holds one persistent WebSocket + per provider from the same eu-west host, timestamps every newHeads + frame on receipt, and scores each Ethereum block as a race: the + earliest arrival sets T0, every other provider's lag is its arrival + minus T0. The headline is pinned to Ethereum, where all three keyless + providers compete; the same harness also races Base heads and Solana + slots for cross-chain context. Because the measurement is relative, + the leader reads near zero by construction, so the win-rate and the + shape of the trailing distribution (p50 vs p99) are the numbers to + read, not the absolute magnitude. {{best_name}} currently leads at + {{best_p50}} (p50, 24 h) across {{count}} measured providers. + +abstract: | + We race RPC providers on WebSocket block delivery. One harness holds a + persistent `eth_subscribe("newHeads")` connection to each provider from + the same host and timestamps every head frame on receipt. Per block, + the earliest arrival across providers defines T0; each provider's + sample is its own arrival minus T0, in milliseconds. A block's cohort + stays open 5 seconds after the first arrival, then it is scored: + lag histogram per provider, a win for the earliest, and a gap counter + for any live provider that never delivered the block. The measurement + is relative to the fastest provider observed from our vantage point, + not an absolute network latency; the same-path subtraction is exactly + what makes the ordering fair. Solana runs the identical race on + `slotSubscribe` slot notifications. + +methodology: + - "Providers measured (keyless WebSocket endpoints): PublicNode (wss://ethereum-rpc.publicnode.com), dRPC (wss://eth.drpc.org), Tenderly (wss://mainnet.gateway.tenderly.co). Keyed providers (Alchemy, Infura, Chainstack) join the cohort when contributor keys are wired; the harness reads them from env vars and skips them cleanly when unset." + - "Chains: Ethereum (headline, all three providers compete), Base via newHeads and Solana via slotSubscribe as secondary cohorts. Headline queries on this page are pinned to chain=ethereum so the ranking never mixes block cadences." + - "Race scoring: the first provider to deliver block N sets T0. Each provider's sample for block N is arrival(N) minus T0 in milliseconds. The cohort closes 5 seconds after the first arrival; a provider arriving later than that is scored as a missed block (ws_block_gap_total), not as a huge latency sample." + - "Relative, not absolute: a single vantage point cannot separate its own network path from provider pipeline time, so we subtract the two arrivals over the same path instead. Consequence: the fastest provider reads ~0 by construction, and the honest readings are win rate, the trailers' lag distribution, and gap counts, not the leader's absolute number." + - "Blocks where only one provider delivered within the window emit no lag samples and no win: a one-horse race carries no relative information. Gap counters still increment for live providers that missed the block." + - "Connection discipline: one persistent WebSocket per (provider, chain), client ping every 30s, read deadline 60s, plus a 90s head watchdog that forces a reconnect when a provider keeps the socket heartbeat alive but silently drops the subscription (a failure mode we have observed in production on other benches). Reconnects use exponential backoff 2s to 60s and are counted in ws_reconnects_total." + - "Coverage score: ws_head_blocks_seen_total per provider divided by the cohort maximum over the same 24h window, clamped to 1. A provider that pushes fast but skips blocks loses on this column rather than hiding in the latency percentiles." + - "Aggregation: per-provider lag histogram (5ms to 10s exponential buckets), p50/p90/p99 via histogram_quantile over the 24h rate. Duplicate announcements of the same block on one connection are dropped; per-provider liveness is exported as ws_health (1 = connected and delivered a message in the last 120s)." + - "Cheat vector disclosure: a provider could theoretically pre-announce heads it has not fully validated to win the race. We keyed the cohort on block number as pushed; hash-level cross-checking against a reference node is on the v2 list. Reading win rate together with the gap counter already flags a provider that pushes early but wrong (retracted heads show up as cohort inconsistencies and reorg noise)." + - "Vantage point: a single eu-west host. Geography moves absolute arrival times but affects all providers over the same path; still, a provider whose nearest edge is far from eu-west is disadvantaged, and a us-east probe is the planned second vantage before any cross-region claim is made." + +findings: + - "{{best_name}} leads the Ethereum cohort at {{best_p50}} (p50, 24 h) across {{count}} measured providers. Remember the construction: the per-block winner defines zero, so the leader's near-zero p50 mostly says it wins most races; the informative numbers are the trailers' medians and the p99 tails." + - "{{name:drpc}} trails the winner by {{p50:drpc}} at the median. That is the extra time a liquidation bot or MEV searcher on dRPC's keyless endpoint waits for block N after the fastest feed already delivered it." + - "{{name:tenderly}} trails by {{p50:tenderly}} (p50, 24 h). Tenderly's gateway is built around simulation and developer tooling rather than raw head-push fan-out, and the push pipeline depth shows up in this race." + - "The p99 column is the integration-grade number: {{p99:publicnode}} / {{p99:drpc}} / {{p99:tenderly}}. A provider with a tight p50 but a multi-second p99 will occasionally hand your bot a stale head exactly when the chain is busiest." + - "Win rate and block coverage are the robust readings. A provider can look decent on median lag while silently skipping blocks; the success column (blocks seen vs cohort max) and ws_block_gap_total keep that visible." + +source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/ws-head-latency + +expected_n: 7000 + +prometheus: + window: 24h + freshness_metric: ws_head_blocks_seen_total + +# Per-cell (region) ranking matrix for scoped badge claims. Chain is +# fixed for the whole bench, so cells key on provider and region. +rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(ws_head_lag_milliseconds_bucket{chain="ethereum"}[24h]))) + +# Region is the only dimension: chain is baked into every query. +dimensions: + region: + - { value: all, label: All regions } + - { value: us-east, label: US-East } + - { value: eu-west, label: EU-West } + - { value: sgp, label: Singapore } + +faq: + - q: "Which RPC provider pushes new Ethereum blocks first over WebSocket?" + a: "{{best_name}} currently wins most per-block races, with a median lag of {{best_p50}} versus the earliest arrival (p50, 24 h) across {{count}} measured keyless providers. The leaderboard re-scores on every Ethereum block, roughly every 12 seconds, so the answer reflects sustained live measurement, not a one-off test. Check the win rate and the trailers' p99 before wiring a latency-sensitive bot to any single provider." + - q: "Why does the fastest provider show a lag near zero?" + a: "By construction. Each block is scored as a race: the earliest arrival among providers defines T0 and every sample is measured against it, so the provider that wins most races accumulates mostly zero samples. This is deliberate. A single vantage point cannot measure absolute chain-to-client latency without a reference node co-located with block producers; it can measure, fairly, which provider delivers the same block earlier over the same network path. Read the leader's win rate and the trailers' distributions, not the leader's absolute number." + - q: "Is this absolute latency from block production to my client?" + a: "No. The number is relative lag versus the fastest provider observed from our eu-west host. Absolute delivery latency includes the propagation time from the block producer to each provider's infrastructure plus the path to you, which shifts with your geography. What transfers to your integration is the ordering and the spread: if a provider trails the winner by 800 ms at the median from our vantage point, its pipeline is structurally behind, and that gap does not disappear because you probe from another region." + - q: "Could a provider cheat by pushing heads before validating them?" + a: "In principle yes: pre-announcing an unvalidated head wins the race at the cost of occasionally pushing a block that gets reorged or retracted. That is why the bench exports more than the lag histogram. A provider that pushes early but wrong shows up as cohort inconsistencies, elevated reorg noise and gaps, and the coverage column (blocks seen versus the cohort maximum) penalises skipped or retracted blocks. Hash-level cross-checking against an independent reference node is planned for v2 of the harness." + - q: "How is WebSocket head lag measured on OpenChainBench?" + a: "One Go harness holds a persistent eth_subscribe('newHeads') WebSocket per provider from the same eu-west host, plus slotSubscribe for the Solana cohort. Every frame is timestamped on receipt with time.Now() precision. A block's cohort stays open for 5 seconds after the first arrival, then each provider's lag versus the earliest is observed into a Prometheus histogram (5 ms to 10 s exponential buckets); p50/p90/p99 come from histogram_quantile over the 24 h window. Connections carry a 90 s silent-subscription watchdog and exponential-backoff reconnects so a flaky endpoint degrades to health=0 instead of corrupting the race." + - q: "Why are only PublicNode, dRPC and Tenderly listed?" + a: "V1 measures the keyless tier: endpoints anyone can open a WebSocket to without an account, which is also the tier where claims are hardest to verify and marketing is loudest. Alchemy, Infura and Chainstack require API keys; the harness already accepts them via environment variables and they join the leaderboard once contributor keys are wired into the deployment. Keyed tiers usually run on better-provisioned infrastructure, so expect the keyless numbers here to be the floor, not the ceiling, of each vendor's performance." + +providers: + - slug: publicnode + name: PublicNode + tag: Keyless WS, wss://ethereum-rpc.publicnode.com + formula: "Median ms behind the earliest provider to push each Ethereum newHeads frame, from one persistent WebSocket per provider on the same eu-west host, histogram_quantile over 24h. The per-block winner scores 0 by construction." + queries: + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[24h]))) + p90: histogram_quantile(0.90, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[24h]))) + p99: histogram_quantile(0.99, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[24h]))) + mean: sum(rate(ws_head_lag_milliseconds_sum{provider="publicnode", chain="ethereum"}[24h])) / sum(rate(ws_head_lag_milliseconds_count{provider="publicnode", chain="ethereum"}[24h])) + success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="publicnode", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) + sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="publicnode", chain="ethereum"}[24h])) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="publicnode", chain="ethereum", region="sgp"}[1h]))) + + - slug: drpc + name: dRPC + tag: Keyless WS, wss://eth.drpc.org + formula: "Median ms behind the earliest provider to push each Ethereum newHeads frame, from one persistent WebSocket per provider on the same eu-west host, histogram_quantile over 24h. Lag is relative to the per-block winner." + queries: + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[24h]))) + p90: histogram_quantile(0.90, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[24h]))) + p99: histogram_quantile(0.99, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[24h]))) + mean: sum(rate(ws_head_lag_milliseconds_sum{provider="drpc", chain="ethereum"}[24h])) / sum(rate(ws_head_lag_milliseconds_count{provider="drpc", chain="ethereum"}[24h])) + success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="drpc", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) + sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="drpc", chain="ethereum"}[24h])) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="drpc", chain="ethereum", region="sgp"}[1h]))) + + - slug: tenderly + name: Tenderly + tag: Keyless WS, wss://mainnet.gateway.tenderly.co + formula: "Median ms behind the earliest provider to push each Ethereum newHeads frame, from one persistent WebSocket per provider on the same eu-west host, histogram_quantile over 24h. Lag is relative to the per-block winner." + queries: + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[24h]))) + p90: histogram_quantile(0.90, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[24h]))) + p99: histogram_quantile(0.99, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[24h]))) + mean: sum(rate(ws_head_lag_milliseconds_sum{provider="tenderly", chain="ethereum"}[24h])) / sum(rate(ws_head_lag_milliseconds_count{provider="tenderly", chain="ethereum"}[24h])) + success: clamp_max(sum(increase(ws_head_blocks_seen_total{provider="tenderly", chain="ethereum"}[24h])) / max(sum by (provider) (increase(ws_head_blocks_seen_total{chain="ethereum"}[24h]))), 1) + sample_size: sum(increase(ws_head_lag_milliseconds_count{provider="tenderly", chain="ethereum"}[24h])) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum"}[1h]))) + regions: + - region: us-east + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="us-east"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="us-east"}[1h]))) + - region: eu-west + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="eu-west"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="eu-west"}[1h]))) + - region: ap-southeast + p50: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="sgp"}[24h]))) + series: histogram_quantile(0.50, sum by (le) (rate(ws_head_lag_milliseconds_bucket{provider="tenderly", chain="ethereum", region="sgp"}[1h]))) diff --git a/harnesses/ws-head-latency/Dockerfile b/harnesses/ws-head-latency/Dockerfile new file mode 100644 index 00000000..63108cfc --- /dev/null +++ b/harnesses/ws-head-latency/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.24-alpine AS builder + +WORKDIR /app +RUN apk add --no-cache git + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 GOOS=linux go build -o /app/monitor ./cmd/script + +FROM debian:bookworm-slim + +WORKDIR /app +RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/monitor /app/monitor + +EXPOSE 2112 + +CMD ["/app/monitor"] diff --git a/harnesses/ws-head-latency/cmd/script/aggregator.go b/harnesses/ws-head-latency/cmd/script/aggregator.go new file mode 100644 index 00000000..87e384bc --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/aggregator.go @@ -0,0 +1,190 @@ +package main + +import ( + "fmt" + "sort" + "sync" + "time" +) + +// Per-chain arrival cohort. +// +// Every (provider, chain) connection reports "provider P saw block N at +// time T" into the chain's aggregator. The first report for a block +// opens a collection window (settleWindow); when it expires the cohort +// is scored: +// +// T0 = earliest arrival among providers +// lag(provider) = arrival(provider) - T0 → histogram sample +// winner = provider at T0 → wins counter +// gap = provider healthy on this chain but absent from the +// cohort → gap counter +// +// The measurement is deliberately RELATIVE: one vantage point cannot +// separate its own network path from provider pipeline latency, but the +// per-block delta between providers over the same path is a fair +// ordering. The fastest provider reads ~0 by construction; the spec's +// editorial copy discloses this. + +const ( + // settleWindow is how long a block cohort stays open after the first + // arrival. 5s covers everything we've observed (worst straggler + // ~1.5s on ethereum); providers later than this are scored as a gap, + // which is the honest reading for a push feed. + settleWindow = 5 * time.Second + // staleAfter bounds "healthy": a provider absent from a cohort only + // counts as a gap when it delivered *something* on this chain within + // staleAfter (otherwise it's just disconnected, which ws_health and + // ws_reconnects_total already report). + staleAfter = 120 * time.Second +) + +type cohortEntry struct { + arrivals map[string]time.Time +} + +type aggregator struct { + chain string + + mu sync.Mutex + blocks map[int64]*cohortEntry + maxFinalized int64 + // lastMsg tracks per-provider liveness on this chain (any accepted + // head/slot). Used for gap attribution and the ws_health sweeper. + lastMsg map[string]time.Time +} + +var ( + aggregatorsMu sync.Mutex + aggregators = map[string]*aggregator{} +) + +func aggregatorFor(chain string) *aggregator { + aggregatorsMu.Lock() + defer aggregatorsMu.Unlock() + if a, ok := aggregators[chain]; ok { + return a + } + a := &aggregator{ + chain: chain, + blocks: map[int64]*cohortEntry{}, + lastMsg: map[string]time.Time{}, + } + aggregators[chain] = a + return a +} + +// record ingests one arrival. Returns false when the arrival was +// dropped (duplicate from the same provider, or a straggler for an +// already-scored block). +func (a *aggregator) record(provider string, height int64, now time.Time) bool { + a.mu.Lock() + defer a.mu.Unlock() + + a.lastMsg[provider] = now + + // Straggler for a cohort that already settled. From a push-feed + // consumer's perspective this block was missed (the gap counter was + // bumped at settle time); opening a fresh single-provider cohort + // here would fabricate a lag=0 "win", so drop it. + if height <= a.maxFinalized { + return false + } + + e, ok := a.blocks[height] + if !ok { + e = &cohortEntry{arrivals: map[string]time.Time{}} + a.blocks[height] = e + time.AfterFunc(settleWindow, func() { a.settle(height) }) + } + if _, dup := e.arrivals[provider]; dup { + return false + } + e.arrivals[provider] = now + headBlocksSeen.WithLabelValues(provider, a.chain).Inc() + return true +} + +// settle scores one block cohort and releases it. +func (a *aggregator) settle(height int64) { + a.mu.Lock() + e, ok := a.blocks[height] + if !ok { + a.mu.Unlock() + return + } + delete(a.blocks, height) + if height > a.maxFinalized { + a.maxFinalized = height + } + // Snapshot healthy providers for gap attribution while still locked. + settleTime := time.Now() + healthy := make([]string, 0, len(a.lastMsg)) + for p, t := range a.lastMsg { + if settleTime.Sub(t) <= staleAfter { + healthy = append(healthy, p) + } + } + a.mu.Unlock() + + // A single-provider cohort carries no relative information: lag + // would be 0 by definition and the "win" unearned. Count gaps for + // the healthy absentees (they truly missed a block others pushed) + // but emit no lag samples. + if len(e.arrivals) >= 2 { + providers := make([]string, 0, len(e.arrivals)) + for p := range e.arrivals { + providers = append(providers, p) + } + // Deterministic tie-break: earliest arrival wins; equal + // timestamps resolve alphabetically. + sort.Strings(providers) + winner := providers[0] + t0 := e.arrivals[winner] + for _, p := range providers[1:] { + if e.arrivals[p].Before(t0) { + winner, t0 = p, e.arrivals[p] + } + } + for _, p := range providers { + lagMs := float64(e.arrivals[p].Sub(t0).Milliseconds()) + headLagHist.WithLabelValues(p, a.chain).Observe(lagMs) + } + headWins.WithLabelValues(winner, a.chain).Inc() + fmt.Printf("[%s] block=%d cohort=%d winner=%s\n", a.chain, height, len(e.arrivals), winner) + } + + for _, p := range healthy { + if _, saw := e.arrivals[p]; !saw { + headBlockGap.WithLabelValues(p, a.chain).Inc() + } + } +} + +// startHealthSweeper flips ws_health to 0 for any (provider, chain) +// that stopped delivering messages, per the gauge's contract (1 = +// connected AND received a message in the last 120s). The connection +// goroutines set it back to 1 on every accepted message. +func startHealthSweeper() { + go func() { + t := time.NewTicker(15 * time.Second) + defer t.Stop() + for now := range t.C { + aggregatorsMu.Lock() + aggs := make([]*aggregator, 0, len(aggregators)) + for _, a := range aggregators { + aggs = append(aggs, a) + } + aggregatorsMu.Unlock() + for _, a := range aggs { + a.mu.Lock() + for p, last := range a.lastMsg { + if now.Sub(last) > staleAfter { + wsHealth.WithLabelValues(p, a.chain).Set(0) + } + } + a.mu.Unlock() + } + } + }() +} diff --git a/harnesses/ws-head-latency/cmd/script/config.go b/harnesses/ws-head-latency/cmd/script/config.go new file mode 100644 index 00000000..1c30cfaa --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/config.go @@ -0,0 +1,70 @@ +package main + +import ( + "os" + "strings" +) + +// Endpoint is one (provider, chain) WebSocket subscription. Each entry +// gets its own goroutine holding a persistent connection; the arrival +// timestamps from every endpoint on the same chain are pooled by the +// per-chain cohort (see cohort.go logic in evm_ws.go / solana_ws.go and +// the shared aggregator below). +type Endpoint struct { + Provider string + Chain string // ethereum | base | solana + Kind string // "evm" (eth_subscribe newHeads) | "solana" (slotSubscribe) + URL string +} + +// endpoints is the source of truth for the measured cohort. All default +// URLs are public and keyless, verified live at harness inception: +// +// publicnode: ethereum / base / solana all push events +// drpc: ethereum verified; base + solana declared and allowed to +// fail at runtime (per-connection reconnect loop degrades +// gracefully, never crashes the process) +// tenderly: ethereum only (no keyless base/solana WS) +// +// Optional keyed providers are appended from env vars so contributor +// keys can widen the cohort without a rebuild. Every default URL is +// also env-overridable via WS_URL__. +func endpoints() []Endpoint { + eps := []Endpoint{ + {Provider: "publicnode", Chain: "ethereum", Kind: "evm", URL: envDefault("WS_URL_PUBLICNODE_ETHEREUM", "wss://ethereum-rpc.publicnode.com")}, + {Provider: "publicnode", Chain: "base", Kind: "evm", URL: envDefault("WS_URL_PUBLICNODE_BASE", "wss://base-rpc.publicnode.com")}, + {Provider: "publicnode", Chain: "solana", Kind: "solana", URL: envDefault("WS_URL_PUBLICNODE_SOLANA", "wss://solana-rpc.publicnode.com")}, + {Provider: "drpc", Chain: "ethereum", Kind: "evm", URL: envDefault("WS_URL_DRPC_ETHEREUM", "wss://eth.drpc.org")}, + {Provider: "drpc", Chain: "base", Kind: "evm", URL: envDefault("WS_URL_DRPC_BASE", "wss://base.drpc.org")}, + {Provider: "drpc", Chain: "solana", Kind: "solana", URL: envDefault("WS_URL_DRPC_SOLANA", "wss://solana.drpc.org")}, + {Provider: "tenderly", Chain: "ethereum", Kind: "evm", URL: envDefault("WS_URL_TENDERLY_ETHEREUM", "wss://mainnet.gateway.tenderly.co")}, + {Provider: "solana-labs", Chain: "solana", Kind: "solana", URL: envDefault("WS_URL_SOLANALABS_SOLANA", "wss://api.mainnet-beta.solana.com")}, + } + // Keyed providers, skipped when the env var is unset. URLs carry the + // key inline (e.g. wss://eth-mainnet.g.alchemy.com/v2/) so they + // live in the deployment env, never in the repo. + for _, opt := range []struct{ env, provider, chain string }{ + {"WS_URL_ALCHEMY_ETHEREUM", "alchemy", "ethereum"}, + {"WS_URL_INFURA_ETHEREUM", "infura", "ethereum"}, + {"WS_URL_CHAINSTACK_ETHEREUM", "chainstack", "ethereum"}, + } { + if url := strings.TrimSpace(os.Getenv(opt.env)); url != "" { + eps = append(eps, Endpoint{Provider: opt.provider, Chain: opt.chain, Kind: "evm", URL: url}) + } + } + return eps +} + +func envDefault(key, def string) string { + if v := strings.TrimSpace(os.Getenv(key)); v != "" { + return v + } + return def +} + +// listenAddr defaults to :2112, the OCB harness convention (scraped by +// the shared Prometheus). LISTEN_ADDR overrides for local runs so a dev +// copy doesn't collide with a deployed harness. +func listenAddr() string { + return envDefault("LISTEN_ADDR", ":2112") +} diff --git a/harnesses/ws-head-latency/cmd/script/evm_ws.go b/harnesses/ws-head-latency/cmd/script/evm_ws.go new file mode 100644 index 00000000..b69fa750 --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/evm_ws.go @@ -0,0 +1,212 @@ +package main + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" +) + +// Persistent WebSocket machinery, ported from the l2-block-time +// harness. One goroutine per (provider, chain) endpoint: +// +// - dial with timeout, subscribe, read loop with rolling deadline +// - client ping every 30s; pong handler extends the read deadline +// - head watchdog: the pong handler keeps the read deadline alive, so +// a provider that heartbeats while silently dropping the +// subscription (observed on publicnode) would wedge the goroutine +// forever. If no head/slot arrives for headWatchdog, slam the +// connection closed and let the outer loop redial. +// - reconnect with exponential backoff 2s → 60s; per-connection, so a +// dead endpoint (e.g. drpc/base if delisted) degrades to health=0 + +// periodic redials without touching the rest of the cohort. + +const ( + dialTimeout = 15 * time.Second + readDeadline = 60 * time.Second + pingEvery = 30 * time.Second + minBackoff = 2 * time.Second + maxBackoff = 60 * time.Second + // headWatchdog must sit well above the slowest cadence in the panel + // (ethereum, 12s slots; solana pushes multiple times per second). + // 90s of silence is unambiguously a broken subscription. + headWatchdog = 90 * time.Second +) + +type rpcReq struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Method string `json:"method"` + Params []any `json:"params"` +} + +// evmEnvelope matches both the eth_subscription push frame and the +// subscribe ack. +type evmEnvelope struct { + Method string `json:"method"` + Params *struct { + Result json.RawMessage `json:"result"` + } `json:"params,omitempty"` +} + +type evmHead struct { + Number string `json:"number"` + Hash string `json:"hash"` +} + +// StartEndpoints spawns the reconnect loop for every configured +// (provider, chain) connection. +func StartEndpoints() { + for _, ep := range endpoints() { + ep := ep + go func() { + backoff := minBackoff + for { + start := time.Now() + var err error + switch ep.Kind { + case "solana": + err = runSolanaConn(ep) + default: + err = runEVMConn(ep) + } + wsHealth.WithLabelValues(ep.Provider, ep.Chain).Set(0) + wsReconnects.WithLabelValues(ep.Provider, ep.Chain).Inc() + if err != nil { + fmt.Printf("[%s/%s] WS error: %v (reconnecting in %v)\n", ep.Provider, ep.Chain, err, backoff) + } + // A connection that held for a while earns a fresh + // backoff; only consecutive fast failures escalate. + if time.Since(start) > 5*time.Minute { + backoff = minBackoff + } + time.Sleep(backoff) + if backoff < maxBackoff { + backoff *= 2 + if backoff > maxBackoff { + backoff = maxBackoff + } + } + } + }() + } +} + +// dialAndGuard opens the connection and installs ping + watchdog +// goroutines shared by the EVM and Solana read loops. The returned +// cleanup must be deferred; bumpHead must be called on every accepted +// head/slot. +func dialAndGuard(ep Endpoint) (conn *websocket.Conn, bumpHead func(), cleanup func(), err error) { + dialer := *websocket.DefaultDialer + dialer.HandshakeTimeout = dialTimeout + conn, _, err = dialer.Dial(ep.URL, nil) + if err != nil { + return nil, nil, nil, fmt.Errorf("dial: %w", err) + } + + done := make(chan struct{}) + var lastHeadUnixNano atomic.Int64 + lastHeadUnixNano.Store(time.Now().UnixNano()) + + go func() { + t := time.NewTicker(pingEvery) + defer t.Stop() + for { + select { + case <-done: + return + case <-t.C: + _ = conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(5*time.Second)) + } + } + }() + go func() { + t := time.NewTicker(15 * time.Second) + defer t.Stop() + for { + select { + case <-done: + return + case now := <-t.C: + silent := now.Sub(time.Unix(0, lastHeadUnixNano.Load())) + if silent > headWatchdog { + fmt.Printf("[%s/%s] watchdog: no event for %s, forcing reconnect\n", ep.Provider, ep.Chain, silent.Round(time.Second)) + _ = conn.Close() + return + } + } + } + }() + + conn.SetPongHandler(func(string) error { + _ = conn.SetReadDeadline(time.Now().Add(readDeadline)) + return nil + }) + + bumpHead = func() { lastHeadUnixNano.Store(time.Now().UnixNano()) } + cleanup = func() { + close(done) + _ = conn.Close() + } + return conn, bumpHead, cleanup, nil +} + +func runEVMConn(ep Endpoint) error { + conn, bumpHead, cleanup, err := dialAndGuard(ep) + if err != nil { + return err + } + defer cleanup() + + if err := conn.WriteJSON(rpcReq{ + JSONRPC: "2.0", ID: 1, Method: "eth_subscribe", + Params: []any{"newHeads"}, + }); err != nil { + return fmt.Errorf("subscribe: %w", err) + } + + wsHealth.WithLabelValues(ep.Provider, ep.Chain).Set(1) + fmt.Printf("[%s/%s] connected, subscribed to newHeads\n", ep.Provider, ep.Chain) + + agg := aggregatorFor(ep.Chain) + for { + _ = conn.SetReadDeadline(time.Now().Add(readDeadline)) + _, msg, err := conn.ReadMessage() + if err != nil { + return fmt.Errorf("read: %w", err) + } + + var env evmEnvelope + if err := json.Unmarshal(msg, &env); err != nil { + continue + } + if env.Method != "eth_subscription" || env.Params == nil { + continue // subscribe ack or unrelated frame + } + var head evmHead + if err := json.Unmarshal(env.Params.Result, &head); err != nil { + continue + } + blockNum := parseHexInt64(head.Number) + if blockNum == 0 { + continue + } + now := time.Now() + bumpHead() + wsHealth.WithLabelValues(ep.Provider, ep.Chain).Set(1) + agg.record(ep.Provider, blockNum, now) + } +} + +func parseHexInt64(s string) int64 { + s = strings.TrimPrefix(s, "0x") + if s == "" { + return 0 + } + n, _ := strconv.ParseInt(s, 16, 64) + return n +} diff --git a/harnesses/ws-head-latency/cmd/script/loghub.go b/harnesses/ws-head-latency/cmd/script/loghub.go new file mode 100644 index 00000000..9dd74488 --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/loghub.go @@ -0,0 +1,114 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "net/http" + "os" + "strconv" + "sync" + "time" +) + +// Auto-generated by the loghub inline pattern. Captures stdout/stderr into a +// bounded ring buffer and exposes GET /logs?tail=N protected by X-Logs-Token +// matching the LOGS_TOKEN env var. +// +// Keep in sync across miniapps (was previously the shared/loghub package; we +// inline because Railway's per-harness Docker build context can't reach a +// sibling shared module via go.mod replace). + +const logRingMax = 5000 + +type logRing struct { + mu sync.Mutex + lines []string + max int +} + +var globalLogRing = &logRing{max: logRingMax} + +func (b *logRing) push(line string) { + entry := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + " " + line + b.mu.Lock() + if len(b.lines) >= b.max { + b.lines = append(b.lines[1:], entry) + } else { + b.lines = append(b.lines, entry) + } + b.mu.Unlock() +} + +func (b *logRing) snapshot(tail int) []string { + b.mu.Lock() + defer b.mu.Unlock() + if tail <= 0 || tail >= len(b.lines) { + out := make([]string, len(b.lines)) + copy(out, b.lines) + return out + } + start := len(b.lines) - tail + out := make([]string, tail) + copy(out, b.lines[start:]) + return out +} + +var logSetupOnce sync.Once + +// installLogCapture replaces os.Stdout (and os.Stderr) with the write-end of a +// pipe, then spawns a goroutine that fan-outs every line to the original +// stdout AND the in-memory ring buffer. Call exactly once, very early in +// main(). +func installLogCapture() { logSetupOnce.Do(doInstallLogCapture) } + +func doInstallLogCapture() { + originalStdout := os.Stdout + originalStderr := os.Stderr + r, w, err := os.Pipe() + if err != nil { + fmt.Fprintf(originalStdout, "[loghub] pipe failed: %v (/logs will be empty)\n", err) + return + } + os.Stdout = w + os.Stderr = w + + go func() { + scanner := bufio.NewScanner(r) + buf := make([]byte, 0, 1024*1024) + scanner.Buffer(buf, 1024*1024) + for scanner.Scan() { + line := scanner.Text() + fmt.Fprintln(originalStdout, line) + globalLogRing.push(line) + } + _, _ = io.Copy(originalStdout, r) + _ = originalStderr + }() +} + +// logsHandler returns an http.Handler for GET /logs?tail=N. Requires header +// X-Logs-Token to match the LOGS_TOKEN env var. Returns 403 if env unset. +func logsHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expected := os.Getenv("LOGS_TOKEN") + if expected == "" { + http.Error(w, "logs disabled: LOGS_TOKEN unset", http.StatusForbidden) + return + } + if r.Header.Get("X-Logs-Token") != expected { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + tail := 500 + if t := r.URL.Query().Get("tail"); t != "" { + if n, err := strconv.Atoi(t); err == nil && n > 0 { + tail = n + } + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + for _, l := range globalLogRing.snapshot(tail) { + fmt.Fprintln(w, l) + } + }) +} diff --git a/harnesses/ws-head-latency/cmd/script/main.go b/harnesses/ws-head-latency/cmd/script/main.go new file mode 100644 index 00000000..2079aeaa --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" +) + +func main() { + installLogCapture() // capture stdout into /logs ring buffer + fmt.Println("=== WS Head Latency Monitor ===") + fmt.Println("Bench № 081 — per-block WebSocket push lag across RPC providers, relative to the earliest arrival in the cohort.") + fmt.Println() + + eps := endpoints() + addr := listenAddr() + fmt.Printf("Connections tracked: %d\n", len(eps)) + for _, ep := range eps { + fmt.Printf(" %-12s %-9s %s\n", ep.Provider, ep.Chain, ep.URL) + } + fmt.Printf("Metrics server: %s/metrics\n\n", addr) + + go func() { + if err := StartMetricsServer(addr); err != nil { + fmt.Printf("[fatal] metrics server: %v\n", err) + os.Exit(1) + } + }() + + StartEndpoints() + startHealthSweeper() + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + s := <-sig + fmt.Printf("\n[shutdown] received %v\n", s) +} diff --git a/harnesses/ws-head-latency/cmd/script/metrics.go b/harnesses/ws-head-latency/cmd/script/metrics.go new file mode 100644 index 00000000..8df51f2f --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/metrics.go @@ -0,0 +1,110 @@ +package main + +import ( + "net/http" + "os" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// Prom metrics for the ws-head-latency harness. Names follow the +// `__` OCB convention. Series carry +// {provider, chain} plus a static {region} ConstLabel bound at +// process startup, so a multi-region deploy (Railway replicas or +// VPS + Railway hybrid) never collides series across vantage points +// and no observation site needs to know about the region. +// +// Region resolution order at startup: OCB_REGION, then +// RAILWAY_REPLICA_REGION (Railway sets it automatically), then the +// eu-west default that matches the historical single-vantage Paris +// VPS deployment. Set OCB_REGION explicitly on each replica to +// guarantee a stable value across redeploys. +var region = resolveRegion() + +func resolveRegion() string { + if r := os.Getenv("OCB_REGION"); r != "" { + return r + } + if r := os.Getenv("RAILWAY_REPLICA_REGION"); r != "" { + return r + } + return "eu-west" +} + +var regionLabels = prometheus.Labels{"region": region} + +var ( + headLagHist = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ws_head_lag_milliseconds", + Help: "Per-block push lag vs the earliest provider in the cohort, in milliseconds. Relative measurement: the fastest provider on a block reads 0 by construction.", + // 5ms → 10.24s exponential; covers same-frame ties through + // multi-second stragglers. + Buckets: prometheus.ExponentialBuckets(5, 2, 12), + ConstLabels: regionLabels, + }, + []string{"provider", "chain"}, + ) + + headWins = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ws_head_wins_total", + Help: "Blocks where this provider was the first in the cohort to push the head/slot. Only counted when at least 2 providers saw the block.", + ConstLabels: regionLabels, + }, + []string{"provider", "chain"}, + ) + + headBlocksSeen = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ws_head_blocks_seen_total", + Help: "Distinct blocks/slots this provider pushed within the cohort settle window. Ratio against the cohort max is the coverage score.", + ConstLabels: regionLabels, + }, + []string{"provider", "chain"}, + ) + + wsReconnects = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ws_reconnects_total", + Help: "WebSocket reconnections per (provider, chain) since process start. Sustained non-zero rate flags an unstable endpoint.", + ConstLabels: regionLabels, + }, + []string{"provider", "chain"}, + ) + + wsHealth = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "ws_health", + Help: "1 when the subscription is connected and delivered a message in the last 120s, 0 otherwise.", + ConstLabels: regionLabels, + }, + []string{"provider", "chain"}, + ) + + headBlockGap = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "ws_block_gap_total", + Help: "Blocks the cohort saw but this provider missed (absent from the 5s settle window while its connection was otherwise live).", + ConstLabels: regionLabels, + }, + []string{"provider", "chain"}, + ) +) + +// StartMetricsServer binds /metrics + /health + /logs on addr. Blocking +// call, run in its own goroutine. +func StartMetricsServer(addr string) error { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/logs", logsHandler()) + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("ok")) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("ws-head-latency harness · OpenChainBench № 081")) + }) + return http.ListenAndServe(addr, mux) +} diff --git a/harnesses/ws-head-latency/cmd/script/solana_ws.go b/harnesses/ws-head-latency/cmd/script/solana_ws.go new file mode 100644 index 00000000..6ebf8cc3 --- /dev/null +++ b/harnesses/ws-head-latency/cmd/script/solana_ws.go @@ -0,0 +1,80 @@ +package main + +import ( + "encoding/json" + "fmt" + "time" +) + +// Solana slot push via `slotSubscribe`. The notification frame looks +// like: +// +// {"jsonrpc":"2.0","method":"slotNotification", +// "params":{"result":{"parent":364,"root":362,"slot":365},"subscription":0}} +// +// The slot number is the cohort key, playing the role the block number +// plays on EVM chains. Solana slots tick every ~400ms, so the cohort +// churn is higher but the settle/score path is identical. + +type solanaEnvelope struct { + Method string `json:"method"` + Params *struct { + Result struct { + Slot int64 `json:"slot"` + } `json:"result"` + } `json:"params,omitempty"` + // Subscribe ack: {"jsonrpc":"2.0","result":23784,"id":1} + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +func runSolanaConn(ep Endpoint) error { + conn, bumpHead, cleanup, err := dialAndGuard(ep) + if err != nil { + return err + } + defer cleanup() + + if err := conn.WriteJSON(rpcReq{ + JSONRPC: "2.0", ID: 1, Method: "slotSubscribe", + Params: []any{}, + }); err != nil { + return fmt.Errorf("subscribe: %w", err) + } + + wsHealth.WithLabelValues(ep.Provider, ep.Chain).Set(1) + fmt.Printf("[%s/%s] connected, subscribed to slotSubscribe\n", ep.Provider, ep.Chain) + + agg := aggregatorFor(ep.Chain) + for { + _ = conn.SetReadDeadline(time.Now().Add(readDeadline)) + _, msg, err := conn.ReadMessage() + if err != nil { + return fmt.Errorf("read: %w", err) + } + + var env solanaEnvelope + if err := json.Unmarshal(msg, &env); err != nil { + continue + } + if env.Error != nil { + // Endpoint answered the subscribe with a JSON-RPC error + // (method disabled, quota, ...). Surface it and let the + // outer loop back off instead of holding a dead socket. + return fmt.Errorf("subscribe error %d: %s", env.Error.Code, env.Error.Message) + } + if env.Method != "slotNotification" || env.Params == nil { + continue // subscribe ack or unrelated frame + } + slot := env.Params.Result.Slot + if slot == 0 { + continue + } + now := time.Now() + bumpHead() + wsHealth.WithLabelValues(ep.Provider, ep.Chain).Set(1) + agg.record(ep.Provider, slot, now) + } +} diff --git a/harnesses/ws-head-latency/go.mod b/harnesses/ws-head-latency/go.mod new file mode 100644 index 00000000..3bca706e --- /dev/null +++ b/harnesses/ws-head-latency/go.mod @@ -0,0 +1,21 @@ +module ws-head-latency + +go 1.24.0 + +require ( + github.com/gorilla/websocket v1.5.3 + github.com/prometheus/client_golang v1.23.2 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/sys v0.35.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect +) diff --git a/harnesses/ws-head-latency/go.sum b/harnesses/ws-head-latency/go.sum new file mode 100644 index 00000000..02807ba4 --- /dev/null +++ b/harnesses/ws-head-latency/go.sum @@ -0,0 +1,48 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/harnesses/ws-head-latency/railway.toml b/harnesses/ws-head-latency/railway.toml new file mode 100644 index 00000000..2abbb1f0 --- /dev/null +++ b/harnesses/ws-head-latency/railway.toml @@ -0,0 +1,7 @@ +[build] +builder = "DOCKERFILE" +dockerfilePath = "Dockerfile" + +[deploy] +healthcheckPath = "/health" +restartPolicyType = "ON_FAILURE"