From 5bef5da9223dc15c5227859215e17130702afaf8 Mon Sep 17 00:00:00 2001 From: proerror Date: Wed, 15 Jul 2026 11:18:46 +0800 Subject: [PATCH 1/6] feat(collector): make Rust cutover fail closed --- .github/workflows/ci.yml | 8 +- deployment/aliyun/README.md | 192 ++- ... binance-lob-archiver-production-spot.env} | 4 +- ... binance-lob-archiver-production-usdm.env} | 3 +- .../binance-lob-archiver-production@.service | 37 + .../aliyun/binance-lob-archiver-rust-spot.env | 4 +- .../binance-lob-archiver-rust-upload@.service | 30 + .../aliyun/binance-lob-archiver-rust-usdm.env | 4 +- .../aliyun/binance-lob-archiver-rust@.service | 8 +- .../binance-lob-archiver-upload@.service | 30 + .../aliyun/binance-lob-archiver@.service | 28 - deployment/aliyun/binance_lob_archiver.py | 1245 ----------------- deployment/aliyun/deploy-rust-lob-release.sh | 304 ++++ deployment/aliyun/host-rust-lob-cutover.sh | 720 ++++++++++ .../aliyun/host-rust-lob-shadow-gate.sh | 723 ++++++++++ .../aliyun/invoke-rust-lob-operation.sh | 153 ++ .../aliyun/rust-lob-control-plane-lib.sh | 35 + .../aliyun/rust-lob-runtime-health-policy.jq | 13 + .../aliyun/rust-lob-shadow-gate-policy.jq | 52 + .../aliyun/test-rust-lob-control-plane.sh | 270 ++++ .../aliyun/test_binance_lob_archiver.py | 518 ------- rust_hft/Cargo.lock | 1 + rust_hft/tools/collector/Cargo.toml | 1 + .../collector/src/bin/binance-lob-archiver.rs | 521 ++++++- rust_hft/tools/collector/src/lob_archiver.rs | 62 +- 25 files changed, 3090 insertions(+), 1876 deletions(-) rename deployment/aliyun/{binance-lob-archiver-spot.env => binance-lob-archiver-production-spot.env} (85%) rename deployment/aliyun/{binance-lob-archiver-usdm.env => binance-lob-archiver-production-usdm.env} (91%) create mode 100644 deployment/aliyun/binance-lob-archiver-production@.service create mode 100644 deployment/aliyun/binance-lob-archiver-rust-upload@.service create mode 100644 deployment/aliyun/binance-lob-archiver-upload@.service delete mode 100644 deployment/aliyun/binance-lob-archiver@.service delete mode 100644 deployment/aliyun/binance_lob_archiver.py create mode 100755 deployment/aliyun/deploy-rust-lob-release.sh create mode 100755 deployment/aliyun/host-rust-lob-cutover.sh create mode 100755 deployment/aliyun/host-rust-lob-shadow-gate.sh create mode 100755 deployment/aliyun/invoke-rust-lob-operation.sh create mode 100644 deployment/aliyun/rust-lob-control-plane-lib.sh create mode 100644 deployment/aliyun/rust-lob-runtime-health-policy.jq create mode 100644 deployment/aliyun/rust-lob-shadow-gate-policy.jq create mode 100755 deployment/aliyun/test-rust-lob-control-plane.sh delete mode 100644 deployment/aliyun/test_binance_lob_archiver.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb3667008..5daeaf68d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: components: rustfmt, clippy - name: Install Linux build tools - run: sudo apt-get update && sudo apt-get install -y clang mold protobuf-compiler + run: sudo apt-get update && sudo apt-get install -y clang mold protobuf-compiler shellcheck - name: Setup sccache uses: mozilla-actions/sccache-action@v0.0.10 @@ -114,6 +114,12 @@ jobs: cargo clippy -p hft-collector --all-targets --features collector-binance --no-deps --locked -- -D warnings cargo test -p hft-collector --features collector-binance --locked + - name: Verify collector release control plane + run: | + bash -n ../deployment/aliyun/*.sh + shellcheck ../deployment/aliyun/*.sh + ../deployment/aliyun/test-rust-lob-control-plane.sh + - name: Build focused packages run: cargo build -p hft-live -p hft-paper -p hft-all-in-one -p alpha-harness -p hft-harnessctl --locked diff --git a/deployment/aliyun/README.md b/deployment/aliyun/README.md index 3a01ceea3..fee7f9224 100644 --- a/deployment/aliyun/README.md +++ b/deployment/aliyun/README.md @@ -1,11 +1,20 @@ # Aliyun Binance data host -The Tokyo ECS runs two public-market-data services. Neither service submits orders. +The Tokyo ECS runs the Rust Binance LOB archiver as two public-market-data +services. Neither service has trading credentials or submits orders. +The legacy Python collector, its systemd template, and its deployment tests have +been removed; the Binance collector deployment lane is Rust-only. + +The current Rust production cutover completed on 2026-07-15 after a 3,672-second +full-catalog shadow with zero restarts and sequence gaps plus Spot and USD-M OSS +round-trip verification. The live evidence is stored under +`/data/monday/evidence/cutovers/`; the rollout below is the required procedure for +future collector releases and host replacements. ```bash -systemctl status binance-lob-archiver@spot.service -systemctl status binance-lob-archiver@usdm.service -journalctl -u 'binance-lob-archiver@*' -f +systemctl status binance-lob-archiver-production@spot.service +systemctl status binance-lob-archiver-production@usdm.service +journalctl -u 'binance-lob-archiver-production@*' -f ``` Polymarket public crypto market updates run separately in dry-run/no-op mode: @@ -167,61 +176,158 @@ ClickHouse is optional for always-on shared analytics, dashboards, and derived realtime features. It is not required for the first backtest pipeline and should not duplicate the complete raw OSS tape. -## Rust collector shadow rollout +## Rust-only collector release workflow + +The Binance collector deployment lane is Rust-only. The legacy Python collector, +its systemd unit, and its deployment tests are removed. A release now has three +separate operations: -The Rust replacement runs beside the Python collector first. Its service name, -spool directory, and OSS dataset are deliberately separate, so installing or -starting it neither stops the Python services nor overwrites their objects: +1. install a digest-pinned candidate without touching production; +2. run a candidate-specific one-hour full-catalog shadow gate; +3. cut over only by consuming that gate's immutable evidence. -| Market | Service | Local spool | OSS dataset | -| --- | --- | --- | --- | -| Spot | `binance-lob-archiver-rust@spot` | `/data/monday/spool/binance-lob-rust-shadow/spot` | `spot_all_rust_shadow` | -| USD-M | `binance-lob-archiver-rust@usdm` | `/data/monday/spool/binance-lob-rust-shadow/usdm` | `usdm_perpetual_all_rust_shadow` | +All host operations go through Alibaba Cloud Assistant from the local Alibaba +Cloud CLI. The scripts reject regions other than Tokyo +(`ap-northeast-1`), use the configured `default` CLI profile, and never put a +credential in command content. The ECS side uses `MondayLobEcsRole`. -Both shadow examples start with `BTCUSDT` only. Keep that bounded scope through -two successful segment rotations and uploads; switch `SYMBOLS=ALL` only after -health, replay continuity, and resource use pass the shadow gate. +### 1. Install a candidate -Build and verify the binary from `rust_hft/`: +The artifact must be a Linux x86-64 Rust binary produced by the approved build, +uploaded to private OSS, and identified by its exact SHA-256. Do not upload a +macOS `target/release` binary. The ACR collector image is a durable container +publication, but the current bare ECS collector consumes the separately pinned +OSS binary. + +Run the committed installer from a clean checkout at `SOURCE_REVISION`: ```bash -cargo build --release --locked --no-default-features \ - -p hft-collector --bin binance-lob-archiver -target/release/binance-lob-archiver --self-test +set -euo pipefail +INSTANCE_ID=i-REPLACE \ +ARTIFACT_OSS_URI=oss://monday-lob-apne1-1045353359/releases/binance-lob-archiver/REPLACE/binance-lob-archiver \ +ARTIFACT_SHA256=REPLACE_WITH_64_HEX_DIGEST \ +SOURCE_REVISION=REPLACE_WITH_GIT_SHA \ +./deployment/aliyun/deploy-rust-lob-release.sh ``` -Install the binary and the Rust-only service templates without changing the -running Python units: +The installer verifies that `SOURCE_REVISION` is the clean current `HEAD`, +uploads a digest-addressed deployment bundle, waits for Cloud Assistant, verifies +both OSS objects on the host, runs the binary self-test, and requires the +`--upload-only` capability. It installs only the isolated shadow unit/env files +and the shadow symlink. Production unit/env files remain staged under: -```bash -sudo install -D -m 0755 target/release/binance-lob-archiver \ - /opt/monday/bin/binance-lob-archiver -sudo install -d -m 0750 -o hftcollector -g hftcollector \ - /data/monday/spool/binance-lob-rust-shadow/{spot,usdm} -sudo install -m 0644 ../deployment/aliyun/binance-lob-archiver-rust@.service \ - /etc/systemd/system/binance-lob-archiver-rust@.service -sudo install -m 0640 ../deployment/aliyun/binance-lob-archiver-rust-spot.env \ - /etc/monday/binance-lob-archiver-rust-spot.env -sudo install -m 0640 ../deployment/aliyun/binance-lob-archiver-rust-usdm.env \ - /etc/monday/binance-lob-archiver-rust-usdm.env -sudo systemctl daemon-reload +```text +/opt/monday/releases/binance-lob-archiver//deployment/ ``` -Starting shadow collection is a separate, explicit operation: +Candidate installation refuses an unmounted `/data`, an active shadow, a digest +mismatch, or a concurrent release operation. It does not start any service and +does not overwrite production configuration or the production symlink. + +The committed shadow environments use `SYMBOLS=ALL`, ten-minute segments, the +isolated spools below, and isolated OSS datasets: + +| Market | Shadow spool | Shadow dataset | +| --- | --- | --- | +| Spot | `/data/monday/spool/binance-lob-rust-shadow/spot` | `spot_all_rust_shadow` | +| USD-M | `/data/monday/spool/binance-lob-rust-shadow/usdm` | `usdm_perpetual_all_rust_shadow` | + +### 2. Run the one-hour full-catalog gate + +Start the gate through the same CLI wrapper: ```bash -sudo systemctl start binance-lob-archiver-rust@spot.service -sudo systemctl start binance-lob-archiver-rust@usdm.service -jq . /data/monday/spool/binance-lob-rust-shadow/{spot,usdm}/health.json +set -euo pipefail +ACTION=gate \ +INSTANCE_ID=i-REPLACE \ +ARTIFACT_SHA256=REPLACE_WITH_64_HEX_DIGEST \ +./deployment/aliyun/invoke-rust-lob-operation.sh +``` + +The host gate owns the complete transition. It verifies the candidate and +`SYMBOLS=ALL`, drains any previous isolated shadow data, restarts both units, +waits for initial full-catalog health, freezes both session IDs and catalog +digests, and then uses monotonic time to observe at least 3,600 seconds. It fails unless all of +these are true for the entire candidate run: + +- both units stay active with `NRestarts=0`; +- Spot has at least 1,000 symbols and USD-M at least 400; +- every discovered symbol has a ready snapshot and sequence gaps remain zero; +- neither session nor catalog membership changes, health never stops advancing + for more than 90 seconds, and the persistent upload-failure count is unchanged; +- pending uploads are zero and queue, disk, and upload warnings are false; +- CPU accounting and peak memory stay inside the systemd limits; +- after stop, the candidate's `--upload-only` drain leaves no partial, + temporary, corrupt, compressed, success-marker, or cleanup-marker artifact; +- for each market, at least two manifests created after gate start are downloaded + from OSS with their data object and reproduce the manifest SHA-256. + +A successful production gate writes: + +```text +/data/monday/evidence/shadow-gates///gate.json +/data/monday/evidence/shadow-gates///PASSED.sha256 ``` -The container image is built from the Rust workspace root and includes pinned -Alibaba Cloud CLI checksums plus `zstd`: +The marker hashes exactly that `gate.json`. Evidence also binds the clean source +revision and deployment-bundle SHA-256, so unit or env changes cannot consume an +older gate for the same binary. A short test override is available +only for script testing; it writes `passed=false` and never creates +`PASSED.sha256`, so it cannot authorize cutover. + +### 3. Cut over or roll back + +After the production gate succeeds, invoke the cutover with the same immutable +artifact digest: ```bash -docker build -f deployment/docker/Dockerfile.binance-lob-archiver \ - -t monday/binance-lob-archiver:shadow . +set -euo pipefail +ACTION=cutover \ +INSTANCE_ID=i-REPLACE \ +ARTIFACT_SHA256=REPLACE_WITH_64_HEX_DIGEST \ +./deployment/aliyun/invoke-rust-lob-operation.sh ``` -For container use, mount the shadow spool read-write and provide an ECS RAM-role -Alibaba Cloud CLI profile. Do not inject long-lived access keys into the image. +The host cutover revalidates the binary, release metadata, staged deployment +files, gate JSON, marker hash, duration, full-catalog counts, and OSS round trips. +Only then does it disable and stop the current production units. After production +is stopped, it installs the target production unit/env files. + +The drain is bootstrap-safe: it runs the digest-pinned target binary directly +against the canonical production env, so the first upgrade does not depend on +the old production binary supporting `--upload-only`. A new host is accepted +only when the canonical spool contains no segment artifact. The script then +atomically changes the production symlink and starts both services without +enabling them. It verifies fresh full-catalog health, no warnings, zero restarts, +and each process's `/proc//exe` resolving to the requested release; only a +verified candidate is enabled for reboot. + +Any failure after production stops triggers a fail-closed Rust-to-Rust restore of +the previous digest-addressed binary and its staged production assets. Rollback +removes candidate health, starts the old units while disabled, requires health +written after that restart, and verifies full catalog, zero restarts, and the old +`/proc//exe` targets before enabling. If a safe restore cannot be proved, +both production units remain disabled and masked. Cutover evidence +is written under `/data/monday/evidence/cutovers/`. + +Rollback uses the same `ACTION=cutover` operation with a previously installed, +previously gated artifact digest. There is no Python fallback and no manual +symlink shortcut. + +### Upload cleanup and failure rules + +After all three OSS objects upload successfully, the Rust collector atomically +writes an uploaded-cleanup marker. Restart recovery consumes that marker first, +removes local data/manifest/success artifacts idempotently, fsyncs the directory, +and removes the marker last. An interrupted or invalid cleanup marker makes +`--upload-only` fail closed. Normal collection and upload-only drain also share +an exclusive per-spool process lock, so they cannot mutate one market spool +concurrently even if an operator bypasses the systemd transition mask. Recursive +spool scans reject root, directory, and file symlinks rather than crossing into +another market or filesystem. + +Do not manually delete a spool, repoint a release symlink, start a second +canonical writer, or bypass `PASSED.sha256`. Do not open general SSH for a +release. When a Cloud Assistant deadline expires, the local wrapper requests +cancellation and waits for a terminal invocation state; host-side `flock` +prevents a retry from racing an earlier operation. diff --git a/deployment/aliyun/binance-lob-archiver-spot.env b/deployment/aliyun/binance-lob-archiver-production-spot.env similarity index 85% rename from deployment/aliyun/binance-lob-archiver-spot.env rename to deployment/aliyun/binance-lob-archiver-production-spot.env index e27296307..0a510b34b 100644 --- a/deployment/aliyun/binance-lob-archiver-spot.env +++ b/deployment/aliyun/binance-lob-archiver-production-spot.env @@ -6,10 +6,10 @@ DEPTH_MODE=diff WS_SHARD_SIZE=100 SNAPSHOT_LIMIT=100 SNAPSHOT_REQUESTS_PER_SECOND=15 +SNAPSHOT_RETRY_ATTEMPTS=6 SYNC_TIMEOUT_SECONDS=120 STALL_TIMEOUT_SECONDS=60 PROCESS_WATCHDOG_SECONDS=180 -TASK_CANCEL_TIMEOUT_SECONDS=5 MAX_BUFFERED_DIFFS=250000 MAX_PENDING_DIFFS_TOTAL=250000 MIN_FREE_GB=20 @@ -21,3 +21,5 @@ OSS_BUCKET=monday-lob-apne1-1045353359 OSS_ENDPOINT=oss-ap-northeast-1-internal.aliyuncs.com OSS_REGION=ap-northeast-1 ALIYUN_PROFILE=ecs-role +BINANCE_REST_BASE=https://data-api.binance.vision +LOG_LEVEL=INFO diff --git a/deployment/aliyun/binance-lob-archiver-usdm.env b/deployment/aliyun/binance-lob-archiver-production-usdm.env similarity index 91% rename from deployment/aliyun/binance-lob-archiver-usdm.env rename to deployment/aliyun/binance-lob-archiver-production-usdm.env index b50a843f6..fb8c6e2f4 100644 --- a/deployment/aliyun/binance-lob-archiver-usdm.env +++ b/deployment/aliyun/binance-lob-archiver-production-usdm.env @@ -10,7 +10,6 @@ SNAPSHOT_RETRY_ATTEMPTS=6 SYNC_TIMEOUT_SECONDS=120 STALL_TIMEOUT_SECONDS=60 PROCESS_WATCHDOG_SECONDS=180 -TASK_CANCEL_TIMEOUT_SECONDS=5 MAX_BUFFERED_DIFFS=250000 MAX_PENDING_DIFFS_TOTAL=250000 MIN_FREE_GB=20 @@ -22,3 +21,5 @@ OSS_BUCKET=monday-lob-apne1-1045353359 OSS_ENDPOINT=oss-ap-northeast-1-internal.aliyuncs.com OSS_REGION=ap-northeast-1 ALIYUN_PROFILE=ecs-role +BINANCE_REST_BASE=https://fapi.binance.com +LOG_LEVEL=INFO diff --git a/deployment/aliyun/binance-lob-archiver-production@.service b/deployment/aliyun/binance-lob-archiver-production@.service new file mode 100644 index 000000000..e9b15d7c4 --- /dev/null +++ b/deployment/aliyun/binance-lob-archiver-production@.service @@ -0,0 +1,37 @@ +[Unit] +Description=Rust Binance LOB archiver production (%i) +After=network-online.target +Wants=network-online.target +RequiresMountsFor=/data +AssertPathIsMountPoint=/data + +[Service] +Type=simple +User=hftcollector +Group=hftcollector +Environment=RUST_LOG=info +Environment=HOME=/var/lib/hft-collector +EnvironmentFile=/etc/monday/binance-lob-archiver-production-%i.env +ExecStartPre=/opt/monday/bin/binance-lob-archiver --self-test +ExecStart=/opt/monday/bin/binance-lob-archiver +Restart=always +RestartSec=5 +RuntimeMaxSec=21600 +TimeoutStopSec=600 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +LockPersonality=true +RestrictSUIDSGID=true +StateDirectory=hft-collector +ReadWritePaths=/data/monday/spool/binance-lob +CPUQuota=80% +MemoryHigh=2500M +MemoryMax=3200M + +[Install] +WantedBy=multi-user.target diff --git a/deployment/aliyun/binance-lob-archiver-rust-spot.env b/deployment/aliyun/binance-lob-archiver-rust-spot.env index 4e05bcf3f..070c1bab5 100644 --- a/deployment/aliyun/binance-lob-archiver-rust-spot.env +++ b/deployment/aliyun/binance-lob-archiver-rust-spot.env @@ -1,7 +1,7 @@ MARKET=spot DATASET=spot_all_rust_shadow SHARD_ID=all -SYMBOLS=BTCUSDT +SYMBOLS=ALL DEPTH_MODE=diff WS_SHARD_SIZE=100 SNAPSHOT_LIMIT=100 @@ -15,7 +15,7 @@ MAX_PENDING_DIFFS_TOTAL=250000 MIN_FREE_GB=20 ZSTD_TIMEOUT_SECONDS=300 OSS_COPY_TIMEOUT_SECONDS=300 -SEGMENT_SECONDS=3600 +SEGMENT_SECONDS=600 SPOOL_DIR=/data/monday/spool/binance-lob-rust-shadow/spot OSS_BUCKET=monday-lob-apne1-1045353359 OSS_ENDPOINT=oss-ap-northeast-1-internal.aliyuncs.com diff --git a/deployment/aliyun/binance-lob-archiver-rust-upload@.service b/deployment/aliyun/binance-lob-archiver-rust-upload@.service new file mode 100644 index 000000000..1de3121c0 --- /dev/null +++ b/deployment/aliyun/binance-lob-archiver-rust-upload@.service @@ -0,0 +1,30 @@ +[Unit] +Description=Rust Binance LOB archiver shadow pending-upload drain (%i) +After=network-online.target +Wants=network-online.target +RequiresMountsFor=/data +AssertPathIsMountPoint=/data + +[Service] +Type=oneshot +User=hftcollector +Group=hftcollector +Environment=RUST_LOG=info +Environment=HOME=/var/lib/hft-collector +EnvironmentFile=/etc/monday/binance-lob-archiver-rust-%i.env +ExecStart=/opt/monday/bin/binance-lob-archiver-shadow --upload-only +TimeoutStartSec=600 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +LockPersonality=true +RestrictSUIDSGID=true +StateDirectory=hft-collector +ReadWritePaths=/data/monday/spool/binance-lob-rust-shadow +CPUQuota=80% +MemoryHigh=2500M +MemoryMax=3200M diff --git a/deployment/aliyun/binance-lob-archiver-rust-usdm.env b/deployment/aliyun/binance-lob-archiver-rust-usdm.env index 917f7fdac..5d66c4a40 100644 --- a/deployment/aliyun/binance-lob-archiver-rust-usdm.env +++ b/deployment/aliyun/binance-lob-archiver-rust-usdm.env @@ -1,7 +1,7 @@ MARKET=usdm DATASET=usdm_perpetual_all_rust_shadow SHARD_ID=all -SYMBOLS=BTCUSDT +SYMBOLS=ALL DEPTH_MODE=diff WS_SHARD_SIZE=100 SNAPSHOT_LIMIT=100 @@ -15,7 +15,7 @@ MAX_PENDING_DIFFS_TOTAL=250000 MIN_FREE_GB=20 ZSTD_TIMEOUT_SECONDS=300 OSS_COPY_TIMEOUT_SECONDS=300 -SEGMENT_SECONDS=3600 +SEGMENT_SECONDS=600 SPOOL_DIR=/data/monday/spool/binance-lob-rust-shadow/usdm OSS_BUCKET=monday-lob-apne1-1045353359 OSS_ENDPOINT=oss-ap-northeast-1-internal.aliyuncs.com diff --git a/deployment/aliyun/binance-lob-archiver-rust@.service b/deployment/aliyun/binance-lob-archiver-rust@.service index 8b8c16a60..7a9e8be49 100644 --- a/deployment/aliyun/binance-lob-archiver-rust@.service +++ b/deployment/aliyun/binance-lob-archiver-rust@.service @@ -3,7 +3,7 @@ Description=Rust Binance LOB archiver shadow (%i) After=network-online.target Wants=network-online.target RequiresMountsFor=/data -ConditionPathIsMountPoint=/data +AssertPathIsMountPoint=/data [Service] Type=simple @@ -12,12 +12,12 @@ Group=hftcollector Environment=RUST_LOG=info Environment=HOME=/var/lib/hft-collector EnvironmentFile=/etc/monday/binance-lob-archiver-rust-%i.env -ExecStartPre=/opt/monday/bin/binance-lob-archiver --self-test -ExecStart=/opt/monday/bin/binance-lob-archiver +ExecStartPre=/opt/monday/bin/binance-lob-archiver-shadow --self-test +ExecStart=/opt/monday/bin/binance-lob-archiver-shadow Restart=always RestartSec=5 RuntimeMaxSec=21600 -TimeoutStopSec=30 +TimeoutStopSec=600 NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict diff --git a/deployment/aliyun/binance-lob-archiver-upload@.service b/deployment/aliyun/binance-lob-archiver-upload@.service new file mode 100644 index 000000000..dbe0df553 --- /dev/null +++ b/deployment/aliyun/binance-lob-archiver-upload@.service @@ -0,0 +1,30 @@ +[Unit] +Description=Rust Binance LOB archiver production pending-upload drain (%i) +After=network-online.target +Wants=network-online.target +RequiresMountsFor=/data +AssertPathIsMountPoint=/data + +[Service] +Type=oneshot +User=hftcollector +Group=hftcollector +Environment=RUST_LOG=info +Environment=HOME=/var/lib/hft-collector +EnvironmentFile=/etc/monday/binance-lob-archiver-production-%i.env +ExecStart=/opt/monday/bin/binance-lob-archiver --upload-only +TimeoutStartSec=600 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +LockPersonality=true +RestrictSUIDSGID=true +StateDirectory=hft-collector +ReadWritePaths=/data/monday/spool/binance-lob +CPUQuota=80% +MemoryHigh=2500M +MemoryMax=3200M diff --git a/deployment/aliyun/binance-lob-archiver@.service b/deployment/aliyun/binance-lob-archiver@.service deleted file mode 100644 index e2ab18185..000000000 --- a/deployment/aliyun/binance-lob-archiver@.service +++ /dev/null @@ -1,28 +0,0 @@ -[Unit] -Description=Binance LOB archiver (%i) -After=network-online.target -Wants=network-online.target -RequiresMountsFor=/data - -[Service] -Type=simple -User=hftcollector -Group=hftcollector -StateDirectory=hft-collector -EnvironmentFile=/etc/monday/binance-lob-archiver-%i.env -ExecStart=/opt/monday/venv/bin/python /opt/monday/binance_lob_archiver.py -Restart=always -RestartSec=5 -RuntimeMaxSec=21600 -TimeoutStopSec=180 -NoNewPrivileges=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=true -ReadWritePaths=/data/monday/spool -CPUQuota=80% -MemoryHigh=2500M -MemoryMax=3200M - -[Install] -WantedBy=multi-user.target diff --git a/deployment/aliyun/binance_lob_archiver.py b/deployment/aliyun/binance_lob_archiver.py deleted file mode 100644 index 79fc98b4a..000000000 --- a/deployment/aliyun/binance_lob_archiver.py +++ /dev/null @@ -1,1245 +0,0 @@ -#!/opt/monday/venv/bin/python -import argparse -import asyncio -import hashlib -import json -import logging -import os -import shutil -import signal -import subprocess -import tempfile -import threading -import time -import urllib.parse -import urllib.request -import uuid -from collections import Counter -from datetime import datetime, timezone -from decimal import Decimal -from pathlib import Path -from urllib.error import HTTPError, URLError - -from websockets.asyncio.client import connect - - -LOG = logging.getLogger("binance-lob-archiver") -MARKET = os.getenv("MARKET", "spot").strip().lower() -DATASET = os.getenv("DATASET", f"{MARKET}_all").strip().lower() -SHARD_ID = os.getenv("SHARD_ID", "all").strip().lower() -SYMBOLS_SETTING = os.getenv( - "SYMBOLS", "btcusdt,ethusdt,bnbusdt,solusdt,xrpusdt,dogeusdt" -).strip() -SYMBOLS = tuple( - item.strip().lower() - for item in SYMBOLS_SETTING.split(",") - if item.strip() -) -SECURITY_TOKEN_SYMBOLS: tuple[str, ...] = () -EXCLUDED_SYMBOLS: tuple[str, ...] = () -MODE = os.getenv("DEPTH_MODE", "diff").strip().lower() -SEGMENT_SECONDS = max(60, int(os.getenv("SEGMENT_SECONDS", "3600"))) -SPOOL_DIR = Path(os.getenv("SPOOL_DIR", "/data/monday/spool/binance-lob")) -BUCKET = os.getenv("OSS_BUCKET", "monday-lob-apne1-1045353359") -ENDPOINT = os.getenv("OSS_ENDPOINT", "oss-ap-northeast-1-internal.aliyuncs.com") -REGION = os.getenv("OSS_REGION", "ap-northeast-1") -PROFILE = os.getenv("ALIYUN_PROFILE", "ecs-role") -REST_BASE = os.getenv( - "BINANCE_REST_BASE", - "https://data-api.binance.vision" - if MARKET == "spot" - else "https://fapi.binance.com", -) -SNAPSHOT_LIMIT = int(os.getenv("SNAPSHOT_LIMIT", "100")) -SNAPSHOT_REQUESTS_PER_SECOND = float( - os.getenv("SNAPSHOT_REQUESTS_PER_SECOND", "15") -) -WS_SHARD_SIZE = int(os.getenv("WS_SHARD_SIZE", "100")) -SYNC_TIMEOUT_SECONDS = int(os.getenv("SYNC_TIMEOUT_SECONDS", "120")) -STALL_TIMEOUT_SECONDS = int(os.getenv("STALL_TIMEOUT_SECONDS", "60")) -MAX_BUFFERED_DIFFS = int(os.getenv("MAX_BUFFERED_DIFFS", "250000")) -STARTUP_DELAY_SECONDS = int(os.getenv("STARTUP_DELAY_SECONDS", "0")) -SNAPSHOT_RETRY_ATTEMPTS = max( - 1, int(os.getenv("SNAPSHOT_RETRY_ATTEMPTS", "6")) -) -MAX_PENDING_DIFFS_TOTAL = int(os.getenv("MAX_PENDING_DIFFS_TOTAL", "250000")) -MIN_FREE_GB = int(os.getenv("MIN_FREE_GB", "20")) -ZSTD_TIMEOUT_SECONDS = int(os.getenv("ZSTD_TIMEOUT_SECONDS", "300")) -OSS_COPY_TIMEOUT_SECONDS = int(os.getenv("OSS_COPY_TIMEOUT_SECONDS", "300")) -PROCESS_WATCHDOG_SECONDS = int(os.getenv("PROCESS_WATCHDOG_SECONDS", "180")) -TASK_CANCEL_TIMEOUT_SECONDS = int(os.getenv("TASK_CANCEL_TIMEOUT_SECONDS", "5")) -LAST_DATA_AT = time.monotonic() -PROCESS_WATCHDOG_ARMED = False -UPLOAD_STATUS: dict[str, str | None] = { - "last_success_at": None, - "last_error_at": None, - "last_error": None, -} - - -class SequenceGap(RuntimeError): - def __init__( - self, symbol: str, expected: int, first_update_id: int, final_update_id: int - ) -> None: - self.symbol = symbol - self.expected = expected - self.first_update_id = first_update_id - self.final_update_id = final_update_id - super().__init__( - f"{symbol} sequence gap expected={expected} " - f"received={first_update_id}-{final_update_id}" - ) - - -class SnapshotUnavailable(RuntimeError): - def __init__(self, symbol: str, status: int) -> None: - self.symbol = symbol.upper() - self.status = status - super().__init__(f"snapshot unavailable symbol={self.symbol} status={status}") - - -class TaskCancellationStuck(RuntimeError): - pass - - -class PendingBudget: - def __init__(self, limit: int) -> None: - self.limit = limit - self.count = 0 - - def reserve(self) -> None: - if self.count >= self.limit: - raise RuntimeError(f"pending diff budget exceeded limit={self.limit}") - self.count += 1 - - def release(self, count: int) -> None: - self.count = max(0, self.count - count) - - -class OrderBookState: - def __init__( - self, - symbol: str, - market: str = "spot", - pending_budget: PendingBudget | None = None, - ) -> None: - self.symbol = symbol.upper() - self.market = market - self.bids: dict[str, str] = {} - self.asks: dict[str, str] = {} - self.last_update_id: int | None = None - self.synced = False - self.bridged = False - self.pending: list[dict] = [] - self.pending_budget = pending_budget or PendingBudget(MAX_PENDING_DIFFS_TOTAL) - - @staticmethod - def _update_side(side: dict[str, str], levels: list[list[str]]) -> None: - for price, quantity in levels: - if Decimal(quantity) == 0: - side.pop(price, None) - else: - side[price] = quantity - - def _apply_levels(self, event: dict) -> None: - self._update_side(self.bids, event.get("b", [])) - self._update_side(self.asks, event.get("a", [])) - self.last_update_id = int(event["u"]) - - def _apply_after_snapshot(self, event: dict) -> bool: - assert self.last_update_id is not None - first_update_id = int(event["U"]) - final_update_id = int(event["u"]) - if final_update_id <= self.last_update_id: - return False - - if not self.bridged: - if self.market == "usdm": - previous_update_id = int(event.get("pu", -1)) - if previous_update_id == self.last_update_id: - self._apply_levels(event) - self.synced = True - self.bridged = True - return True - expected = self.last_update_id + (1 if self.market == "spot" else 0) - if first_update_id <= expected <= final_update_id: - self._apply_levels(event) - self.synced = True - self.bridged = True - return True - self.synced = False - raise SequenceGap( - self.symbol, expected, first_update_id, final_update_id - ) - - if self.market == "usdm": - previous_update_id = int(event.get("pu", -1)) - if previous_update_id == self.last_update_id: - self._apply_levels(event) - return True - self.synced = False - raise SequenceGap( - self.symbol, - self.last_update_id, - previous_update_id, - final_update_id, - ) - - expected = self.last_update_id + 1 - if first_update_id <= expected <= final_update_id: - self._apply_levels(event) - return True - self.synced = False - raise SequenceGap( - self.symbol, expected, first_update_id, final_update_id - ) - - def apply_diff(self, event: dict) -> bool: - if self.last_update_id is None: - self.pending_budget.reserve() - self.pending.append(event) - return False - return self._apply_after_snapshot(event) - - def install_snapshot(self, snapshot: dict) -> None: - self.bids = {price: quantity for price, quantity in snapshot["bids"]} - self.asks = {price: quantity for price, quantity in snapshot["asks"]} - self.last_update_id = int(snapshot["lastUpdateId"]) - self.synced = True - self.bridged = False - pending, self.pending = self.pending, [] - self.pending_budget.release(len(pending)) - for event in pending: - self._apply_after_snapshot(event) - - def invalidate_for_resync(self) -> None: - self.pending_budget.release(len(self.pending)) - self.last_update_id = None - self.synced = False - self.bridged = False - self.pending = [] - - def checkpoint(self, session_id: str) -> dict: - if self.last_update_id is None: - raise RuntimeError(f"{self.symbol} has no snapshot") - bids = sorted( - self.bids.items(), key=lambda item: Decimal(item[0]), reverse=True - ) - asks = sorted(self.asks.items(), key=lambda item: Decimal(item[0])) - return { - "session_id": session_id, - "symbol": self.symbol, - "last_update_id": self.last_update_id, - "synced": self.synced, - "bridged": self.bridged, - "bids": [list(level) for level in bids], - "asks": [list(level) for level in asks], - } - - -def stream_suffix() -> str: - if MODE != "diff": - raise ValueError(f"snapshot reconciliation requires DEPTH_MODE=diff, got {MODE}") - return "@depth@100ms" - - -def discover_symbols_sync() -> tuple[tuple[str, ...], tuple[str, ...]]: - if MARKET == "spot": - url = f"{REST_BASE}/api/v3/exchangeInfo" - elif MARKET == "usdm": - url = f"{REST_BASE}/fapi/v1/exchangeInfo" - else: - raise ValueError(f"unsupported MARKET={MARKET}") - request = urllib.request.Request( - url, headers={"User-Agent": "monday-lob-archiver/2"} - ) - with urllib.request.urlopen(request, timeout=30) as response: - exchange_info = json.load(response) - - symbols = [] - security_tokens = [] - for item in exchange_info["symbols"]: - if item.get("status") != "TRADING": - continue - if MARKET == "spot" and not item.get("isSpotTradingAllowed", True): - continue - if MARKET == "usdm" and item.get("contractType") != "PERPETUAL": - continue - symbol = item["symbol"].lower() - symbols.append(symbol) - permission_groups = { - permission - for group in item.get("permissionSets", []) - for permission in group - } - if "TRD_GRP_261" in permission_groups: - security_tokens.append(symbol) - return tuple(sorted(symbols)), tuple(sorted(security_tokens)) - - -def stream_urls() -> tuple[str, ...]: - base = ( - "wss://data-stream.binance.vision/stream?streams=" - if MARKET == "spot" - else "wss://fstream.binance.com/stream?streams=" - ) - urls = [] - for index in range(0, len(SYMBOLS), WS_SHARD_SIZE): - streams = "/".join( - symbol + stream_suffix() - for symbol in SYMBOLS[index : index + WS_SHARD_SIZE] - ) - urls.append(base + streams) - return tuple(urls) - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for block in iter(lambda: source.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def segment_partition(timestamp_ns: int) -> tuple[str, str]: - value = datetime.fromtimestamp(timestamp_ns / 1_000_000_000, timezone.utc) - return value.strftime("%Y-%m-%d"), value.strftime("%H") - - -def finalize_segment( - path: Path, - counts: Counter, - start_ns: int, - end_ns: int, - schema: str = "binance.lob_tape.v2", - replay_safe: bool = False, -) -> Path | None: - if not path.exists() or path.stat().st_size == 0: - path.unlink(missing_ok=True) - return None - - output = path.with_suffix("").with_suffix(".jsonl.zst") - temporary_output = output.with_suffix(output.suffix + ".tmp") - subprocess.run( - [ - "zstd", - "-q", - "-f", - "-T1", - "-3", - str(path), - "-o", - str(temporary_output), - ], - check=True, - timeout=ZSTD_TIMEOUT_SECONDS, - ) - with temporary_output.open("rb") as compressed: - os.fsync(compressed.fileno()) - temporary_output.replace(output) - date, hour = segment_partition(start_ns) - metadata = { - "schema": schema, - "venue": "binance", - "market": MARKET, - "dataset": DATASET, - "shard_id": SHARD_ID, - "mode": MODE, - "symbols": list(SYMBOLS), - "security_token_symbols": list(SECURITY_TOKEN_SYMBOLS), - "excluded_symbols": list(EXCLUDED_SYMBOLS), - "snapshot_limit": SNAPSHOT_LIMIT, - "replay_scope": "captured_snapshot_seed_plus_sequence_checked_diffs", - "venue_depth_complete": False, - "events": sum(counts.values()), - "event_types": dict(sorted(counts.items())), - "has_replay_safe_checkpoint": replay_safe and counts["checkpoint"] > 0, - "start_received_at_ns": start_ns, - "end_received_at_ns": end_ns, - "date": date, - "hour": hour, - "file": output.name, - "bytes": output.stat().st_size, - "sha256": sha256(output), - } - manifest = output.with_name(output.name + ".manifest.json") - temporary = manifest.with_suffix(manifest.suffix + ".tmp") - temporary.write_text(json.dumps(metadata, sort_keys=True) + "\n") - temporary.replace(manifest) - path.unlink() - return manifest - - -def upload(manifest: Path) -> None: - metadata = json.loads(manifest.read_text()) - data = manifest.with_name(metadata["file"]) - prefix = ( - f"lake/raw/venue=binance/market={metadata['market']}" - f"/dataset={metadata['dataset']}/shard={metadata['shard_id']}" - f"/date={metadata['date']}/hour={metadata['hour']}" - ) - common = [ - "--profile", - PROFILE, - "--endpoint", - ENDPOINT, - "--region", - REGION, - "--force", - ] - - def copy(source: Path, name: str) -> None: - subprocess.run( - [ - "aliyun", - "ossutil", - "cp", - str(source), - f"oss://{BUCKET}/{prefix}/{name}", - *common, - ], - check=True, - timeout=OSS_COPY_TIMEOUT_SECONDS, - ) - - copy(data, data.name) - copy(manifest, manifest.name) - success = manifest.with_name(data.name + "._SUCCESS") - success.write_text(metadata["sha256"] + "\n") - copy(success, success.name) - data.unlink() - manifest.unlink() - success.unlink() - UPLOAD_STATUS["last_success_at"] = datetime.now(timezone.utc).isoformat() - LOG.info( - "uploaded segment events=%s types=%s key=%s/%s", - metadata["events"], - metadata["event_types"], - prefix, - data.name, - ) - - -def upload_pending() -> None: - had_error = False - for manifest in sorted(SPOOL_DIR.rglob("*.manifest.json")): - try: - upload(manifest) - except Exception as error: - had_error = True - UPLOAD_STATUS["last_error_at"] = datetime.now(timezone.utc).isoformat() - UPLOAD_STATUS["last_error"] = str(error)[:500] - LOG.exception("pending upload failed: %s", manifest) - if not had_error: - UPLOAD_STATUS["last_error_at"] = None - UPLOAD_STATUS["last_error"] = None - - -def pending_upload_count() -> int: - return sum(1 for _ in SPOOL_DIR.rglob("*.manifest.json")) - - -def recover_parts() -> None: - for path in sorted(SPOOL_DIR.rglob("*.jsonl.part")): - counts = Counter() - start_ns = end_ns = path.stat().st_mtime_ns - schema = "binance.lob_tape.v2" - valid_lines = 0 - corrupted = False - path.with_suffix(path.suffix + ".recovering").unlink(missing_ok=True) - with path.open("rb+") as source: - while True: - line_start = source.tell() - line = source.readline() - if not line: - break - try: - event = json.loads(line) - if not isinstance(event, dict): - raise ValueError("archive event must be a JSON object") - received = int(event["received_at_ns"]) - event_type = event.get("type") - if event_type is None: - event_type = "diff" - schema = "binance.raw_diff_tape.v1" - counts[event_type] += 1 - if valid_lines == 0: - start_ns = received - end_ns = received - valid_lines += 1 - except ( - AttributeError, - TypeError, - ValueError, - KeyError, - json.JSONDecodeError, - ): - if source.read(1): - quarantine = path.with_suffix(path.suffix + ".corrupt") - source.close() - path.replace(quarantine) - LOG.error( - "recovery quarantined JSONL with an invalid middle " - "record: %s", - quarantine, - ) - corrupted = True - else: - source.seek(line_start) - source.truncate() - source.flush() - os.fsync(source.fileno()) - LOG.warning( - "recovery dropped 1 invalid trailing JSONL line: %s", - path, - ) - break - if corrupted: - continue - finalize_segment( - path, - counts, - start_ns, - end_ns, - schema=schema, - replay_safe=False, - ) - - -class Segment: - def __init__(self) -> None: - self.start_ns = time.time_ns() - self.end_ns = self.start_ns - date, hour = segment_partition(self.start_ns) - directory = SPOOL_DIR / f"date={date}" / f"hour={hour}" - directory.mkdir(parents=True, exist_ok=True) - self.path = directory / f"part-{self.start_ns}.jsonl.part" - self.file = self.path.open("ab", buffering=1024 * 1024) - self.counts: Counter = Counter() - self.replay_safe = True - - def write(self, event_type: str, payload: dict, received_at_ns: int | None = None) -> None: - received_at_ns = received_at_ns or time.time_ns() - self.end_ns = max(self.end_ns, received_at_ns) - envelope = { - "received_at_ns": received_at_ns, - "type": event_type, - **payload, - } - self.file.write( - json.dumps(envelope, separators=(",", ":"), sort_keys=False).encode() - + b"\n" - ) - self.counts[event_type] += 1 - - def due(self) -> bool: - old_date, old_hour = segment_partition(self.start_ns) - new_date, new_hour = segment_partition(time.time_ns()) - return ( - time.time_ns() - self.start_ns >= SEGMENT_SECONDS * 1_000_000_000 - or (old_date, old_hour) != (new_date, new_hour) - ) - - def mark_replay_unsafe(self) -> None: - self.replay_safe = False - - def close(self) -> Path | None: - self.file.flush() - os.fsync(self.file.fileno()) - self.file.close() - return finalize_segment( - self.path, - self.counts, - self.start_ns, - self.end_ns, - replay_safe=self.replay_safe, - ) - - -def fetch_snapshot_sync(symbol: str) -> dict: - query = urllib.parse.urlencode({"symbol": symbol.upper(), "limit": SNAPSHOT_LIMIT}) - path = "/api/v3/depth" if MARKET == "spot" else "/fapi/v1/depth" - request = urllib.request.Request( - f"{REST_BASE}{path}?{query}", - headers={"User-Agent": "monday-lob-archiver/2"}, - ) - started_at_ns = time.time_ns() - for attempt in range(SNAPSHOT_RETRY_ATTEMPTS): - try: - with urllib.request.urlopen(request, timeout=15) as response: - snapshot = json.load(response) - break - except HTTPError as error: - if error.code == 400: - try: - payload = json.load(error) - except (ValueError, json.JSONDecodeError): - payload = {} - if payload.get("code") == -1121: - raise SnapshotUnavailable(symbol, error.code) from error - retryable = error.code == 429 or 500 <= error.code < 600 - if not retryable or attempt + 1 == SNAPSHOT_RETRY_ATTEMPTS: - raise - retry_after = error.headers.get("Retry-After") - delay = float(retry_after) if retry_after else min(60, 2**attempt) - LOG.warning( - "snapshot retry symbol=%s status=%s delay=%ss", - symbol, - error.code, - delay, - ) - time.sleep(delay) - except URLError: - if attempt + 1 == SNAPSHOT_RETRY_ATTEMPTS: - raise - time.sleep(min(30, 2**attempt)) - received_at_ns = time.time_ns() - if "lastUpdateId" not in snapshot: - raise RuntimeError(f"snapshot missing lastUpdateId for {symbol}") - return { - "symbol": symbol.upper(), - "request_started_at_ns": started_at_ns, - "received_at_ns": received_at_ns, - "snapshot": snapshot, - } - - -async def receive_url(url: str, queue: asyncio.Queue, stop: asyncio.Event) -> None: - global LAST_DATA_AT - async with connect( - url, open_timeout=20, ping_interval=20, max_size=8 * 1024 * 1024 - ) as websocket: - while not stop.is_set(): - try: - message = await asyncio.wait_for( - websocket.recv(), timeout=STALL_TIMEOUT_SECONDS - ) - except asyncio.TimeoutError: - raise RuntimeError( - f"no depth frames for {STALL_TIMEOUT_SECONDS}s on websocket shard" - ) - if isinstance(message, str): - LAST_DATA_AT = time.monotonic() - await queue.put(("diff", time.time_ns(), json.loads(message))) - - -async def produce_snapshots( - queue: asyncio.Queue, - limiter: "SnapshotRateLimiter | None" = None, -) -> None: - limiter = limiter or SnapshotRateLimiter(SNAPSHOT_REQUESTS_PER_SECOND) - for symbol in SYMBOLS: - snapshot = await limiter.fetch(symbol) - await queue.put(("snapshot", snapshot["received_at_ns"], snapshot)) - - -class SnapshotRateLimiter: - def __init__(self, requests_per_second: float) -> None: - self.interval = 1 / requests_per_second - self.next_started_at = 0.0 - self.lock = asyncio.Lock() - - async def fetch(self, symbol: str) -> dict: - async with self.lock: - now = time.monotonic() - delay = max(0.0, self.next_started_at - now) - if delay: - await asyncio.sleep(delay) - self.next_started_at = time.monotonic() + self.interval - return await asyncio.to_thread(fetch_snapshot_sync, symbol) - - -async def produce_snapshot( - symbol: str, - queue: asyncio.Queue, - limiter: SnapshotRateLimiter | None = None, -) -> None: - limiter = limiter or SnapshotRateLimiter(SNAPSHOT_REQUESTS_PER_SECOND) - snapshot = await limiter.fetch(symbol) - await queue.put(("snapshot", snapshot["received_at_ns"], snapshot)) - - -def frame_data(frame: dict) -> tuple[str, dict]: - data = frame.get("data", frame) - symbol = str(data.get("s", "")).upper() - if not symbol or "U" not in data or "u" not in data: - raise ValueError("depth frame missing symbol or sequence fields") - return symbol, data - - -def is_stalled(last_frame_at: float, now: float) -> bool: - return now - last_frame_at > STALL_TIMEOUT_SECONDS - - -def process_watchdog_expired(last_data_at: float, now: float) -> bool: - return now - last_data_at > PROCESS_WATCHDOG_SECONDS - - -def run_process_watchdog() -> None: - interval = max(1.0, min(10.0, PROCESS_WATCHDOG_SECONDS / 4)) - while True: - time.sleep(interval) - if not PROCESS_WATCHDOG_ARMED: - continue - now = time.monotonic() - if process_watchdog_expired(LAST_DATA_AT, now): - LOG.critical( - "process watchdog exiting after %.1fs without market data", - now - LAST_DATA_AT, - ) - os._exit(75) - - -def task_cancelling(task: asyncio.Task) -> int | str: - return task.cancelling() if hasattr(task, "cancelling") else "unsupported" - - -async def cancel_tasks_bounded(tasks: tuple[asyncio.Task, ...]) -> None: - if not tasks: - return - cancel_requested = {task: task.cancel() for task in tasks} - _, pending = await asyncio.wait( - tasks, timeout=TASK_CANCEL_TIMEOUT_SECONDS - ) - if pending: - for task in pending: - stack = task.get_stack(limit=1) - location = ( - f"{stack[-1].f_code.co_name}:{stack[-1].f_lineno}" - if stack - else "unknown" - ) - LOG.error( - "task cancellation timed out name=%s coro=%s " - "cancel_requested=%s cancelling=%s location=%s", - task.get_name(), - task.get_coro().__qualname__, - cancel_requested[task], - task_cancelling(task), - location, - ) - raise TaskCancellationStuck( - f"task cancellation timed out pending={len(pending)}" - ) - - -def bridge_timed_out( - previously_synced: bool, sync_deadline: float | None, now: float -) -> bool: - return ( - not previously_synced - and sync_deadline is not None - and now > sync_deadline - ) - - -def begin_resync(now: float) -> tuple[bool, float]: - return False, now + SYNC_TIMEOUT_SECONDS - - -def exclude_unavailable_symbols( - excluded_symbols: tuple[str, ...], - symbols: tuple[str, ...], - security_tokens: tuple[str, ...], -) -> tuple[tuple[str, ...], tuple[str, ...]]: - excluded = {symbol.lower() for symbol in excluded_symbols} - return ( - tuple(item for item in symbols if item not in excluded), - tuple(item for item in security_tokens if item not in excluded), - ) - - -def disk_headroom() -> tuple[float, bool]: - free_gb = shutil.disk_usage(SPOOL_DIR).free / 1024**3 - return round(free_gb, 1), free_gb < MIN_FREE_GB - - -def warn_if_disk_low() -> tuple[float, bool]: - free_gb, warning = disk_headroom() - if warning: - LOG.warning( - "spool free space below %sGiB: %.1fGiB; continuing collection; " - "successfully uploaded segments are removed locally, but pending " - "segments are retained to prevent data loss", - MIN_FREE_GB, - free_gb, - ) - return free_gb, warning - - -def write_health( - states: dict[str, OrderBookState], session_id: str, status: str, gaps: int -) -> None: - disk_free_gb, disk_warning = disk_headroom() - pending_segments = pending_upload_count() - health = { - "updated_at": datetime.now(timezone.utc).isoformat(), - "status": status, - "market": MARKET, - "dataset": DATASET, - "symbol_count": len(states), - "security_token_count": len(SECURITY_TOKEN_SYMBOLS), - "excluded_symbols": list(EXCLUDED_SYMBOLS), - "session_id": session_id, - "sequence_gaps": gaps, - "disk_free_gb": disk_free_gb, - "disk_warning": disk_warning, - "disk_warning_threshold_gb": MIN_FREE_GB, - "pending_upload_segments": pending_segments, - "upload_warning": UPLOAD_STATUS["last_error_at"] is not None, - "last_upload_success_at": UPLOAD_STATUS["last_success_at"], - "last_upload_error_at": UPLOAD_STATUS["last_error_at"], - "last_upload_error": UPLOAD_STATUS["last_error"], - "symbols": { - symbol: { - "synced": state.synced, - "bridged": state.bridged, - "last_update_id": state.last_update_id, - "bid_levels": len(state.bids), - "ask_levels": len(state.asks), - } - for symbol, state in sorted(states.items()) - }, - } - path = SPOOL_DIR / "health.json" - temporary = path.with_suffix(".json.tmp") - temporary.write_text(json.dumps(health, sort_keys=True) + "\n") - temporary.replace(path) - - -class ArchiveRuntime: - def __init__(self) -> None: - self.segment = Segment() - self.total_gaps = 0 - self.last_upload_retry = 0.0 - self.upload_thread: threading.Thread | None = None - - def write_checkpoints( - self, states: dict[str, OrderBookState], session_id: str, reason: str - ) -> None: - if not self.segment.replay_safe: - return - if not states or any(not state.synced for state in states.values()): - self.segment.mark_replay_unsafe() - return - for state in states.values(): - self.segment.write( - "checkpoint", - { - "reason": reason, - "replay_safe": True, - **state.checkpoint(session_id), - }, - ) - - async def retry_uploads_if_due(self, force: bool = False) -> None: - if self.upload_thread is not None and not self.upload_thread.is_alive(): - self.upload_thread = None - now = time.monotonic() - if not force and now - self.last_upload_retry < 300: - return - if self.upload_thread is not None: - return - self.last_upload_retry = now - self.upload_thread = threading.Thread( - target=upload_pending, - name=f"binance-oss-upload-{MARKET}", - daemon=True, - ) - self.upload_thread.start() - - async def finish_uploads(self) -> None: - if self.upload_thread is not None and self.upload_thread.is_alive(): - LOG.warning( - "shutdown leaving OSS upload in progress; local segment remains " - "retryable if the process is terminated" - ) - - async def rotate( - self, - states: dict[str, OrderBookState], - session_id: str, - reason: str, - create_next: bool = True, - ) -> None: - self.write_checkpoints(states, session_id, reason) - closing_segment = self.segment - if create_next: - self.segment = Segment() - manifest = await asyncio.to_thread(closing_segment.close) - if manifest: - await self.retry_uploads_if_due(force=True) - - -def archive_and_apply_diff( - runtime: ArchiveRuntime, - states: dict[str, OrderBookState], - session_id: str, - received_at_ns: int, - frame: dict, -) -> None: - symbol, data = frame_data(frame) - runtime.segment.write( - "diff", {"session_id": session_id, "frame": frame}, received_at_ns - ) - try: - states[symbol].apply_diff(data) - except SequenceGap as gap: - runtime.segment.mark_replay_unsafe() - runtime.segment.write( - "sequence_gap", - { - "session_id": session_id, - "symbol": gap.symbol, - "expected": gap.expected, - "first_update_id": gap.first_update_id, - "final_update_id": gap.final_update_id, - }, - ) - raise - - -async def run_session( - stop: asyncio.Event, - runtime: ArchiveRuntime, - states: dict[str, OrderBookState], - session_id: str, -) -> None: - queue: asyncio.Queue = asyncio.Queue(maxsize=MAX_BUFFERED_DIFFS) - failure: BaseException | None = None - receivers = [ - asyncio.create_task( - receive_url(url, queue, stop), name=f"receiver-{index}" - ) - for index, url in enumerate(stream_urls()) - ] - snapshot_limiter = SnapshotRateLimiter(SNAPSHOT_REQUESTS_PER_SECOND) - snapshotter = asyncio.create_task( - produce_snapshots(queue, snapshot_limiter), name="snapshotter" - ) - tasks = [*receivers, snapshotter] - resync_tasks: dict[str, asyncio.Task] = {} - LOG.info( - "connected market=%s symbols=%s websocket_shards=%s session=%s", - MARKET, - len(SYMBOLS), - len(receivers), - session_id, - ) - runtime.segment.write( - "session_start", - { - "session_id": session_id, - "market": MARKET, - "symbols": len(SYMBOLS), - "websocket_shards": len(receivers), - }, - ) - sync_deadline: float | None = None - last_frame_at = time.monotonic() - last_health_write = 0.0 - last_task_check = 0.0 - last_disk_check = 0.0 - previously_synced = False - try: - while not stop.is_set(): - try: - event_type, received_at_ns, payload = await asyncio.wait_for( - queue.get(), timeout=1 - ) - except asyncio.TimeoutError: - for receiver in receivers: - if receiver.done(): - await receiver - if snapshotter.done(): - await snapshotter - sync_deadline = sync_deadline or ( - time.monotonic() + SYNC_TIMEOUT_SECONDS - ) - if is_stalled(last_frame_at, time.monotonic()): - raise RuntimeError( - f"no depth frames for {STALL_TIMEOUT_SECONDS}s" - ) - if runtime.segment.due(): - await runtime.rotate(states, session_id, "scheduled") - now = time.monotonic() - if now - last_disk_check >= 60: - warn_if_disk_low() - last_disk_check = now - await runtime.retry_uploads_if_due() - continue - - try: - if event_type == "diff": - archive_and_apply_diff( - runtime, states, session_id, received_at_ns, payload - ) - last_frame_at = time.monotonic() - else: - symbol = payload["symbol"] - runtime.segment.write( - "snapshot", - { - "session_id": session_id, - "symbol": symbol, - "request_started_at_ns": payload["request_started_at_ns"], - "snapshot": payload["snapshot"], - }, - received_at_ns, - ) - states[symbol].install_snapshot(payload["snapshot"]) - except SequenceGap as gap: - if event_type == "snapshot": - runtime.segment.mark_replay_unsafe() - runtime.segment.write( - "sequence_gap", - { - "session_id": session_id, - "symbol": gap.symbol, - "expected": gap.expected, - "first_update_id": gap.first_update_id, - "final_update_id": gap.final_update_id, - }, - ) - runtime.total_gaps += 1 - states[gap.symbol].invalidate_for_resync() - previously_synced, sync_deadline = begin_resync(time.monotonic()) - write_health(states, session_id, "resyncing", runtime.total_gaps) - await runtime.rotate(states, session_id, "sequence_gap") - if gap.symbol not in resync_tasks or resync_tasks[gap.symbol].done(): - resync_tasks[gap.symbol] = asyncio.create_task( - produce_snapshot( - gap.symbol.lower(), queue, snapshot_limiter - ), - name=f"resync-{gap.symbol}", - ) - continue - - if snapshotter.done(): - await snapshotter - sync_deadline = sync_deadline or ( - time.monotonic() + SYNC_TIMEOUT_SECONDS - ) - all_synced = all(state.synced for state in states.values()) - now = time.monotonic() - if all_synced and snapshotter.done(): - if not previously_synced or now - last_health_write >= 30: - write_health(states, session_id, "synced", runtime.total_gaps) - last_health_write = now - previously_synced = True - elif bridge_timed_out(previously_synced, sync_deadline, now): - missing = [symbol for symbol, state in states.items() if not state.synced] - raise RuntimeError(f"snapshot bridge timed out: {missing}") - if runtime.segment.due(): - await runtime.rotate(states, session_id, "scheduled") - if now - last_task_check >= 1: - for receiver in receivers: - if receiver.done(): - await receiver - for symbol, task in tuple(resync_tasks.items()): - if task.done(): - await task - del resync_tasks[symbol] - last_task_check = now - if now - last_disk_check >= 60: - warn_if_disk_low() - last_disk_check = now - await runtime.retry_uploads_if_due() - except BaseException as error: - failure = error - raise - finally: - LOG.info( - "cancelling session tasks queue_size=%s queue_maxsize=%s", - queue.qsize(), - queue.maxsize, - ) - await cancel_tasks_bounded(tuple([*tasks, *resync_tasks.values()])) - archive_only = isinstance(failure, SequenceGap) - drain_gap: SequenceGap | None = None - while not queue.empty(): - event_type, received_at_ns, payload = queue.get_nowait() - if event_type == "snapshot": - symbol = payload["symbol"] - runtime.segment.write( - "snapshot", - { - "session_id": session_id, - "symbol": symbol, - "request_started_at_ns": payload["request_started_at_ns"], - "snapshot": payload["snapshot"], - }, - received_at_ns, - ) - if not archive_only: - states[symbol].install_snapshot(payload["snapshot"]) - continue - if archive_only: - runtime.segment.mark_replay_unsafe() - runtime.segment.write( - "diff", - { - "session_id": session_id, - "archived_only": True, - "frame": payload, - }, - received_at_ns, - ) - continue - try: - archive_and_apply_diff( - runtime, states, session_id, received_at_ns, payload - ) - except SequenceGap as gap: - drain_gap = gap - archive_only = True - if drain_gap is not None: - raise drain_gap - - -async def collect(stop: asyncio.Event) -> None: - global SYMBOLS, SECURITY_TOKEN_SYMBOLS, EXCLUDED_SYMBOLS - global LAST_DATA_AT, PROCESS_WATCHDOG_ARMED - SPOOL_DIR.mkdir(parents=True, exist_ok=True) - recover_parts() - LAST_DATA_AT = time.monotonic() - PROCESS_WATCHDOG_ARMED = True - runtime = ArchiveRuntime() - backoff = 1 - initial_budget = PendingBudget(MAX_PENDING_DIFFS_TOTAL) - last_states = { - symbol.upper(): OrderBookState(symbol, MARKET, initial_budget) - for symbol in SYMBOLS - } - last_session_id = "startup" - try: - while not stop.is_set(): - pending_budget = PendingBudget(MAX_PENDING_DIFFS_TOTAL) - states = { - symbol.upper(): OrderBookState(symbol, MARKET, pending_budget) - for symbol in SYMBOLS - } - session_id = uuid.uuid4().hex - last_states = states - last_session_id = session_id - try: - await run_session(stop, runtime, states, session_id) - backoff = 1 - except SequenceGap as gap: - runtime.total_gaps += 1 - write_health(states, session_id, "sequence_gap", runtime.total_gaps) - LOG.exception("sequence gap; reconnecting in %ss", backoff) - await runtime.rotate(states, session_id, "sequence_gap") - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 30) - except SnapshotUnavailable as error: - write_health( - states, session_id, "catalog_refresh", runtime.total_gaps - ) - LOG.exception( - "snapshot rejected; refreshing catalog and excluding %s", - error.symbol, - ) - await runtime.rotate(states, session_id, "snapshot_unavailable") - if SYMBOLS_SETTING.upper() == "ALL": - discovered, security_tokens = await asyncio.to_thread( - discover_symbols_sync - ) - EXCLUDED_SYMBOLS = tuple( - sorted({*EXCLUDED_SYMBOLS, error.symbol.lower()}) - ) - SYMBOLS, SECURITY_TOKEN_SYMBOLS = exclude_unavailable_symbols( - EXCLUDED_SYMBOLS, discovered, security_tokens - ) - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 30) - except TaskCancellationStuck: - write_health( - states, session_id, "fatal_task_stall", runtime.total_gaps - ) - LOG.critical( - "receiver cancellation stuck; exiting for systemd restart", - exc_info=True, - ) - raise - except Exception: - write_health(states, session_id, "reconnecting", runtime.total_gaps) - LOG.exception("websocket session failed; reconnecting in %ss", backoff) - await runtime.rotate(states, session_id, "reconnect") - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 30) - finally: - await runtime.rotate( - last_states, last_session_id, "shutdown", create_next=False - ) - await runtime.finish_uploads() - - -def self_test() -> None: - global SPOOL_DIR - state = OrderBookState("BTCUSDT") - state.apply_diff({"U": 101, "u": 102, "b": [["100", "0"]], "a": []}) - state.install_snapshot( - {"lastUpdateId": 100, "bids": [["100", "2"]], "asks": [["101", "3"]]} - ) - assert state.synced - assert state.last_update_id == 102 - assert "100" not in state.bids - - with tempfile.TemporaryDirectory() as temporary: - SPOOL_DIR = Path(temporary) - segment = Segment() - segment.write("snapshot", {"symbol": "BTCUSDT", "snapshot": {"lastUpdateId": 1}}) - segment.write("checkpoint", state.checkpoint("self-test")) - manifest = segment.close() - assert manifest is not None - metadata = json.loads(manifest.read_text()) - assert metadata["schema"] == "binance.lob_tape.v2" - assert metadata["replay_scope"] == ( - "captured_snapshot_seed_plus_sequence_checked_diffs" - ) - assert metadata["venue_depth_complete"] is False - assert metadata["event_types"] == {"checkpoint": 1, "snapshot": 1} - assert metadata["sha256"] == sha256(manifest.with_name(metadata["file"])) - print("self-test: ok") - - -def main() -> None: - global SYMBOLS, SECURITY_TOKEN_SYMBOLS - parser = argparse.ArgumentParser() - parser.add_argument("--self-test", action="store_true") - args = parser.parse_args() - if args.self_test: - self_test() - return - - if SYMBOLS_SETTING.upper() == "ALL": - SYMBOLS, SECURITY_TOKEN_SYMBOLS = discover_symbols_sync() - if not SYMBOLS: - raise SystemExit("SYMBOLS must not be empty") - if MODE != "diff": - raise SystemExit("snapshot reconciliation requires DEPTH_MODE=diff") - if MARKET not in {"spot", "usdm"}: - raise SystemExit(f"unsupported MARKET={MARKET}") - logging.basicConfig( - level=os.getenv("LOG_LEVEL", "INFO"), - format="%(asctime)s %(levelname)s %(message)s", - ) - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - stop = asyncio.Event() - for name in (signal.SIGINT, signal.SIGTERM): - loop.add_signal_handler(name, stop.set) - if STARTUP_DELAY_SECONDS: - LOG.info("startup delay=%ss", STARTUP_DELAY_SECONDS) - time.sleep(STARTUP_DELAY_SECONDS) - threading.Thread( - target=run_process_watchdog, - name=f"binance-data-watchdog-{MARKET}", - daemon=True, - ).start() - loop.run_until_complete(collect(stop)) - - -if __name__ == "__main__": - main() diff --git a/deployment/aliyun/deploy-rust-lob-release.sh b/deployment/aliyun/deploy-rust-lob-release.sh new file mode 100755 index 000000000..17516e802 --- /dev/null +++ b/deployment/aliyun/deploy-rust-lob-release.sh @@ -0,0 +1,304 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + printf '%s\n' \ + 'Usage: INSTANCE_ID=i-... ARTIFACT_OSS_URI=oss://... ARTIFACT_SHA256=<64 hex> SOURCE_REVISION= deploy-rust-lob-release.sh' \ + '' \ + 'Optional: REGION_ID=ap-northeast-1 ALIYUN_LOCAL_PROFILE=default' +} + +for command in aliyun base64 git jq tar; do + if ! command -v "$command" >/dev/null 2>&1; then + printf 'missing required command: %s\n' "$command" >&2 + exit 2 + fi +done +if ! command -v sha256sum >/dev/null 2>&1 && ! command -v shasum >/dev/null 2>&1; then + printf 'missing required SHA-256 command: sha256sum or shasum\n' >&2 + exit 2 +fi + +: "${INSTANCE_ID:?set INSTANCE_ID}" +: "${ARTIFACT_OSS_URI:?set ARTIFACT_OSS_URI}" +: "${ARTIFACT_SHA256:?set ARTIFACT_SHA256}" +: "${SOURCE_REVISION:?set SOURCE_REVISION}" + +REGION_ID=${REGION_ID:-ap-northeast-1} +ALIYUN_LOCAL_PROFILE=${ALIYUN_LOCAL_PROFILE:-default} +BUNDLE_OSS_PREFIX=${BUNDLE_OSS_PREFIX:-oss://monday-lob-apne1-1045353359/releases/binance-lob-archiver} + +if [[ "$REGION_ID" != 'ap-northeast-1' ]]; then + printf 'refusing non-Tokyo region: %s\n' "$REGION_ID" >&2 + exit 2 +fi +if [[ ! "$INSTANCE_ID" =~ ^i-[a-z0-9]+$ ]]; then + usage >&2 + exit 2 +fi +if [[ ! "$ARTIFACT_OSS_URI" =~ ^oss://[A-Za-z0-9][A-Za-z0-9.-]*/[A-Za-z0-9._/@+=:-]+$ ]]; then + printf 'ARTIFACT_OSS_URI must be a private OSS object URI without query parameters\n' >&2 + exit 2 +fi +if [[ ! "$ARTIFACT_SHA256" =~ ^[A-Fa-f0-9]{64}$ ]]; then + printf 'ARTIFACT_SHA256 must contain exactly 64 hexadecimal characters\n' >&2 + exit 2 +fi +if [[ ! "$SOURCE_REVISION" =~ ^[A-Fa-f0-9]{7,64}$ ]]; then + printf 'SOURCE_REVISION must be a 7-64 character hexadecimal Git revision\n' >&2 + exit 2 +fi +if [[ ! "$BUNDLE_OSS_PREFIX" =~ ^oss://[A-Za-z0-9][A-Za-z0-9.-]*/[A-Za-z0-9._/@+=:-]+$ ]]; then + printf 'BUNDLE_OSS_PREFIX is not a valid OSS prefix\n' >&2 + exit 2 +fi + +ARTIFACT_SHA256=$(printf '%s' "$ARTIFACT_SHA256" | tr '[:upper:]' '[:lower:]') +SOURCE_REVISION=$(printf '%s' "$SOURCE_REVISION" | tr '[:upper:]' '[:lower:]') +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT=$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel) +RESOLVED_SOURCE_REVISION=$(git -C "$REPO_ROOT" rev-parse "${SOURCE_REVISION}^{commit}") +HEAD_REVISION=$(git -C "$REPO_ROOT" rev-parse HEAD) +if [[ "$RESOLVED_SOURCE_REVISION" != "$HEAD_REVISION" ]]; then + printf 'SOURCE_REVISION must resolve to the current HEAD (%s)\n' "$HEAD_REVISION" >&2 + exit 2 +fi +if [[ -n $(git -C "$REPO_ROOT" status --porcelain --untracked-files=normal) ]]; then + printf 'refusing to build a deployment bundle from a dirty working tree\n' >&2 + exit 2 +fi +SOURCE_REVISION=$RESOLVED_SOURCE_REVISION +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT + +assets=( + binance-lob-archiver-production@.service + binance-lob-archiver-rust@.service + binance-lob-archiver-upload@.service + binance-lob-archiver-rust-upload@.service + binance-lob-archiver-production-spot.env + binance-lob-archiver-production-usdm.env + binance-lob-archiver-rust-spot.env + binance-lob-archiver-rust-usdm.env + host-rust-lob-shadow-gate.sh + host-rust-lob-cutover.sh + rust-lob-control-plane-lib.sh + rust-lob-runtime-health-policy.jq + rust-lob-shadow-gate-policy.jq +) +for asset in "${assets[@]}"; do + if [[ ! -f "$SCRIPT_DIR/$asset" ]]; then + printf 'missing deployment asset: %s\n' "$SCRIPT_DIR/$asset" >&2 + exit 2 + fi +done + +BUNDLE_PATH="$TMP_DIR/deployment.tar" +tar -C "$SCRIPT_DIR" -cf "$BUNDLE_PATH" "${assets[@]}" +if command -v sha256sum >/dev/null 2>&1; then + BUNDLE_SHA256=$(sha256sum "$BUNDLE_PATH" | awk '{print $1}') +else + BUNDLE_SHA256=$(shasum -a 256 "$BUNDLE_PATH" | awk '{print $1}') +fi +BUNDLE_OSS_URI="${BUNDLE_OSS_PREFIX%/}/${SOURCE_REVISION}/deployment-${BUNDLE_SHA256}.tar" + +aliyun_profile_args=() +if [[ -n "$ALIYUN_LOCAL_PROFILE" ]]; then + aliyun_profile_args=(--profile "$ALIYUN_LOCAL_PROFILE") +fi + +aliyun ossutil cp \ + "$BUNDLE_PATH" \ + "$BUNDLE_OSS_URI" \ + --endpoint oss-ap-northeast-1.aliyuncs.com \ + --region "$REGION_ID" \ + --force \ + "${aliyun_profile_args[@]}" + +printf -v remote_variables \ + 'artifact_uri=%q\nartifact_sha256=%q\nsource_revision=%q\nbundle_uri=%q\nbundle_sha256=%q\n' \ + "$ARTIFACT_OSS_URI" \ + "$ARTIFACT_SHA256" \ + "$SOURCE_REVISION" \ + "$BUNDLE_OSS_URI" \ + "$BUNDLE_SHA256" + +read -r -d '' remote_body <<'REMOTE_SCRIPT' || true +set -euo pipefail +umask 027 + +install -d -m 0755 /run/lock +exec 9>/run/lock/monday-rust-lob-release.lock +if ! flock -w 30 9; then + printf 'another Rust collector release operation holds the host lock\n' >&2 + exit 1 +fi +if ! mountpoint -q /data; then + printf '/data must be a mounted filesystem before collector installation\n' >&2 + exit 1 +fi +for path in \ + /data/monday \ + /data/monday/spool \ + /data/monday/spool/binance-lob-rust-shadow \ + /data/monday/spool/binance-lob-rust-shadow/spot \ + /data/monday/spool/binance-lob-rust-shadow/usdm; do + if [[ -L $path ]]; then + printf 'refusing symlink in shadow spool path: %s\n' "$path" >&2 + exit 1 + fi +done + +if systemctl is-active --quiet binance-lob-archiver-rust@spot.service \ + || systemctl is-active --quiet binance-lob-archiver-rust@usdm.service; then + printf 'refusing to replace the shadow candidate while a shadow unit is active\n' >&2 + exit 1 +fi + +work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' EXIT +artifact_tmp="$work_dir/binance-lob-archiver" +bundle_tmp="$work_dir/deployment.tar" +bundle_dir="$work_dir/deployment" +mkdir -p "$bundle_dir" + +aliyun ossutil cp "$artifact_uri" "$artifact_tmp" \ + --profile ecs-role \ + --endpoint oss-ap-northeast-1-internal.aliyuncs.com \ + --region ap-northeast-1 \ + --force +printf '%s %s\n' "$artifact_sha256" "$artifact_tmp" | sha256sum --check --strict + +aliyun ossutil cp "$bundle_uri" "$bundle_tmp" \ + --profile ecs-role \ + --endpoint oss-ap-northeast-1-internal.aliyuncs.com \ + --region ap-northeast-1 \ + --force +printf '%s %s\n' "$bundle_sha256" "$bundle_tmp" | sha256sum --check --strict +tar --no-same-owner --no-same-permissions -xf "$bundle_tmp" -C "$bundle_dir" + +if ! id hftcollector >/dev/null 2>&1; then + useradd --system --create-home --home-dir /var/lib/hft-collector \ + --shell /usr/sbin/nologin hftcollector +fi +install -d -m 0755 /opt/monday/bin +install -d -m 0755 "/opt/monday/releases/binance-lob-archiver/$artifact_sha256" +install -d -m 0755 "/opt/monday/releases/binance-lob-archiver/$artifact_sha256/deployment" +install -d -m 0755 /etc/monday +install -d -m 0750 -o hftcollector -g hftcollector \ + /data/monday/spool/binance-lob-rust-shadow/spot \ + /data/monday/spool/binance-lob-rust-shadow/usdm +for path in \ + /data/monday/spool/binance-lob-rust-shadow \ + /data/monday/spool/binance-lob-rust-shadow/spot \ + /data/monday/spool/binance-lob-rust-shadow/usdm; do + if [[ $(readlink -f "$path") != "$path" ]]; then + printf 'shadow spool resolved outside its canonical path: %s\n' "$path" >&2 + exit 1 + fi +done + +release_binary="/opt/monday/releases/binance-lob-archiver/$artifact_sha256/binance-lob-archiver" +install -m 0755 "$artifact_tmp" "$release_binary" +printf '%s %s\n' "$artifact_sha256" "$release_binary" | sha256sum --check --strict +"$release_binary" --self-test +"$release_binary" --help | grep -Fq -- '--upload-only' + +cp -a "$bundle_dir/." \ + "/opt/monday/releases/binance-lob-archiver/$artifact_sha256/deployment/" +install -m 0644 "$bundle_dir/binance-lob-archiver-rust@.service" \ + /etc/systemd/system/binance-lob-archiver-rust@.service +install -m 0644 "$bundle_dir/binance-lob-archiver-rust-upload@.service" \ + /etc/systemd/system/binance-lob-archiver-rust-upload@.service +install -m 0640 "$bundle_dir/binance-lob-archiver-rust-spot.env" \ + /etc/monday/binance-lob-archiver-rust-spot.env +install -m 0640 "$bundle_dir/binance-lob-archiver-rust-usdm.env" \ + /etc/monday/binance-lob-archiver-rust-usdm.env + +ln -sfn "$release_binary" /opt/monday/bin/binance-lob-archiver-shadow +printf '%s %s\n' "$artifact_sha256" /opt/monday/bin/binance-lob-archiver-shadow \ + | sha256sum --check --strict + +metadata_tmp="/opt/monday/releases/binance-lob-archiver/$artifact_sha256/release.json.tmp" +printf '{"artifact_uri":"%s","artifact_sha256":"%s","deployment_source_revision":"%s","deployment_bundle_uri":"%s","deployment_bundle_sha256":"%s"}\n' \ + "$artifact_uri" "$artifact_sha256" "$source_revision" "$bundle_uri" "$bundle_sha256" \ + > "$metadata_tmp" +mv "$metadata_tmp" "/opt/monday/releases/binance-lob-archiver/$artifact_sha256/release.json" +systemctl daemon-reload +printf 'installed Rust collector candidate %s from %s; no service was started\n' \ + "$artifact_sha256" "$source_revision" +REMOTE_SCRIPT + +remote_script=$'#!/usr/bin/env bash\n' +remote_script+="$remote_variables" +remote_script+="$remote_body" +command_content=$(printf '%s' "$remote_script" | base64 | tr -d '\n') + +run_json=$(aliyun ecs RunCommand \ + --RegionId "$REGION_ID" \ + --InstanceId.1 "$INSTANCE_ID" \ + --Type RunShellScript \ + --ContentEncoding Base64 \ + --CommandContent "$command_content" \ + --KeepCommand false \ + --Name monday-rust-lob-release-install \ + --Timeout 1200 \ + "${aliyun_profile_args[@]}") +invoke_id=$(printf '%s' "$run_json" | jq -er '.InvokeId') +printf 'Cloud Assistant invocation: %s\nDeployment bundle: %s\n' \ + "$invoke_id" "$BUNDLE_OSS_URI" + +for _ in $(seq 1 240); do + if ! result_json=$(aliyun ecs DescribeInvocationResults \ + --RegionId "$REGION_ID" \ + --InvokeId "$invoke_id" \ + --InstanceId "$INSTANCE_ID" \ + "${aliyun_profile_args[@]}"); then + sleep 5 + continue + fi + status=$(printf '%s' "$result_json" \ + | jq -r '[.. | objects | .InvocationStatus? // empty][0] // empty') + exit_code=$(printf '%s' "$result_json" \ + | jq -r '[.. | objects | .ExitCode? // empty][0] // empty') + case "$status" in + Success|Finished) + if [[ "$exit_code" == '0' ]]; then + printf 'candidate install completed successfully: %s\n' "$invoke_id" + exit 0 + fi + printf '%s\n' "$result_json" >&2 + exit 1 + ;; + Failed|Stopped|PartialFailed|Timeout) + printf '%s\n' "$result_json" >&2 + exit 1 + ;; + esac + sleep 5 +done + +printf 'timed out waiting for Cloud Assistant invocation %s\n' "$invoke_id" >&2 +aliyun ecs StopInvocation \ + --RegionId "$REGION_ID" \ + --InvokeId "$invoke_id" \ + --InstanceId.1 "$INSTANCE_ID" \ + "${aliyun_profile_args[@]}" >/dev/null || true +for _ in $(seq 1 12); do + result_json=$(aliyun ecs DescribeInvocationResults \ + --RegionId "$REGION_ID" \ + --InvokeId "$invoke_id" \ + --InstanceId "$INSTANCE_ID" \ + "${aliyun_profile_args[@]}" || true) + status=$(printf '%s' "$result_json" \ + | jq -r '[.. | objects | .InvocationStatus? // empty][0] // empty') + case "$status" in + Success|Finished|Failed|Stopped|PartialFailed|Timeout) + printf 'invocation reached terminal state after cancellation: %s\n' "$status" >&2 + exit 1 + ;; + esac + sleep 5 +done +printf 'invocation did not confirm cancellation: %s\n' "$invoke_id" >&2 +exit 1 diff --git a/deployment/aliyun/host-rust-lob-cutover.sh b/deployment/aliyun/host-rust-lob-cutover.sh new file mode 100755 index 000000000..778a14e49 --- /dev/null +++ b/deployment/aliyun/host-rust-lob-cutover.sh @@ -0,0 +1,720 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 027 +export LC_ALL=C + +usage() { + printf 'Usage: %s \n' "${0##*/}" >&2 +} + +if [[ ${EUID:-$(id -u)} -ne 0 ]]; then + printf 'must run as root\n' >&2 + exit 2 +fi +if [[ $# -ne 1 || ! $1 =~ ^[A-Fa-f0-9]{64}$ ]]; then + usage + exit 2 +fi + +for command in awk chmod date env find flock grep id install jq ln mountpoint mv readlink rm runuser sha256sum sleep stat systemctl tr wc; do + if ! command -v "$command" >/dev/null 2>&1; then + printf 'missing required command: %s\n' "$command" >&2 + exit 2 + fi +done + +CANDIDATE_SHA256=$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]') +RELEASE_ROOT=/opt/monday/releases/binance-lob-archiver +CANDIDATE_RELEASE="$RELEASE_ROOT/$CANDIDATE_SHA256" +CANDIDATE_BINARY="$CANDIDATE_RELEASE/binance-lob-archiver" +CANDIDATE_DEPLOYMENT="$CANDIDATE_RELEASE/deployment" +GATE_POLICY="$CANDIDATE_DEPLOYMENT/rust-lob-shadow-gate-policy.jq" +RUNTIME_HEALTH_POLICY="$CANDIDATE_DEPLOYMENT/rust-lob-runtime-health-policy.jq" +GATE_ROOT=/data/monday/evidence/shadow-gates +GATE_DIR= +GATE_JSON= +GATE_MARKER= +DEPLOYMENT_BUNDLE_SHA256= +DEPLOYMENT_SOURCE_REVISION= +PRODUCTION_LINK=/opt/monday/bin/binance-lob-archiver +SHADOW_LINK=/opt/monday/bin/binance-lob-archiver-shadow +CANONICAL_SPOOL=/data/monday/spool/binance-lob +HEALTH_TIMEOUT_SECONDS=300 +SAFE_PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +STARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ) +EVIDENCE_DIR="/data/monday/evidence/cutovers/$(date -u +%Y%m%dT%H%M%SZ)-${CANDIDATE_SHA256:0:12}-$$" + +PRODUCTION_UNITS=( + binance-lob-archiver-production@spot.service + binance-lob-archiver-production@usdm.service +) +UPLOAD_UNITS=( + binance-lob-archiver-upload@spot.service + binance-lob-archiver-upload@usdm.service +) +TRANSITION_MASK_UNITS=("${PRODUCTION_UNITS[@]}" "${UPLOAD_UNITS[@]}") +QUIESCENT_UNITS=( + binance-lob-archiver-rust@spot.service + binance-lob-archiver-rust@usdm.service + binance-lob-archiver-rust-upload@spot.service + binance-lob-archiver-rust-upload@usdm.service + binance-lob-archiver-upload@spot.service + binance-lob-archiver-upload@usdm.service +) +DEPLOYMENT_ASSETS=( + binance-lob-archiver-production@.service + binance-lob-archiver-upload@.service + binance-lob-archiver-production-spot.env + binance-lob-archiver-production-usdm.env +) +DRAIN_ENV_KEYS=( + SPOOL_DIR + OSS_BUCKET + OSS_ENDPOINT + OSS_REGION + ALIYUN_PROFILE + OSS_COPY_TIMEOUT_SECONDS +) + +install -d -m 0755 /run/lock +exec 9>/run/lock/monday-rust-lob-release.lock +if ! flock -n 9; then + printf 'another Rust collector release operation holds the host lock\n' >&2 + exit 1 +fi +exec 8>/run/lock/monday-rust-lob-shadow-gate.lock +if ! flock -n 8; then + printf 'a Rust collector shadow gate is still running\n' >&2 + exit 1 +fi +if [[ ! -d /data || -L /data ]] || ! mountpoint -q /data; then + printf '/data must be a mounted filesystem\n' >&2 + exit 1 +fi + +path_is_direct_or_absent() { + local path=$1 resolved + [[ -e $path || -L $path ]] || return 0 + [[ -d $path && ! -L $path ]] || return 1 + resolved=$(readlink -f -- "$path") || return 1 + [[ $resolved == "$path" ]] +} + +for path in /data/monday /data/monday/evidence /data/monday/evidence/cutovers; do + if ! path_is_direct_or_absent "$path"; then + printf 'evidence path contains a symlink: %s\n' "$path" >&2 + exit 1 + fi +done +install -d -m 0750 "$EVIDENCE_DIR" + +STEP=preflight +RESULT=preflight +FAILURE_REASON= +ROLLBACK_RESULT=not-needed +OLD_SHA256= +OLD_BINARY= +OLD_DEPLOYMENT= +OLD_MODE=new-host +TRANSITION_STARTED=0 +SUCCESS=0 +OLD_SESSION_SPOT= +OLD_SESSION_USDM= + +fail() { + FAILURE_REASON=$* + printf '%s\n' "$FAILURE_REASON" >&2 + exit 1 +} + +secure_regular_file() { + local path=$1 mode owner + [[ -f $path && ! -L $path ]] || fail "required regular file is missing or a symlink: $path" + owner=$(stat -c %u -- "$path") + mode=$(stat -c %a -- "$path") + [[ $owner == 0 ]] || fail "required file is not root-owned: $path" + (( (8#$mode & 022) == 0 )) || fail "required file is group/world writable: $path" +} + +env_value() { + local file=$1 key=$2 + awk -F= -v key="$key" ' + $1 == key { count += 1; value = substr($0, length(key) + 2) } + END { if (count != 1) exit 1; print value } + ' "$file" +} + +require_env_value() { + local file=$1 key=$2 expected=$3 actual + if ! actual=$(env_value "$file" "$key"); then + fail "$file must contain exactly one $key setting" + fi + [[ $actual == "$expected" ]] || fail "$file has unsafe $key=$actual (expected $expected)" +} + +validate_production_env() { + local file=$1 market=$2 dataset=$3 spool=$4 + require_env_value "$file" MARKET "$market" + require_env_value "$file" DATASET "$dataset" + require_env_value "$file" SHARD_ID all + require_env_value "$file" SYMBOLS ALL + require_env_value "$file" DEPTH_MODE diff + require_env_value "$file" SEGMENT_SECONDS 3600 + require_env_value "$file" SPOOL_DIR "$spool" + require_env_value "$file" OSS_BUCKET monday-lob-apne1-1045353359 + require_env_value "$file" OSS_ENDPOINT oss-ap-northeast-1-internal.aliyuncs.com + require_env_value "$file" OSS_REGION ap-northeast-1 + require_env_value "$file" ALIYUN_PROFILE ecs-role +} + +validate_deployment() { + local directory=$1 strict=${2:-false} asset + [[ -d $directory && ! -L $directory ]] || fail "staged deployment is missing: $directory" + for asset in "${DEPLOYMENT_ASSETS[@]}"; do + secure_regular_file "$directory/$asset" + done + + validate_production_env \ + "$directory/binance-lob-archiver-production-spot.env" \ + spot spot_all /data/monday/spool/binance-lob/spot + validate_production_env \ + "$directory/binance-lob-archiver-production-usdm.env" \ + usdm usdm_perpetual_all /data/monday/spool/binance-lob/usdm + + if [[ $strict == true ]]; then + grep -Fxq 'AssertPathIsMountPoint=/data' \ + "$directory/binance-lob-archiver-production@.service" \ + || fail 'candidate production unit does not assert the /data mount' + grep -Fxq 'EnvironmentFile=/etc/monday/binance-lob-archiver-production-%i.env' \ + "$directory/binance-lob-archiver-production@.service" \ + || fail 'candidate production unit has the wrong environment file' + grep -Fxq 'ExecStart=/opt/monday/bin/binance-lob-archiver' \ + "$directory/binance-lob-archiver-production@.service" \ + || fail 'candidate production unit has the wrong executable' + grep -Fxq 'ExecStartPre=/opt/monday/bin/binance-lob-archiver --self-test' \ + "$directory/binance-lob-archiver-production@.service" \ + || fail 'candidate production unit does not run the binary self-test' + grep -Fxq 'AssertPathIsMountPoint=/data' \ + "$directory/binance-lob-archiver-upload@.service" \ + || fail 'candidate upload unit does not assert the /data mount' + grep -Fxq 'ExecStart=/opt/monday/bin/binance-lob-archiver --upload-only' \ + "$directory/binance-lob-archiver-upload@.service" \ + || fail 'candidate upload unit is not explicitly upload-only' + grep -Fxq 'EnvironmentFile=/etc/monday/binance-lob-archiver-production-%i.env' \ + "$directory/binance-lob-archiver-upload@.service" \ + || fail 'candidate upload unit has the wrong environment file' + fi +} + +atomic_install() { + local mode=$1 source=$2 destination=$3 temporary + temporary="${destination}.new.$$" + install -m "$mode" "$source" "$temporary" || return 1 + mv -Tf "$temporary" "$destination" || return 1 +} + +install_deployment() { + local directory=$1 + install -d -m 0755 /etc/systemd/system || return 1 + install -d -m 0755 /etc/monday || return 1 + atomic_install 0644 "$directory/binance-lob-archiver-production@.service" \ + /etc/systemd/system/binance-lob-archiver-production@.service || return 1 + atomic_install 0644 "$directory/binance-lob-archiver-upload@.service" \ + /etc/systemd/system/binance-lob-archiver-upload@.service || return 1 + atomic_install 0640 "$directory/binance-lob-archiver-production-spot.env" \ + /etc/monday/binance-lob-archiver-production-spot.env || return 1 + atomic_install 0640 "$directory/binance-lob-archiver-production-usdm.env" \ + /etc/monday/binance-lob-archiver-production-usdm.env || return 1 +} + +atomic_symlink() { + local target=$1 link=$2 temporary + temporary="${link}.new.$$" + rm -f "$temporary" || return 1 + ln -s "$target" "$temporary" || return 1 + mv -Tf "$temporary" "$link" || return 1 +} + +canonical_spool_paths_safe() { + local path + for path in \ + /data/monday \ + /data/monday/spool \ + "$CANONICAL_SPOOL" \ + "$CANONICAL_SPOOL/spot" \ + "$CANONICAL_SPOOL/usdm"; do + path_is_direct_or_absent "$path" || return 1 + done +} + +segment_artifacts() { + canonical_spool_paths_safe || return 1 + [[ -d $CANONICAL_SPOOL ]] || return 0 + find "$CANONICAL_SPOOL" \( -type f -o -type l \) \( \ + -name 'part-*' -o \ + -name '*.manifest.json' -o \ + -name '*.jsonl.part' -o \ + -name '*.zst.tmp' -o \ + -name '*.part.corrupt' -o \ + -name '*.jsonl.zst' -o \ + -name '*._SUCCESS' -o \ + -name '*.uploaded-cleanup.json' -o \ + -name '*.uploaded-cleanup.json.tmp' \ + \) -print +} + +require_empty_segment_spool() { + local remaining + remaining=$(segment_artifacts) || return 1 + if [[ -n $remaining ]]; then + printf '%s\n' "$remaining" >&2 + return 1 + fi +} + +run_candidate_drain() { + local deployment=$1 market env_file key value + local -a env_args + canonical_spool_paths_safe || return 1 + for market in spot usdm; do + canonical_spool_paths_safe || return 1 + env_file="$deployment/binance-lob-archiver-production-$market.env" + env_args=() + for key in "${DRAIN_ENV_KEYS[@]}"; do + value=$(env_value "$env_file" "$key") || return 1 + [[ -n $value ]] || return 1 + env_args+=("$key=$value") + done + runuser --user hftcollector -- env -i \ + HOME=/var/lib/hft-collector \ + PATH="$SAFE_PATH" \ + RUST_LOG=info \ + "${env_args[@]}" \ + "$CANDIDATE_BINARY" --upload-only || return 1 + jq -e '.last_error == null' "$CANONICAL_SPOOL/$market/upload-status.json" >/dev/null \ + || return 1 + done + require_empty_segment_spool || return 1 +} + +stage_existing_deployment_for_rollback() { + local existing=0 asset source mode + [[ ! -L $OLD_DEPLOYMENT ]] || fail "old staged deployment is a symlink: $OLD_DEPLOYMENT" + for asset in "${DEPLOYMENT_ASSETS[@]}"; do + if [[ -e $OLD_DEPLOYMENT/$asset ]]; then + ((existing += 1)) + fi + done + if (( existing == ${#DEPLOYMENT_ASSETS[@]} )); then + validate_deployment "$OLD_DEPLOYMENT" false + return + fi + (( existing == 0 )) || fail "old release has a partial staged deployment: $OLD_DEPLOYMENT" + + install -d -m 0755 "$OLD_DEPLOYMENT" + for asset in "${DEPLOYMENT_ASSETS[@]}"; do + case "$asset" in + *.service) + source="/etc/systemd/system/$asset" + mode=0644 + ;; + *.env) + source="/etc/monday/$asset" + mode=0640 + ;; + esac + secure_regular_file "$source" + atomic_install "$mode" "$source" "$OLD_DEPLOYMENT/$asset" + done + validate_deployment "$OLD_DEPLOYMENT" false +} + +unit_active_json() { + local unit=$1 + if systemctl is-active --quiet "$unit"; then + printf true + else + printf false + fi +} + +copy_health_evidence() { + local label=$1 market source + for market in spot usdm; do + source="$CANONICAL_SPOOL/$market/health.json" + if [[ -f $source && ! -L $source ]]; then + install -m 0640 "$source" "$EVIDENCE_DIR/$label-$market-health.json" + fi + done +} + +health_ready_for_release() { + local market=$1 minimum_symbols=$2 old_session=$3 minimum_updated_ns=${4:-0} health + health="$CANONICAL_SPOOL/$market/health.json" + [[ -f $health && ! -L $health ]] || return 1 + jq -e \ + --arg old_session "$old_session" \ + --argjson minimum_symbols "$minimum_symbols" \ + --argjson minimum_updated_ns "$minimum_updated_ns" \ + -f "$RUNTIME_HEALTH_POLICY" "$health" >/dev/null +} + +runtime_matches_release() { + local binary=$1 require_enabled=$2 unit restarts main_pid main_exe + for unit in "${PRODUCTION_UNITS[@]}"; do + systemctl is-active --quiet "$unit" || return 1 + restarts=$(systemctl show "$unit" --property=NRestarts --value) || return 1 + [[ $restarts == 0 ]] || return 1 + main_pid=$(systemctl show "$unit" --property=MainPID --value) || return 1 + [[ $main_pid =~ ^[1-9][0-9]*$ ]] || return 1 + main_exe=$(readlink -f "/proc/$main_pid/exe" 2>/dev/null || true) + [[ $main_exe == "$binary" ]] || return 1 + if [[ $require_enabled == true ]]; then + systemctl is-enabled --quiet "$unit" || return 1 + fi + done +} + +wait_for_release_health() { + local binary=$1 old_spot_session=$2 old_usdm_session=$3 + local minimum_updated_ns=${4:-0} deadline unit + deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS)) + while (( SECONDS < deadline )); do + for unit in "${PRODUCTION_UNITS[@]}"; do + systemctl is-active --quiet "$unit" || return 1 + done + if health_ready_for_release spot 1000 "$old_spot_session" "$minimum_updated_ns" \ + && health_ready_for_release usdm 400 "$old_usdm_session" "$minimum_updated_ns" \ + && runtime_matches_release "$binary" false; then + return 0 + fi + sleep 5 + done + return 1 +} + +clear_health_before_restart() { + local market health + canonical_spool_paths_safe || return 1 + for market in spot usdm; do + health="$CANONICAL_SPOOL/$market/health.json" + rm -f -- "$health" || return 1 + [[ ! -e $health && ! -L $health ]] || return 1 + done +} + +production_is_fail_closed() { + local unit state + for unit in "${PRODUCTION_UNITS[@]}"; do + systemctl is-active --quiet "$unit" && return 1 + systemctl is-enabled --quiet "$unit" && return 1 + done + for unit in "${TRANSITION_MASK_UNITS[@]}"; do + state=$(systemctl is-enabled "$unit" 2>/dev/null || true) + [[ $state == masked || $state == masked-runtime ]] || return 1 + done +} + +write_evidence() { + local temporary current_target spot_active usdm_active + temporary="$EVIDENCE_DIR/cutover.json.tmp" + current_target=$(readlink -f "$PRODUCTION_LINK" 2>/dev/null || true) + spot_active=$(unit_active_json "${PRODUCTION_UNITS[0]}") + usdm_active=$(unit_active_json "${PRODUCTION_UNITS[1]}") + jq -n \ + --arg started_at "$STARTED_AT" \ + --arg completed_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg result "$RESULT" \ + --arg step "$STEP" \ + --arg failure_reason "$FAILURE_REASON" \ + --arg rollback_result "$ROLLBACK_RESULT" \ + --arg candidate_sha256 "$CANDIDATE_SHA256" \ + --arg deployment_bundle_sha256 "$DEPLOYMENT_BUNDLE_SHA256" \ + --arg previous_sha256 "$OLD_SHA256" \ + --arg mode "$OLD_MODE" \ + --arg current_binary "$current_target" \ + --argjson spot_active "$spot_active" \ + --argjson usdm_active "$usdm_active" \ + '{ + started_at: $started_at, + completed_at: $completed_at, + result: $result, + last_step: $step, + failure_reason: (if $failure_reason == "" then null else $failure_reason end), + rollback_result: $rollback_result, + candidate_sha256: $candidate_sha256, + deployment_bundle_sha256: (if $deployment_bundle_sha256 == "" then null else $deployment_bundle_sha256 end), + previous_sha256: (if $previous_sha256 == "" then null else $previous_sha256 end), + host_mode: $mode, + current_binary: (if $current_binary == "" then null else $current_binary end), + production_units_active: {spot: $spot_active, usdm: $usdm_active} + }' > "$temporary" || return 1 + chmod 0640 "$temporary" || return 1 + mv -Tf "$temporary" "$EVIDENCE_DIR/cutover.json" || return 1 +} + +rollback_after_failure() { + local safe_to_restart=1 unit rollback_started_ns=0 + ROLLBACK_RESULT=disabled + systemctl disable --now "${PRODUCTION_UNITS[@]}" >/dev/null 2>&1 || true + systemctl mask --runtime "${TRANSITION_MASK_UNITS[@]}" >/dev/null 2>&1 || true + for unit in "${PRODUCTION_UNITS[@]}"; do + if systemctl is-active --quiet "$unit" || systemctl is-enabled --quiet "$unit"; then + safe_to_restart=0 + fi + done + if (( safe_to_restart == 0 )); then + if production_is_fail_closed; then + ROLLBACK_RESULT=production-stop-or-disable-failed-but-contained + else + ROLLBACK_RESULT=production-stop-or-disable-containment-failed + fi + copy_health_evidence rollback + return + fi + + if [[ -d $CANONICAL_SPOOL ]]; then + if ! run_candidate_drain "$CANDIDATE_DEPLOYMENT"; then + safe_to_restart=0 + fi + fi + + if [[ $OLD_MODE == upgrade ]]; then + if ! install_deployment "$OLD_DEPLOYMENT"; then + safe_to_restart=0 + ROLLBACK_RESULT=restore-assets-failed-disabled + elif ! atomic_symlink "$OLD_BINARY" "$PRODUCTION_LINK"; then + safe_to_restart=0 + ROLLBACK_RESULT=restore-symlink-failed-disabled + else + systemctl daemon-reload || safe_to_restart=0 + systemctl unmask --runtime "${PRODUCTION_UNITS[@]}" >/dev/null \ + || safe_to_restart=0 + fi + + if (( safe_to_restart )); then + copy_health_evidence failed-candidate + if ! clear_health_before_restart; then + safe_to_restart=0 + ROLLBACK_RESULT=stale-health-clear-failed-disabled + else + rollback_started_ns=$(date +%s%N) + fi + fi + + if (( safe_to_restart )); then + systemctl reset-failed "${PRODUCTION_UNITS[@]}" >/dev/null 2>&1 || true + if systemctl start "${PRODUCTION_UNITS[@]}" \ + && wait_for_release_health \ + "$OLD_BINARY" "$OLD_SESSION_SPOT" "$OLD_SESSION_USDM" "$rollback_started_ns" \ + && systemctl enable "${PRODUCTION_UNITS[@]}" >/dev/null \ + && runtime_matches_release "$OLD_BINARY" true \ + && health_ready_for_release spot 1000 "$OLD_SESSION_SPOT" "$rollback_started_ns" \ + && health_ready_for_release usdm 400 "$OLD_SESSION_USDM" "$rollback_started_ns"; then + ROLLBACK_RESULT=previous-release-health-verified + systemctl unmask --runtime "${UPLOAD_UNITS[@]}" >/dev/null 2>&1 || true + else + systemctl disable --now "${PRODUCTION_UNITS[@]}" >/dev/null 2>&1 || true + systemctl mask --runtime "${TRANSITION_MASK_UNITS[@]}" >/dev/null 2>&1 || true + if production_is_fail_closed; then + ROLLBACK_RESULT=previous-release-health-unverified-disabled + else + ROLLBACK_RESULT=previous-release-health-unverified-containment-failed + fi + fi + else + systemctl disable --now "${PRODUCTION_UNITS[@]}" >/dev/null 2>&1 || true + systemctl mask --runtime "${TRANSITION_MASK_UNITS[@]}" >/dev/null 2>&1 || true + if ! production_is_fail_closed; then + ROLLBACK_RESULT=previous-release-restore-containment-failed + elif [[ $ROLLBACK_RESULT == disabled ]]; then + ROLLBACK_RESULT=previous-release-restored-disabled + fi + fi + else + if [[ $(readlink -f "$PRODUCTION_LINK" 2>/dev/null || true) == "$CANDIDATE_BINARY" ]]; then + rm -f "$PRODUCTION_LINK" + fi + ROLLBACK_RESULT=new-host-disabled + fi + copy_health_evidence rollback +} + +on_error() { + local rc=$? + if [[ -z $FAILURE_REASON ]]; then + FAILURE_REASON="command failed with exit $rc during $STEP: $BASH_COMMAND" + fi +} + +on_exit() { + local rc=$? + trap - EXIT ERR + set +e + if (( SUCCESS == 0 )); then + RESULT=failed + if (( TRANSITION_STARTED )); then + rollback_after_failure + fi + if write_evidence; then + printf 'cutover failed; evidence: %s/cutover.json\n' "$EVIDENCE_DIR" >&2 + else + printf 'cutover failed and evidence could not be written under %s\n' "$EVIDENCE_DIR" >&2 + fi + fi + exit "$rc" +} + +trap on_error ERR +trap on_exit EXIT + +STEP=validate-candidate-release +for path in /opt/monday /opt/monday/bin "$RELEASE_ROOT" "$CANDIDATE_RELEASE" "$CANDIDATE_DEPLOYMENT"; do + path_is_direct_or_absent "$path" || fail "release path contains a symlink: $path" +done +secure_regular_file "$CANDIDATE_BINARY" +[[ -x $CANDIDATE_BINARY ]] || fail "candidate binary is not executable: $CANDIDATE_BINARY" +printf '%s %s\n' "$CANDIDATE_SHA256" "$CANDIDATE_BINARY" | sha256sum --check --strict +secure_regular_file "$CANDIDATE_RELEASE/release.json" +secure_regular_file "$GATE_POLICY" +secure_regular_file "$RUNTIME_HEALTH_POLICY" +DEPLOYMENT_BUNDLE_SHA256=$(jq -er '.deployment_bundle_sha256' \ + "$CANDIDATE_RELEASE/release.json") +DEPLOYMENT_SOURCE_REVISION=$(jq -er '.deployment_source_revision' \ + "$CANDIDATE_RELEASE/release.json") +[[ $DEPLOYMENT_BUNDLE_SHA256 =~ ^[a-f0-9]{64}$ ]] \ + || fail 'candidate release has an invalid deployment bundle SHA-256' +[[ $DEPLOYMENT_SOURCE_REVISION =~ ^[a-f0-9]{40,64}$ ]] \ + || fail 'candidate release has an invalid deployment source revision' +jq -e --arg sha "$CANDIDATE_SHA256" --arg bundle "$DEPLOYMENT_BUNDLE_SHA256" \ + '.artifact_sha256 == $sha and .deployment_bundle_sha256 == $bundle' \ + "$CANDIDATE_RELEASE/release.json" >/dev/null \ + || fail 'candidate release metadata does not match the requested identity' +GATE_DIR="$GATE_ROOT/$CANDIDATE_SHA256/$DEPLOYMENT_BUNDLE_SHA256" +GATE_JSON="$GATE_DIR/gate.json" +GATE_MARKER="$GATE_DIR/PASSED.sha256" +validate_deployment "$CANDIDATE_DEPLOYMENT" true +id hftcollector >/dev/null 2>&1 || fail 'service account hftcollector is missing' +runuser -u hftcollector -- "$CANDIDATE_BINARY" --self-test +"$CANDIDATE_BINARY" --help | grep -Fq -- '--upload-only' +[[ $(readlink -f "$SHADOW_LINK" 2>/dev/null || true) == "$CANDIDATE_BINARY" ]] \ + || fail 'shadow symlink does not point to the gated candidate binary' + +STEP=validate-shadow-gate +for path in "$GATE_ROOT" "$GATE_ROOT/$CANDIDATE_SHA256" "$GATE_DIR"; do + path_is_direct_or_absent "$path" || fail "shadow gate path contains a symlink: $path" +done +secure_regular_file "$GATE_JSON" +secure_regular_file "$GATE_MARKER" +[[ $(wc -l < "$GATE_MARKER") -eq 1 ]] || fail 'PASSED.sha256 must contain exactly one entry' +marker_entry=$(<"$GATE_MARKER") +[[ $marker_entry =~ ^[A-Fa-f0-9]{64}[[:space:]]+gate\.json$ ]] \ + || fail 'PASSED.sha256 must contain only the gate.json SHA-256 entry' +(cd "$GATE_DIR" && sha256sum --check --strict PASSED.sha256) +jq -e \ + --arg candidate_sha256 "$CANDIDATE_SHA256" \ + --arg deployment_bundle_sha256 "$DEPLOYMENT_BUNDLE_SHA256" \ + --arg deployment_source_revision "$DEPLOYMENT_SOURCE_REVISION" \ + -f "$GATE_POLICY" "$GATE_JSON" >/dev/null \ + || fail 'candidate shadow gate does not meet production thresholds' +install -d -m 0750 "$EVIDENCE_DIR/shadow-gate" +install -m 0640 "$GATE_JSON" "$EVIDENCE_DIR/shadow-gate/gate.json" +install -m 0640 "$GATE_MARKER" "$EVIDENCE_DIR/shadow-gate/PASSED.sha256" + +STEP=validate-host-state +canonical_spool_paths_safe || fail 'canonical spool path contains a symlink or escapes /data' +for unit in "${QUIESCENT_UNITS[@]}"; do + systemctl is-active --quiet "$unit" && fail "unit must be inactive before cutover: $unit" +done + +active_count=0 +enabled_count=0 +OLD_SESSION_SPOT=$(jq -r '.session_id // empty' "$CANONICAL_SPOOL/spot/health.json" 2>/dev/null || true) +OLD_SESSION_USDM=$(jq -r '.session_id // empty' "$CANONICAL_SPOOL/usdm/health.json" 2>/dev/null || true) +for unit in "${PRODUCTION_UNITS[@]}"; do + if systemctl is-active --quiet "$unit"; then + ((active_count += 1)) + fi + if systemctl is-enabled --quiet "$unit"; then + ((enabled_count += 1)) + fi +done + +if (( active_count == 2 && enabled_count == 2 )); then + OLD_MODE=upgrade + [[ -L $PRODUCTION_LINK ]] || fail 'running production binary must be a release symlink' + OLD_BINARY=$(readlink -f "$PRODUCTION_LINK") + [[ $OLD_BINARY =~ ^$RELEASE_ROOT/([a-f0-9]{64})/binance-lob-archiver$ ]] \ + || fail "running production symlink is not digest-addressed: $OLD_BINARY" + OLD_SHA256=${BASH_REMATCH[1]} + [[ $OLD_SHA256 != "$CANDIDATE_SHA256" ]] || fail 'candidate is already the production release' + printf '%s %s\n' "$OLD_SHA256" "$OLD_BINARY" | sha256sum --check --strict + OLD_DEPLOYMENT="$RELEASE_ROOT/$OLD_SHA256/deployment" + stage_existing_deployment_for_rollback +elif (( active_count == 0 && enabled_count == 0 )) && [[ ! -e $PRODUCTION_LINK && ! -L $PRODUCTION_LINK ]]; then + OLD_MODE=new-host + require_empty_segment_spool || fail 'new host canonical spool contains segment artifacts' +else + fail "ambiguous production state: active=$active_count enabled=$enabled_count symlink=$PRODUCTION_LINK" +fi + +STEP=stop-production +TRANSITION_STARTED=1 +if [[ $OLD_MODE == upgrade ]]; then + systemctl disable --now "${PRODUCTION_UNITS[@]}" +else + systemctl disable "${PRODUCTION_UNITS[@]}" >/dev/null 2>&1 || true +fi +for unit in "${PRODUCTION_UNITS[@]}"; do + systemctl is-active --quiet "$unit" && fail "production unit did not stop: $unit" +done +systemctl mask --runtime "${TRANSITION_MASK_UNITS[@]}" >/dev/null +canonical_spool_paths_safe || fail 'canonical spool path changed during production stop' + +STEP=install-candidate-production-assets +validate_deployment "$CANDIDATE_DEPLOYMENT" true +install_deployment "$CANDIDATE_DEPLOYMENT" +install -d -m 0750 -o hftcollector -g hftcollector \ + "$CANONICAL_SPOOL/spot" "$CANONICAL_SPOOL/usdm" +systemctl daemon-reload + +if [[ $OLD_MODE == upgrade ]]; then + STEP=drain-old-production-with-candidate + run_candidate_drain "$CANDIDATE_DEPLOYMENT" +else + STEP=verify-new-host-spool + require_empty_segment_spool || fail 'new host canonical spool contains segment artifacts' +fi + +STEP=switch-production-symlink +atomic_symlink "$CANDIDATE_BINARY" "$PRODUCTION_LINK" +printf '%s %s\n' "$CANDIDATE_SHA256" "$PRODUCTION_LINK" | sha256sum --check --strict + +STEP=start-candidate-production +systemctl reset-failed "${PRODUCTION_UNITS[@]}" >/dev/null 2>&1 || true +systemctl unmask --runtime "${PRODUCTION_UNITS[@]}" >/dev/null +systemctl start "${PRODUCTION_UNITS[@]}" + +STEP=verify-candidate-production +wait_for_release_health "$CANDIDATE_BINARY" "$OLD_SESSION_SPOT" "$OLD_SESSION_USDM" \ + || fail 'candidate production did not reach verified full-catalog health' +copy_health_evidence production + +STEP=enable-verified-candidate +systemctl enable "${PRODUCTION_UNITS[@]}" >/dev/null +runtime_matches_release "$CANDIDATE_BINARY" true \ + || fail 'candidate runtime identity changed while enabling production' +health_ready_for_release spot 1000 "$OLD_SESSION_SPOT" \ + || fail 'Spot health changed while enabling production' +health_ready_for_release usdm 400 "$OLD_SESSION_USDM" \ + || fail 'USD-M health changed while enabling production' +systemctl unmask --runtime "${UPLOAD_UNITS[@]}" >/dev/null + +STEP=write-cutover-evidence +RESULT=passed +ROLLBACK_RESULT=not-needed +write_evidence +SUCCESS=1 +trap - EXIT ERR +printf 'Rust collector cutover passed: %s\nEvidence: %s/cutover.json\n' \ + "$CANDIDATE_SHA256" "$EVIDENCE_DIR" diff --git a/deployment/aliyun/host-rust-lob-shadow-gate.sh b/deployment/aliyun/host-rust-lob-shadow-gate.sh new file mode 100755 index 000000000..9fceb6178 --- /dev/null +++ b/deployment/aliyun/host-rust-lob-shadow-gate.sh @@ -0,0 +1,723 @@ +#!/usr/bin/env bash +set -euo pipefail + +umask 027 +export LC_ALL=C + +readonly REQUIRED_DURATION_SECONDS=3600 +readonly HEALTH_SETTLE_SECONDS=360 +readonly MAX_HEALTH_SILENCE_SECONDS=90 +readonly SHADOW_BINARY=/opt/monday/bin/binance-lob-archiver-shadow +readonly RELEASE_ROOT=/opt/monday/releases/binance-lob-archiver +readonly EVIDENCE_ROOT=/data/monday/evidence/shadow-gates +readonly LOCK_FILE=/run/lock/monday-rust-lob-release.lock +readonly SERVICE_USER=hftcollector +readonly SERVICE_HOME=/var/lib/hft-collector +readonly SAFE_PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +die() { + printf 'shadow gate failed: %s\n' "$*" >&2 + exit 1 +} + +usage() { + printf '%s\n' \ + 'Usage: host-rust-lob-shadow-gate.sh ' \ + '' \ + 'Production gates always observe at least 3600 seconds.' \ + 'Tests may set MONDAY_GATE_TEST_SECONDS only with' \ + 'MONDAY_ALLOW_SHORT_GATE_FOR_TESTS=1; test evidence cannot pass cutover.' +} + +[[ ${EUID} -eq 0 ]] || die 'must run as root' +[[ $# -eq 1 ]] || { + usage >&2 + exit 2 +} + +for command in aliyun awk chmod chown cmp date dirname find flock grep id install jq mktemp \ + mountpoint mv readlink rm runuser sed sha256sum sleep sort stat systemctl tr wc; do + command -v "$command" >/dev/null 2>&1 || die "missing required command: $command" +done + +mountpoint -q /data || die '/data must be a mount point' +[[ -r /proc/uptime ]] || die '/proc/uptime is required for monotonic timing' +id "$SERVICE_USER" >/dev/null 2>&1 || die "missing service user: $SERVICE_USER" +install -d -m 0755 "$(dirname "$LOCK_FILE")" +exec 9>"$LOCK_FILE" +flock -n 9 || die 'another Rust collector release operation is running' + +candidate_sha=$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]') +[[ $candidate_sha =~ ^[a-f0-9]{64}$ ]] || die 'candidate SHA-256 must be 64 hexadecimal characters' +candidate_release="$RELEASE_ROOT/$candidate_sha" +candidate_binary="$candidate_release/binance-lob-archiver" +candidate_deployment="$candidate_release/deployment" +release_json="$candidate_release/release.json" +control_plane_lib="$candidate_deployment/rust-lob-control-plane-lib.sh" + +direct_directory() { + local path=$1 + [[ -d $path && ! -L $path ]] || return 1 + [[ $(readlink -f -- "$path") == "$path" ]] +} + +direct_directory_or_absent() { + local path=$1 + [[ ! -e $path && ! -L $path ]] || direct_directory "$path" +} + +secure_regular_file() { + local path=$1 mode owner + [[ -f $path && ! -L $path ]] || die "required regular file is missing or a symlink: $path" + owner=$(stat -c %u -- "$path") + mode=$(stat -c %a -- "$path") + [[ $owner == 0 ]] || die "required file is not root-owned: $path" + (( (8#$mode & 022) == 0 )) || die "required file is group/world writable: $path" +} + +for path in /opt/monday /opt/monday/releases "$RELEASE_ROOT" "$candidate_release" \ + "$candidate_deployment"; do + direct_directory "$path" || die "release path is missing, indirect, or a symlink: $path" +done +secure_regular_file "$release_json" +secure_regular_file "$control_plane_lib" +# shellcheck disable=SC1090,SC1091 +. "$control_plane_lib" +deployment_bundle_sha256=$(jq -er '.deployment_bundle_sha256' "$release_json") +deployment_source_revision=$(jq -er '.deployment_source_revision' "$release_json") +[[ $deployment_bundle_sha256 =~ ^[a-f0-9]{64}$ ]] \ + || die 'release metadata has an invalid deployment bundle SHA-256' +[[ $deployment_source_revision =~ ^[a-f0-9]{40,64}$ ]] \ + || die 'release metadata has an invalid deployment source revision' +jq -e --arg artifact "$candidate_sha" --arg bundle "$deployment_bundle_sha256" \ + '.artifact_sha256 == $artifact and .deployment_bundle_sha256 == $bundle' \ + "$release_json" >/dev/null || die 'release metadata does not match the candidate identity' + +[[ -f $candidate_binary && -x $candidate_binary ]] || die "candidate is not executable: $candidate_binary" +secure_regular_file "$candidate_binary" +printf '%s %s\n' "$candidate_sha" "$candidate_binary" | sha256sum --check --strict >/dev/null +[[ -L $SHADOW_BINARY ]] || die "$SHADOW_BINARY must be a symlink" +[[ $(readlink -f "$SHADOW_BINARY") == "$candidate_binary" ]] \ + || die 'shadow symlink does not point to the requested candidate' +printf '%s %s\n' "$candidate_sha" "$SHADOW_BINARY" | sha256sum --check --strict >/dev/null + +gate_seconds=${MONDAY_GATE_TEST_SECONDS:-$REQUIRED_DURATION_SECONDS} +[[ $gate_seconds =~ ^[1-9][0-9]*$ ]] || die 'gate duration must be a positive integer' +test_only=false +if ((gate_seconds < REQUIRED_DURATION_SECONDS)); then + [[ ${MONDAY_ALLOW_SHORT_GATE_FOR_TESTS:-0} == 1 ]] \ + || die 'short gates require MONDAY_ALLOW_SHORT_GATE_FOR_TESTS=1' + test_only=true +fi + +env_value() { + local file=$1 + local key=$2 + local count value + count=$(grep -c "^${key}=" "$file" || true) + [[ $count -eq 1 ]] || die "$file must contain exactly one ${key}= entry" + value=$(sed -n "s/^${key}=//p" "$file") + [[ -n $value ]] || die "$file has an empty $key" + printf '%s\n' "$value" +} + +readonly -a markets=(spot usdm) +declare -A env_file spool_dir dataset shard_id oss_bucket oss_endpoint oss_region +declare -A aliyun_profile oss_copy_timeout min_symbols unit +for market in "${markets[@]}"; do + env_file[$market]="/etc/monday/binance-lob-archiver-rust-${market}.env" + [[ -f ${env_file[$market]} ]] || die "missing ${env_file[$market]}" + [[ $(env_value "${env_file[$market]}" MARKET) == "$market" ]] \ + || die "${env_file[$market]} has the wrong MARKET" + [[ $(env_value "${env_file[$market]}" SYMBOLS) == ALL ]] \ + || die "${env_file[$market]} must set SYMBOLS=ALL" + shard_id[$market]=$(env_value "${env_file[$market]}" SHARD_ID) + [[ ${shard_id[$market]} == all ]] || die "${env_file[$market]} must set SHARD_ID=all" + spool_dir[$market]=$(env_value "${env_file[$market]}" SPOOL_DIR) + dataset[$market]=$(env_value "${env_file[$market]}" DATASET) + oss_bucket[$market]=$(env_value "${env_file[$market]}" OSS_BUCKET) + oss_endpoint[$market]=$(env_value "${env_file[$market]}" OSS_ENDPOINT) + oss_region[$market]=$(env_value "${env_file[$market]}" OSS_REGION) + aliyun_profile[$market]=$(env_value "${env_file[$market]}" ALIYUN_PROFILE) + oss_copy_timeout[$market]=$(env_value "${env_file[$market]}" OSS_COPY_TIMEOUT_SECONDS) + [[ ${oss_copy_timeout[$market]} =~ ^[1-9][0-9]*$ ]] \ + || die "${env_file[$market]} has an invalid OSS_COPY_TIMEOUT_SECONDS" + [[ ${oss_bucket[$market]} =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]] \ + || die "${env_file[$market]} has an invalid OSS_BUCKET" + [[ ${oss_region[$market]} == ap-northeast-1 ]] \ + || die "${env_file[$market]} must use the Tokyo OSS region" + [[ ${oss_endpoint[$market]} == oss-ap-northeast-1-internal.aliyuncs.com ]] \ + || die "${env_file[$market]} must use the Tokyo internal OSS endpoint" + [[ ${aliyun_profile[$market]} == ecs-role ]] \ + || die "${env_file[$market]} must use the ECS RAM-role profile" + unit[$market]="binance-lob-archiver-rust@${market}.service" +done + +for asset in \ + binance-lob-archiver-rust@.service \ + binance-lob-archiver-rust-upload@.service \ + binance-lob-archiver-rust-spot.env \ + binance-lob-archiver-rust-usdm.env; do + secure_regular_file "$candidate_deployment/$asset" + case "$asset" in + *.service) installed_asset="/etc/systemd/system/$asset" ;; + *.env) installed_asset="/etc/monday/$asset" ;; + esac + secure_regular_file "$installed_asset" + cmp -s "$candidate_deployment/$asset" "$installed_asset" \ + || die "installed shadow asset differs from the gated deployment bundle: $asset" +done + +[[ ${spool_dir[spot]} == /data/monday/spool/binance-lob-rust-shadow/spot ]] \ + || die 'Spot shadow spool path is not isolated' +[[ ${spool_dir[usdm]} == /data/monday/spool/binance-lob-rust-shadow/usdm ]] \ + || die 'USD-M shadow spool path is not isolated' +for path in \ + /data/monday/spool/binance-lob-rust-shadow \ + "${spool_dir[spot]}" \ + "${spool_dir[usdm]}"; do + [[ -d $path && ! -L $path ]] || die "shadow spool is missing or a symlink: $path" + [[ $(readlink -f "$path") == "$path" ]] \ + || die "shadow spool resolved outside its canonical path: $path" +done +[[ ${dataset[spot]} == spot_all_rust_shadow ]] || die 'Spot shadow dataset is not isolated' +[[ ${dataset[usdm]} == usdm_perpetual_all_rust_shadow ]] \ + || die 'USD-M shadow dataset is not isolated' +min_symbols[spot]=1000 +min_symbols[usdm]=400 + +binary_evidence_dir="$EVIDENCE_ROOT/$candidate_sha" +evidence_dir="$binary_evidence_dir/$deployment_bundle_sha256" +gate_json="$evidence_dir/gate.json" +passed_marker="$evidence_dir/PASSED.sha256" +gate_tmp="$evidence_dir/.gate.json.tmp" +marker_tmp="$evidence_dir/.PASSED.sha256.tmp" +for path in /data/monday /data/monday/evidence "$EVIDENCE_ROOT"; do + direct_directory_or_absent "$path" || die "evidence path is indirect or a symlink: $path" +done +install -d -m 0755 /data/monday +install -d -m 0750 /data/monday/evidence "$EVIDENCE_ROOT" +for path in /data/monday /data/monday/evidence "$EVIDENCE_ROOT"; do + direct_directory "$path" || die "evidence path is indirect or a symlink: $path" +done +install -d -m 0750 "$binary_evidence_dir" +direct_directory "$binary_evidence_dir" \ + || die 'binary evidence directory is indirect or a symlink' +install -d -m 0750 "$evidence_dir" +direct_directory "$evidence_dir" || die 'bundle evidence directory is indirect or a symlink' +rm -f "$gate_json" "$passed_marker" "$gate_tmp" "$marker_tmp" + +tmp_dir=$(mktemp -d) +chown "$SERVICE_USER:$SERVICE_USER" "$tmp_dir" +chmod 0750 "$tmp_dir" +gate_finished=false +cleanup() { + local status=$? + rm -rf "$tmp_dir" + rm -f "$gate_tmp" "$marker_tmp" + if [[ $gate_finished != true ]]; then + rm -f "$passed_marker" + systemctl stop "${unit[spot]}" "${unit[usdm]}" >/dev/null 2>&1 || true + fi + exit "$status" +} +trap cleanup EXIT + +assert_candidate() { + [[ -L $SHADOW_BINARY ]] || die 'shadow candidate symlink disappeared' + [[ $(readlink -f "$SHADOW_BINARY") == "$candidate_binary" ]] \ + || die 'shadow candidate symlink changed during the gate' + printf '%s %s\n' "$candidate_sha" "$SHADOW_BINARY" \ + | sha256sum --check --strict >/dev/null +} + +assert_spool_drained() { + local market=$1 + local remaining + remaining=$(find "${spool_dir[$market]}" -type f \( \ + -name '*.manifest.json' -o -name '*.jsonl.part' -o \ + -name '*.zst.tmp' -o -name '*.part.corrupt' -o \ + -name '*.jsonl.zst' -o -name '*._SUCCESS' -o \ + -name '*.uploaded-cleanup.json' -o -name '*.uploaded-cleanup.json.tmp' \ + \) -print -quit) + [[ -z $remaining ]] || die "$market shadow spool still contains segment artifact: $remaining" +} + +run_candidate_drain() { + local market=$1 + runuser --user "$SERVICE_USER" -- env -i \ + HOME="$SERVICE_HOME" \ + PATH="$SAFE_PATH" \ + RUST_LOG=info \ + SPOOL_DIR="${spool_dir[$market]}" \ + OSS_BUCKET="${oss_bucket[$market]}" \ + OSS_ENDPOINT="${oss_endpoint[$market]}" \ + OSS_REGION="${oss_region[$market]}" \ + ALIYUN_PROFILE="${aliyun_profile[$market]}" \ + OSS_COPY_TIMEOUT_SECONDS="${oss_copy_timeout[$market]}" \ + "$candidate_binary" --upload-only + assert_spool_drained "$market" +} + +systemctl stop "${unit[spot]}" "${unit[usdm]}" +for market in "${markets[@]}"; do + run_candidate_drain "$market" + rm -f "${spool_dir[$market]}/health.json" +done +systemctl reset-failed \ + "${unit[spot]}" "${unit[usdm]}" \ + binance-lob-archiver-rust-upload@spot.service \ + binance-lob-archiver-rust-upload@usdm.service +assert_candidate + +gate_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +gate_started_epoch=$(date +%s) +gate_started_ns="${gate_started_epoch}000000000" +systemctl start "${unit[spot]}" "${unit[usdm]}" + +systemctl_value() { + local market=$1 + local property=$2 + systemctl show "${unit[$market]}" --property="$property" --value +} + +require_uint() { + local value=$1 + local label=$2 + [[ $value =~ ^[0-9]+$ ]] || die "$label is unavailable: $value" + printf '%s\n' "$value" +} + +timespan_to_us() { + local value=$1 + case "$value" in + *us) + value=${value%us} + [[ $value =~ ^[0-9]+$ ]] || return 1 + printf '%s\n' "$value" + ;; + *ms) + value=${value%ms} + [[ $value =~ ^[0-9]+$ ]] || return 1 + printf '%s\n' "$((value * 1000))" + ;; + *min) + value=${value%min} + [[ $value =~ ^[0-9]+$ ]] || return 1 + printf '%s\n' "$((value * 60 * 1000000))" + ;; + *s) + value=${value%s} + [[ $value =~ ^[0-9]+$ ]] || return 1 + printf '%s\n' "$((value * 1000000))" + ;; + [0-9]*) + [[ $value =~ ^[0-9]+$ ]] || return 1 + printf '%s\n' "$value" + ;; + *) return 1 ;; + esac +} + +monotonic_seconds() { + awk '{print int($1)}' /proc/uptime +} + +declare -A active_enter_us cpu_start_ns cpu_quota_us memory_max_bytes max_memory_bytes +for market in "${markets[@]}"; do + systemctl is-active --quiet "${unit[$market]}" || die "$market shadow service is not active" + [[ $(systemctl_value "$market" ActiveState) == active ]] \ + || die "$market shadow service did not enter ActiveState=active" + [[ $(systemctl_value "$market" SubState) == running ]] \ + || die "$market shadow service did not enter SubState=running" + [[ $(systemctl_value "$market" NRestarts) == 0 ]] \ + || die "$market shadow service restarted during startup" + active_enter_us[$market]=$(require_uint \ + "$(systemctl_value "$market" ActiveEnterTimestampMonotonic)" \ + "$market ActiveEnterTimestampMonotonic") + cpu_start_ns[$market]=$(require_uint "$(systemctl_value "$market" CPUUsageNSec)" \ + "$market CPUUsageNSec") + quota_raw=$(systemctl_value "$market" CPUQuotaPerSecUSec) + cpu_quota_us[$market]=$(timespan_to_us "$quota_raw") \ + || die "$market CPU quota is unavailable: $quota_raw" + ((cpu_quota_us[$market] > 0)) || die "$market CPU quota must be finite" + memory_max_bytes[$market]=$(require_uint "$(systemctl_value "$market" MemoryMax)" \ + "$market MemoryMax") + ((memory_max_bytes[$market] > 0)) || die "$market MemoryMax must be finite" + max_memory_bytes[$market]=$(require_uint "$(systemctl_value "$market" MemoryCurrent)" \ + "$market MemoryCurrent") +done + +health_passes() { + local market=$1 + local health="${spool_dir[$market]}/health.json" + [[ -f $health && ! -L $health ]] || return 1 + jq -e \ + --arg market "$market" \ + --arg dataset "${dataset[$market]}" \ + --argjson minimum_symbols "${min_symbols[$market]}" \ + --argjson gate_started_ns "$gate_started_ns" \ + '.market == $market + and .dataset == $dataset + and .updated_at_ns >= $gate_started_ns + and .status == "synced" + and .sequence_gaps == 0 + and .symbol_count >= $minimum_symbols + and .snapshot_ready_count == .symbol_count + and .pending_upload_segments == 0 + and .queue_saturated == false + and .disk_warning == false + and .upload_warning == false + and (.upload_failure_count | type) == "number" + and .upload_failure_count >= 0 + and .upload_failure_count == (.upload_failure_count | floor) + and (.session_id | type) == "string" + and (.session_id | length) > 0' \ + "$health" >/dev/null +} + +health_catalog_sha256() { + local market=$1 + jq -c '.symbols | keys | sort' "${spool_dir[$market]}/health.json" \ + | sha256sum | awk '{print $1}' +} + +settle_deadline=$(( $(monotonic_seconds) + HEALTH_SETTLE_SECONDS )) +while ! health_passes spot || ! health_passes usdm; do + (( $(monotonic_seconds) < settle_deadline )) \ + || die 'shadow health did not reach the fail-closed gate before the settle deadline' + for market in "${markets[@]}"; do + systemctl is-active --quiet "${unit[$market]}" || die "$market shadow service stopped while settling" + [[ $(systemctl_value "$market" NRestarts) == 0 ]] \ + || die "$market shadow service restarted while settling" + done + sleep 10 +done + +declare -A observed_session frozen_symbol_count frozen_catalog_sha256 +declare -A initial_upload_failure_count last_health_updated_ns health_samples +declare -A last_health_advance_mono max_health_silence_seconds +for market in "${markets[@]}"; do + health="${spool_dir[$market]}/health.json" + observed_session[$market]=$(jq -er '.session_id' "$health") + frozen_symbol_count[$market]=$(jq -er '.symbol_count' "$health") + frozen_catalog_sha256[$market]=$(health_catalog_sha256 "$market") + initial_upload_failure_count[$market]=$(jq -er '.upload_failure_count' "$health") + last_health_updated_ns[$market]=$(jq -er '.updated_at_ns' "$health") + last_health_advance_mono[$market]=$(monotonic_seconds) + max_health_silence_seconds[$market]=0 + health_samples[$market]=1 +done + +validate_observation_sample() { + local market=$1 health session symbols catalog upload_failures updated_ns + local current_mono next_updated_ns next_advance_mono next_max_gap sample_increment + health="${spool_dir[$market]}/health.json" + health_passes "$market" || die "$market health failed during observation" + session=$(jq -er '.session_id' "$health") + [[ $session == "${observed_session[$market]}" ]] \ + || die "$market collector session changed during observation" + symbols=$(jq -er '.symbol_count' "$health") + [[ $symbols == "${frozen_symbol_count[$market]}" ]] \ + || die "$market full catalog changed during observation" + catalog=$(health_catalog_sha256 "$market") + [[ $catalog == "${frozen_catalog_sha256[$market]}" ]] \ + || die "$market catalog membership changed during observation" + upload_failures=$(jq -er '.upload_failure_count' "$health") + [[ $upload_failures == "${initial_upload_failure_count[$market]}" ]] \ + || die "$market recorded an OSS upload failure during observation" + updated_ns=$(jq -er '.updated_at_ns' "$health") + current_mono=$(monotonic_seconds) + if ! read -r next_updated_ns next_advance_mono next_max_gap sample_increment < <( + monday_observe_health_freshness \ + "${last_health_updated_ns[$market]}" \ + "${last_health_advance_mono[$market]}" \ + "${max_health_silence_seconds[$market]}" \ + "$updated_ns" "$current_mono" "$MAX_HEALTH_SILENCE_SECONDS" + ); then + die "$market health timestamp regressed or stopped advancing for more than ${MAX_HEALTH_SILENCE_SECONDS}s" + fi + last_health_updated_ns[$market]=$next_updated_ns + last_health_advance_mono[$market]=$next_advance_mono + max_health_silence_seconds[$market]=$next_max_gap + health_samples[$market]=$((health_samples[$market] + sample_increment)) +} + +observation_started_mono=$(monotonic_seconds) +observation_deadline=$((observation_started_mono + gate_seconds)) +while (( $(monotonic_seconds) < observation_deadline )); do + now_mono=$(monotonic_seconds) + remaining=$((observation_deadline - now_mono)) + interval=30 + ((remaining < interval)) && interval=$remaining + ((interval > 0)) && sleep "$interval" + assert_candidate + for market in "${markets[@]}"; do + systemctl is-active --quiet "${unit[$market]}" || die "$market shadow service stopped during observation" + [[ $(systemctl_value "$market" NRestarts) == 0 ]] \ + || die "$market shadow service restarted during observation" + memory_now=$(require_uint "$(systemctl_value "$market" MemoryCurrent)" \ + "$market MemoryCurrent") + ((memory_now > max_memory_bytes[$market])) && max_memory_bytes[$market]=$memory_now + ((memory_now <= memory_max_bytes[$market])) \ + || die "$market memory usage exceeded MemoryMax" + validate_observation_sample "$market" + done +done + +if [[ $test_only != true ]]; then + minimum_health_samples=$((REQUIRED_DURATION_SECONDS / 90)) + for market in "${markets[@]}"; do + ((health_samples[$market] >= minimum_health_samples)) \ + || die "$market health did not advance often enough during observation" + done +fi + +declare -A observed_runtime_seconds cpu_usage_ns memory_peak_bytes health_sha256 +declare -A symbol_count snapshot_ready_count sequence_gaps +now_monotonic_us=$(awk '{printf "%.0f\n", $1 * 1000000}' /proc/uptime) +for market in "${markets[@]}"; do + assert_candidate + systemctl is-active --quiet "${unit[$market]}" || die "$market shadow service is not active at gate close" + [[ $(systemctl_value "$market" NRestarts) == 0 ]] \ + || die "$market shadow service has a non-zero restart count" + observed_runtime_seconds[$market]=$(( \ + (now_monotonic_us - active_enter_us[$market]) / 1000000 \ + )) + ((observed_runtime_seconds[$market] >= gate_seconds)) \ + || die "$market actual runtime was shorter than the requested gate" + + cpu_end_ns=$(require_uint "$(systemctl_value "$market" CPUUsageNSec)" "$market CPUUsageNSec") + ((cpu_end_ns >= cpu_start_ns[$market])) || die "$market CPU accounting moved backwards" + cpu_usage_ns[$market]=$((cpu_end_ns - cpu_start_ns[$market])) + allowed_cpu_ns=$(( \ + observed_runtime_seconds[$market] * cpu_quota_us[$market] * 1000 \ + + cpu_quota_us[$market] * 1000 \ + )) + ((cpu_usage_ns[$market] <= allowed_cpu_ns)) || die "$market average CPU exceeded CPUQuota" + + memory_now=$(require_uint "$(systemctl_value "$market" MemoryCurrent)" "$market MemoryCurrent") + ((memory_now > max_memory_bytes[$market])) && max_memory_bytes[$market]=$memory_now + peak_raw=$(systemctl_value "$market" MemoryPeak) + if [[ $peak_raw =~ ^[0-9]+$ ]] && ((peak_raw > max_memory_bytes[$market])); then + max_memory_bytes[$market]=$peak_raw + fi + memory_peak_bytes[$market]=${max_memory_bytes[$market]} + ((memory_peak_bytes[$market] <= memory_max_bytes[$market])) \ + || die "$market memory peak exceeded MemoryMax" + + validate_observation_sample "$market" + health="${spool_dir[$market]}/health.json" + health_copy="$evidence_dir/${market}-health.json" + install -m 0640 "$health" "$health_copy" + health_sha256[$market]=$(sha256sum "$health_copy" | awk '{print $1}') + symbol_count[$market]=$(jq -er '.symbol_count' "$health") + snapshot_ready_count[$market]=$(jq -er '.snapshot_ready_count' "$health") + sequence_gaps[$market]=$(jq -er '.sequence_gaps' "$health") +done + +systemctl stop "${unit[spot]}" "${unit[usdm]}" +for market in "${markets[@]}"; do + systemctl is-active --quiet "${unit[$market]}" \ + && die "$market shadow service remained active after stop" + assert_candidate + run_candidate_drain "$market" +done + +run_oss() { + local market=$1 + shift + runuser --user "$SERVICE_USER" -- env -i \ + HOME="$SERVICE_HOME" \ + PATH="$SAFE_PATH" \ + aliyun ossutil "$@" \ + --profile "${aliyun_profile[$market]}" \ + --endpoint "${oss_endpoint[$market]}" \ + --region "${oss_region[$market]}" +} + +manifest_uris() { + local market=$1 + local listing=$2 + local prefix line token max_age_seconds + prefix="oss://${oss_bucket[$market]}/lake/raw/venue=binance/market=${market}/dataset=${dataset[$market]}/shard=${shard_id[$market]}/" + max_age_seconds=$((gate_seconds + HEALTH_SETTLE_SECONDS + 3600)) + run_oss "$market" ls "$prefix" --recursive --short-format \ + --max-age "${max_age_seconds}s" >"$listing" + while IFS= read -r line; do + line=${line%$'\r'} + if [[ $line =~ (oss://[^[:space:]]+\.manifest\.json) ]]; then + printf '%s\n' "${BASH_REMATCH[1]}" + continue + fi + token=${line##*[$' \t']} + token=${token#/} + if [[ $token == *.manifest.json && $token == lake/* ]]; then + printf 'oss://%s/%s\n' "${oss_bucket[$market]}" "$token" + fi + done <"$listing" | sort -u +} + +verify_oss_round_trips() { + local market=$1 + local listing="$tmp_dir/${market}-oss-list.txt" + local uris="$tmp_dir/${market}-manifest-uris.txt" + local candidates="$tmp_dir/${market}-manifest-candidates.tsv" + local index=0 + local uri manifest start_ns file digest zst_uri zst_path actual_digest bytes + local round_trips='[]' + + manifest_uris "$market" "$listing" >"$uris" + : >"$candidates" + while IFS= read -r uri; do + [[ -n $uri ]] || continue + manifest="$tmp_dir/${market}-scan-$index.json" + index=$((index + 1)) + run_oss "$market" cp "$uri" "$manifest" --force --no-progress >/dev/null + if ! jq -e \ + --arg market "$market" \ + --arg dataset "${dataset[$market]}" \ + --arg shard "${shard_id[$market]}" \ + --argjson gate_started_ns "$gate_started_ns" \ + '.market == $market + and .dataset == $dataset + and .shard_id == $shard + and .start_received_at_ns >= $gate_started_ns + and (.file | type == "string" and test("^[A-Za-z0-9._-]+\\.jsonl\\.zst$")) + and (.sha256 | type == "string" and test("^[a-f0-9]{64}$"))' \ + "$manifest" >/dev/null; then + continue + fi + start_ns=$(jq -er '.start_received_at_ns' "$manifest") + file=$(jq -er '.file' "$manifest") + digest=$(jq -er '.sha256' "$manifest") + printf '%s\t%s\t%s\t%s\t%s\n' "$start_ns" "$uri" "$file" "$digest" "$manifest" \ + >>"$candidates" + done <"$uris" + + candidate_count=$(wc -l <"$candidates" | tr -d ' ') + ((candidate_count >= 2)) \ + || die "$market has fewer than two OSS manifests created after gate start" + + index=0 + while IFS=$'\t' read -r start_ns uri file digest manifest; do + index=$((index + 1)) + ((index <= 2)) || break + zst_uri="${uri%/*}/$file" + zst_path="$tmp_dir/${market}-${index}.jsonl.zst" + run_oss "$market" cp "$zst_uri" "$zst_path" --force --no-progress >/dev/null + actual_digest=$(sha256sum "$zst_path" | awk '{print $1}') + [[ $actual_digest == "$digest" ]] || die "$market OSS round-trip digest mismatch: $zst_uri" + bytes=$(stat -c '%s' "$zst_path") + install -m 0640 "$manifest" "$evidence_dir/${market}-manifest-${index}.json" + round_trip=$(jq -cn \ + --arg manifest_uri "$uri" \ + --arg data_uri "$zst_uri" \ + --arg sha256 "$digest" \ + --argjson start_received_at_ns "$start_ns" \ + --argjson bytes "$bytes" \ + '{manifest_uri:$manifest_uri,data_uri:$data_uri,sha256:$sha256, + start_received_at_ns:$start_received_at_ns,bytes:$bytes}') + round_trips=$(jq -cn --argjson values "$round_trips" --argjson value "$round_trip" \ + '$values + [$value]') + done < <(sort -n -k1,1 "$candidates") + + printf '%s\n' "$round_trips" +} + +duration_seconds=${observed_runtime_seconds[spot]} +if ((observed_runtime_seconds[usdm] < duration_seconds)); then + duration_seconds=${observed_runtime_seconds[usdm]} +fi + +markets_json='{}' +for market in "${markets[@]}"; do + round_trips=$(verify_oss_round_trips "$market") + market_json=$(jq -cn \ + --arg market "$market" \ + --arg unit "${unit[$market]}" \ + --arg dataset "${dataset[$market]}" \ + --arg session_id "${observed_session[$market]}" \ + --arg catalog_sha256 "${frozen_catalog_sha256[$market]}" \ + --arg health_sha256 "${health_sha256[$market]}" \ + --argjson symbol_count "${symbol_count[$market]}" \ + --argjson snapshot_ready_count "${snapshot_ready_count[$market]}" \ + --argjson sequence_gaps "${sequence_gaps[$market]}" \ + --argjson upload_failure_count "${initial_upload_failure_count[$market]}" \ + --argjson health_samples "${health_samples[$market]}" \ + --argjson max_health_silence_seconds "${max_health_silence_seconds[$market]}" \ + --argjson n_restarts 0 \ + --argjson observed_runtime_seconds "${observed_runtime_seconds[$market]}" \ + --argjson cpu_usage_ns "${cpu_usage_ns[$market]}" \ + --argjson cpu_quota_per_sec_us "${cpu_quota_us[$market]}" \ + --argjson memory_peak_bytes "${memory_peak_bytes[$market]}" \ + --argjson memory_max_bytes "${memory_max_bytes[$market]}" \ + --argjson oss_round_trips "$round_trips" \ + '{market:$market,unit:$unit,dataset:$dataset,session_id:$session_id, + symbols_config:"ALL",catalog_sha256:$catalog_sha256, + symbol_count:$symbol_count,snapshot_ready_count:$snapshot_ready_count, + sequence_gaps:$sequence_gaps,upload_failure_count:$upload_failure_count, + health_samples:$health_samples, + max_health_silence_seconds:$max_health_silence_seconds, + n_restarts:$n_restarts, + observed_runtime_seconds:$observed_runtime_seconds, + cpu_usage_ns:$cpu_usage_ns,cpu_quota_per_sec_us:$cpu_quota_per_sec_us, + memory_peak_bytes:$memory_peak_bytes,memory_max_bytes:$memory_max_bytes, + health_sha256:$health_sha256, + oss_roundtrips:($oss_round_trips | length), + oss_roundtrip_evidence:$oss_round_trips}') + markets_json=$(jq -cn \ + --argjson values "$markets_json" \ + --arg market "$market" \ + --argjson value "$market_json" \ + '$values + {($market):$value}') +done + +gate_finished_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +passed=true +production_eligible=true +if [[ $test_only == true ]] || ((duration_seconds < REQUIRED_DURATION_SECONDS)); then + passed=false + production_eligible=false +fi + +jq -n \ + --arg schema monday.rust_lob_shadow_gate.v2 \ + --arg candidate_sha256 "$candidate_sha" \ + --arg candidate_binary "$candidate_binary" \ + --arg deployment_bundle_sha256 "$deployment_bundle_sha256" \ + --arg deployment_source_revision "$deployment_source_revision" \ + --arg started_at "$gate_started_at" \ + --arg finished_at "$gate_finished_at" \ + --argjson required_duration_seconds "$REQUIRED_DURATION_SECONDS" \ + --argjson requested_duration_seconds "$gate_seconds" \ + --argjson duration_seconds "$duration_seconds" \ + --argjson test_only "$test_only" \ + --argjson checks_passed true \ + --argjson production_eligible "$production_eligible" \ + --argjson passed "$passed" \ + --argjson markets "$markets_json" \ + '{schema:$schema,candidate_sha256:$candidate_sha256,candidate_binary:$candidate_binary, + deployment_bundle_sha256:$deployment_bundle_sha256, + deployment_source_revision:$deployment_source_revision, + started_at:$started_at,finished_at:$finished_at, + required_duration_seconds:$required_duration_seconds, + requested_duration_seconds:$requested_duration_seconds, + duration_seconds:$duration_seconds, + test_only:$test_only,checks_passed:$checks_passed, + production_eligible:$production_eligible,passed:$passed,markets:$markets}' \ + >"$gate_tmp" +install -m 0640 "$gate_tmp" "$gate_json" +rm -f "$gate_tmp" + +if [[ $production_eligible == true ]]; then + gate_sha=$(sha256sum "$gate_json" | awk '{print $1}') + printf '%s gate.json\n' "$gate_sha" >"$marker_tmp" + chmod 0640 "$marker_tmp" + mv "$marker_tmp" "$passed_marker" + printf 'production shadow gate passed: %s\nmarker: %s\n' "$gate_json" "$passed_marker" +else + printf 'short test completed; evidence is not eligible for cutover: %s\n' "$gate_json" +fi + +gate_finished=true diff --git a/deployment/aliyun/invoke-rust-lob-operation.sh b/deployment/aliyun/invoke-rust-lob-operation.sh new file mode 100755 index 000000000..ff208eef9 --- /dev/null +++ b/deployment/aliyun/invoke-rust-lob-operation.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + printf '%s\n' \ + 'Usage: ACTION=gate|cutover INSTANCE_ID=i-... ARTIFACT_SHA256=<64 hex> invoke-rust-lob-operation.sh' \ + '' \ + 'The command always targets ap-northeast-1 and uses Alibaba Cloud Assistant.' +} + +for command in aliyun base64 jq; do + if ! command -v "$command" >/dev/null 2>&1; then + printf 'missing required command: %s\n' "$command" >&2 + exit 2 + fi +done + +: "${ACTION:?set ACTION to gate or cutover}" +: "${INSTANCE_ID:?set INSTANCE_ID}" +: "${ARTIFACT_SHA256:?set ARTIFACT_SHA256}" + +REGION_ID=${REGION_ID:-ap-northeast-1} +ALIYUN_LOCAL_PROFILE=${ALIYUN_LOCAL_PROFILE:-default} +if [[ "$REGION_ID" != 'ap-northeast-1' ]]; then + printf 'refusing non-Tokyo region: %s\n' "$REGION_ID" >&2 + exit 2 +fi +if [[ ! "$INSTANCE_ID" =~ ^i-[a-z0-9]+$ ]]; then + usage >&2 + exit 2 +fi +if [[ ! "$ARTIFACT_SHA256" =~ ^[A-Fa-f0-9]{64}$ ]]; then + printf 'ARTIFACT_SHA256 must contain exactly 64 hexadecimal characters\n' >&2 + exit 2 +fi +ARTIFACT_SHA256=$(printf '%s' "$ARTIFACT_SHA256" | tr '[:upper:]' '[:lower:]') + +case "$ACTION" in + gate) + host_script=host-rust-lob-shadow-gate.sh + timeout_seconds=7200 + command_name=monday-rust-lob-shadow-gate + ;; + cutover) + host_script=host-rust-lob-cutover.sh + timeout_seconds=3600 + command_name=monday-rust-lob-cutover + ;; + *) + usage >&2 + exit 2 + ;; +esac + +polls=$((timeout_seconds / 5)) +cancel_polls=12 +if [[ -n ${MONDAY_OPERATION_TEST_POLLS:-} || -n ${MONDAY_OPERATION_TEST_CANCEL_POLLS:-} ]]; then + [[ ${MONDAY_ALLOW_SHORT_OPERATION_TEST:-0} == 1 ]] || { + printf 'short operation polling requires MONDAY_ALLOW_SHORT_OPERATION_TEST=1\n' >&2 + exit 2 + } + polls=${MONDAY_OPERATION_TEST_POLLS:-$polls} + cancel_polls=${MONDAY_OPERATION_TEST_CANCEL_POLLS:-$cancel_polls} + [[ $polls =~ ^[1-9][0-9]*$ && $cancel_polls =~ ^[1-9][0-9]*$ ]] || { + printf 'test poll counts must be positive integers\n' >&2 + exit 2 + } +fi + +aliyun_profile_args=() +if [[ -n "$ALIYUN_LOCAL_PROFILE" ]]; then + aliyun_profile_args=(--profile "$ALIYUN_LOCAL_PROFILE") +fi + +host_path="/opt/monday/releases/binance-lob-archiver/$ARTIFACT_SHA256/deployment/$host_script" +printf -v remote_script '#!/usr/bin/env bash\nset -euo pipefail\nexec %q %q\n' \ + "$host_path" "$ARTIFACT_SHA256" +command_content=$(printf '%s' "$remote_script" | base64 | tr -d '\n') + +run_json=$(aliyun ecs RunCommand \ + --RegionId "$REGION_ID" \ + --InstanceId.1 "$INSTANCE_ID" \ + --Type RunShellScript \ + --ContentEncoding Base64 \ + --CommandContent "$command_content" \ + --KeepCommand false \ + --Name "$command_name" \ + --Timeout "$timeout_seconds" \ + "${aliyun_profile_args[@]}") +invoke_id=$(printf '%s' "$run_json" | jq -er '.InvokeId') +printf 'Cloud Assistant invocation: %s (%s)\n' "$invoke_id" "$ACTION" + +result_json='' +for _ in $(seq 1 "$polls"); do + if ! result_json=$(aliyun ecs DescribeInvocationResults \ + --RegionId "$REGION_ID" \ + --InvokeId "$invoke_id" \ + --InstanceId "$INSTANCE_ID" \ + "${aliyun_profile_args[@]}"); then + sleep 5 + continue + fi + status=$(printf '%s' "$result_json" \ + | jq -r '[.. | objects | .InvocationStatus? // empty][0] // empty') + exit_code=$(printf '%s' "$result_json" \ + | jq -r '[.. | objects | .ExitCode? // empty][0] // empty') + case "$status" in + Success|Finished) + output=$(printf '%s' "$result_json" \ + | jq -r '[.. | objects | .Output? // empty][0] // empty') + if [[ -n "$output" ]]; then + printf '%s' "$output" | base64 --decode || true + printf '\n' + fi + if [[ "$exit_code" == '0' ]]; then + printf '%s completed successfully: %s\n' "$ACTION" "$invoke_id" + exit 0 + fi + printf '%s\n' "$result_json" >&2 + exit 1 + ;; + Failed|Stopped|PartialFailed|Timeout) + printf '%s\n' "$result_json" >&2 + exit 1 + ;; + esac + sleep 5 +done + +printf 'timed out locally; stopping Cloud Assistant invocation %s\n' "$invoke_id" >&2 +aliyun ecs StopInvocation \ + --RegionId "$REGION_ID" \ + --InvokeId "$invoke_id" \ + --InstanceId.1 "$INSTANCE_ID" \ + "${aliyun_profile_args[@]}" >/dev/null || true +for _ in $(seq 1 "$cancel_polls"); do + result_json=$(aliyun ecs DescribeInvocationResults \ + --RegionId "$REGION_ID" \ + --InvokeId "$invoke_id" \ + --InstanceId "$INSTANCE_ID" \ + "${aliyun_profile_args[@]}" || true) + status=$(printf '%s' "$result_json" \ + | jq -r '[.. | objects | .InvocationStatus? // empty][0] // empty') + case "$status" in + Success|Finished|Failed|Stopped|PartialFailed|Timeout) + printf 'invocation reached terminal state after cancellation: %s\n' "$status" >&2 + exit 1 + ;; + esac + sleep 5 +done +printf 'invocation did not confirm cancellation: %s\n' "$invoke_id" >&2 +exit 1 diff --git a/deployment/aliyun/rust-lob-control-plane-lib.sh b/deployment/aliyun/rust-lob-control-plane-lib.sh new file mode 100644 index 000000000..833e957ba --- /dev/null +++ b/deployment/aliyun/rust-lob-control-plane-lib.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Pure monotonic freshness transition used by the host gate and its tests. +# Output: last_updated_ns last_advance_mono max_gap_seconds sample_increment +monday_observe_health_freshness() { + [[ $# -eq 6 ]] || return 2 + local last_updated_ns=$1 + local last_advance_mono=$2 + local max_gap_seconds=$3 + local current_updated_ns=$4 + local current_mono=$5 + local allowed_gap_seconds=$6 + local gap_seconds sample_increment=0 + + [[ $last_updated_ns =~ ^[0-9]+$ \ + && $last_advance_mono =~ ^[0-9]+$ \ + && $max_gap_seconds =~ ^[0-9]+$ \ + && $current_updated_ns =~ ^[0-9]+$ \ + && $current_mono =~ ^[0-9]+$ \ + && $allowed_gap_seconds =~ ^[1-9][0-9]*$ ]] || return 2 + ((current_updated_ns >= last_updated_ns)) || return 1 + ((current_mono >= last_advance_mono)) || return 1 + + gap_seconds=$((current_mono - last_advance_mono)) + ((gap_seconds > max_gap_seconds)) && max_gap_seconds=$gap_seconds + ((gap_seconds <= allowed_gap_seconds)) || return 1 + + if ((current_updated_ns > last_updated_ns)); then + last_updated_ns=$current_updated_ns + last_advance_mono=$current_mono + sample_increment=1 + fi + printf '%s %s %s %s\n' \ + "$last_updated_ns" "$last_advance_mono" "$max_gap_seconds" "$sample_increment" +} diff --git a/deployment/aliyun/rust-lob-runtime-health-policy.jq b/deployment/aliyun/rust-lob-runtime-health-policy.jq new file mode 100644 index 000000000..bfb50e0a7 --- /dev/null +++ b/deployment/aliyun/rust-lob-runtime-health-policy.jq @@ -0,0 +1,13 @@ +.status == "synced" +and .sequence_gaps == 0 +and .symbol_count >= $minimum_symbols +and .snapshot_ready_count == .symbol_count +and .pending_upload_segments == 0 +and .queue_saturated == false +and .disk_warning == false +and .upload_warning == false +and (.updated_at_ns | type) == "number" +and .updated_at_ns > $minimum_updated_ns +and (.session_id | type) == "string" +and (.session_id | length) > 0 +and ($old_session == "" or .session_id != $old_session) diff --git a/deployment/aliyun/rust-lob-shadow-gate-policy.jq b/deployment/aliyun/rust-lob-shadow-gate-policy.jq new file mode 100644 index 000000000..cd6546aaa --- /dev/null +++ b/deployment/aliyun/rust-lob-shadow-gate-policy.jq @@ -0,0 +1,52 @@ +.schema == "monday.rust_lob_shadow_gate.v2" +and .candidate_sha256 == $candidate_sha256 +and .deployment_bundle_sha256 == $deployment_bundle_sha256 +and .deployment_source_revision == $deployment_source_revision +and .passed == true +and .production_eligible == true +and .checks_passed == true +and (.duration_seconds | type) == "number" +and .duration_seconds == (.duration_seconds | floor) +and .duration_seconds >= 3600 +and (.markets.spot.symbol_count | type) == "number" +and .markets.spot.symbol_count == (.markets.spot.symbol_count | floor) +and .markets.spot.symbol_count >= 1000 +and .markets.spot.snapshot_ready_count == .markets.spot.symbol_count +and .markets.spot.sequence_gaps == 0 +and (.markets.spot.upload_failure_count | type) == "number" +and .markets.spot.upload_failure_count == (.markets.spot.upload_failure_count | floor) +and .markets.spot.upload_failure_count >= 0 +and (.markets.spot.health_samples | type) == "number" +and .markets.spot.health_samples == (.markets.spot.health_samples | floor) +and .markets.spot.health_samples >= 40 +and (.markets.spot.max_health_silence_seconds | type) == "number" +and .markets.spot.max_health_silence_seconds >= 0 +and .markets.spot.max_health_silence_seconds <= 90 +and (.markets.spot.catalog_sha256 | type) == "string" +and (.markets.spot.catalog_sha256 | test("^[a-f0-9]{64}$")) +and (.markets.spot.session_id | type) == "string" +and (.markets.spot.session_id | length) > 0 +and (.markets.spot.oss_roundtrips | type) == "number" +and .markets.spot.oss_roundtrips == (.markets.spot.oss_roundtrips | floor) +and .markets.spot.oss_roundtrips >= 2 +and (.markets.usdm.symbol_count | type) == "number" +and .markets.usdm.symbol_count == (.markets.usdm.symbol_count | floor) +and .markets.usdm.symbol_count >= 400 +and .markets.usdm.snapshot_ready_count == .markets.usdm.symbol_count +and .markets.usdm.sequence_gaps == 0 +and (.markets.usdm.upload_failure_count | type) == "number" +and .markets.usdm.upload_failure_count == (.markets.usdm.upload_failure_count | floor) +and .markets.usdm.upload_failure_count >= 0 +and (.markets.usdm.health_samples | type) == "number" +and .markets.usdm.health_samples == (.markets.usdm.health_samples | floor) +and .markets.usdm.health_samples >= 40 +and (.markets.usdm.max_health_silence_seconds | type) == "number" +and .markets.usdm.max_health_silence_seconds >= 0 +and .markets.usdm.max_health_silence_seconds <= 90 +and (.markets.usdm.catalog_sha256 | type) == "string" +and (.markets.usdm.catalog_sha256 | test("^[a-f0-9]{64}$")) +and (.markets.usdm.session_id | type) == "string" +and (.markets.usdm.session_id | length) > 0 +and (.markets.usdm.oss_roundtrips | type) == "number" +and .markets.usdm.oss_roundtrips == (.markets.usdm.oss_roundtrips | floor) +and .markets.usdm.oss_roundtrips >= 2 diff --git a/deployment/aliyun/test-rust-lob-control-plane.sh b/deployment/aliyun/test-rust-lob-control-plane.sh new file mode 100755 index 000000000..ffdc9e8bc --- /dev/null +++ b/deployment/aliyun/test-rust-lob-control-plane.sh @@ -0,0 +1,270 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016 +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd) +CUTOVER="$SCRIPT_DIR/host-rust-lob-cutover.sh" +INVOKE="$SCRIPT_DIR/invoke-rust-lob-operation.sh" +POLICY="$SCRIPT_DIR/rust-lob-shadow-gate-policy.jq" +RUNTIME_POLICY="$SCRIPT_DIR/rust-lob-runtime-health-policy.jq" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/rust-lob-control-plane-lib.sh" + +for command in awk base64 cut grep jq mktemp sed seq; do + command -v "$command" >/dev/null 2>&1 || { + printf 'missing test dependency: %s\n' "$command" >&2 + exit 2 + } +done + +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT + +last_updated_ns=1 +last_advance_mono=0 +max_gap=0 +health_sample_increments=0 +for current_mono in $(seq 30 30 3600); do + current_updated_ns=$((current_mono * 1000000000)) + read -r last_updated_ns last_advance_mono max_gap sample_increment < <( + monday_observe_health_freshness \ + "$last_updated_ns" "$last_advance_mono" "$max_gap" \ + "$current_updated_ns" "$current_mono" 90 + ) + health_sample_increments=$((health_sample_increments + sample_increment)) +done +((health_sample_increments == 120 && max_gap <= 90)) || { + printf 'fresh one-hour health sequence did not pass the monotonic observer\n' >&2 + exit 1 +} +if monday_observe_health_freshness \ + "$last_updated_ns" "$last_advance_mono" "$max_gap" \ + "$last_updated_ns" "$((last_advance_mono + 91))" 90 >/dev/null; then + printf 'monotonic observer accepted a 91-second health freeze\n' >&2 + exit 1 +fi + +artifact=$(printf 'a%.0s' {1..64}) +bundle=$(printf 'b%.0s' {1..64}) +source_revision=$(printf 'c%.0s' {1..40}) +catalog=$(printf 'd%.0s' {1..64}) + +market_json=$(jq -cn \ + --arg catalog "$catalog" \ + '{symbol_count:1200,snapshot_ready_count:1200,sequence_gaps:0, + upload_failure_count:0,health_samples:121,max_health_silence_seconds:30, + catalog_sha256:$catalog, + session_id:"session-1",oss_roundtrips:2}') +jq -n \ + --arg artifact "$artifact" \ + --arg bundle "$bundle" \ + --arg source "$source_revision" \ + --argjson market "$market_json" \ + '{schema:"monday.rust_lob_shadow_gate.v2",candidate_sha256:$artifact, + deployment_bundle_sha256:$bundle,deployment_source_revision:$source, + passed:true,production_eligible:true,checks_passed:true,duration_seconds:3600, + markets:{spot:$market,usdm:($market + {symbol_count:500,snapshot_ready_count:500})}}' \ + >"$tmp_dir/gate.json" + +jq -e \ + --arg candidate_sha256 "$artifact" \ + --arg deployment_bundle_sha256 "$bundle" \ + --arg deployment_source_revision "$source_revision" \ + -f "$POLICY" "$tmp_dir/gate.json" >/dev/null + +wrong_bundle=$(printf 'e%.0s' {1..64}) +if jq -e \ + --arg candidate_sha256 "$artifact" \ + --arg deployment_bundle_sha256 "$wrong_bundle" \ + --arg deployment_source_revision "$source_revision" \ + -f "$POLICY" "$tmp_dir/gate.json" >/dev/null; then + printf 'gate policy accepted evidence from a different deployment bundle\n' >&2 + exit 1 +fi + +wrong_artifact=$(printf 'f%.0s' {1..64}) +if jq -e \ + --arg candidate_sha256 "$wrong_artifact" \ + --arg deployment_bundle_sha256 "$bundle" \ + --arg deployment_source_revision "$source_revision" \ + -f "$POLICY" "$tmp_dir/gate.json" >/dev/null; then + printf 'gate policy accepted evidence from a different binary artifact\n' >&2 + exit 1 +fi + +wrong_source=$(printf '9%.0s' {1..40}) +if jq -e \ + --arg candidate_sha256 "$artifact" \ + --arg deployment_bundle_sha256 "$bundle" \ + --arg deployment_source_revision "$wrong_source" \ + -f "$POLICY" "$tmp_dir/gate.json" >/dev/null; then + printf 'gate policy accepted evidence from a different source revision\n' >&2 + exit 1 +fi + +jq '.markets.spot.health_samples = 1' "$tmp_dir/gate.json" >"$tmp_dir/short-sampling.json" +if jq -e \ + --arg candidate_sha256 "$artifact" \ + --arg deployment_bundle_sha256 "$bundle" \ + --arg deployment_source_revision "$source_revision" \ + -f "$POLICY" "$tmp_dir/short-sampling.json" >/dev/null; then + printf 'gate policy accepted insufficient continuous health samples\n' >&2 + exit 1 +fi + +jq '.markets.usdm.max_health_silence_seconds = 91' \ + "$tmp_dir/gate.json" >"$tmp_dir/stale-health.json" +if jq -e \ + --arg candidate_sha256 "$artifact" \ + --arg deployment_bundle_sha256 "$bundle" \ + --arg deployment_source_revision "$source_revision" \ + -f "$POLICY" "$tmp_dir/stale-health.json" >/dev/null; then + printf 'gate policy accepted a health freshness gap over 90 seconds\n' >&2 + exit 1 +fi + +jq -n '{status:"synced",sequence_gaps:0,symbol_count:1200, + snapshot_ready_count:1200,pending_upload_segments:0,queue_saturated:false, + disk_warning:false,upload_warning:false,updated_at_ns:200,session_id:"new-session"}' \ + >"$tmp_dir/runtime-health.json" +jq -e \ + --arg old_session old-session \ + --argjson minimum_symbols 1000 \ + --argjson minimum_updated_ns 100 \ + -f "$RUNTIME_POLICY" "$tmp_dir/runtime-health.json" >/dev/null +if jq -e \ + --arg old_session old-session \ + --argjson minimum_symbols 1000 \ + --argjson minimum_updated_ns 200 \ + -f "$RUNTIME_POLICY" "$tmp_dir/runtime-health.json" >/dev/null; then + printf 'runtime policy accepted health that was not newer than restart\n' >&2 + exit 1 +fi +if jq -e \ + --arg old_session new-session \ + --argjson minimum_symbols 1000 \ + --argjson minimum_updated_ns 100 \ + -f "$RUNTIME_POLICY" "$tmp_dir/runtime-health.json" >/dev/null; then + printf 'runtime policy accepted a stale session\n' >&2 + exit 1 +fi + +rollback_body="$tmp_dir/rollback.sh" +sed -n '/^rollback_after_failure()/,/^}/p' "$CUTOVER" >"$rollback_body" +start_line=$(grep -n 'systemctl start "${PRODUCTION_UNITS\[@\]}"' "$rollback_body" | tail -1 | cut -d: -f1) +clear_line=$(grep -n 'clear_health_before_restart' "$rollback_body" | cut -d: -f1) +health_line=$(grep -n 'wait_for_release_health' "$rollback_body" | cut -d: -f1) +enable_line=$(grep -n 'systemctl enable "${PRODUCTION_UNITS\[@\]}"' "$rollback_body" | cut -d: -f1) +((clear_line < start_line && start_line < health_line && health_line < enable_line)) || { + printf 'rollback no longer follows clear stale health -> start -> verify -> enable\n' >&2 + exit 1 +} +grep -Fq 'runtime_matches_release "$OLD_BINARY" true' "$rollback_body" +grep -Fq '"$rollback_started_ns"' "$rollback_body" +grep -Fq 'previous-release-health-unverified-disabled' "$rollback_body" +grep -Fq 'systemctl mask --runtime "${TRANSITION_MASK_UNITS[@]}"' "$rollback_body" + +mock_bin="$tmp_dir/bin" +mock_state="$tmp_dir/mock-state" +mkdir -p "$mock_bin" "$mock_state" +cat >"$mock_bin/aliyun" <<'MOCK_ALIYUN' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"$MOCK_STATE_DIR/calls.log" +case "${1:-} ${2:-}" in + 'ecs RunCommand') + printf '{"InvokeId":"mock-invoke"}\n' + ;; + 'ecs DescribeInvocationResults') + if [[ ${MOCK_TRANSIENT_ONCE:-0} == 1 && ! -f $MOCK_STATE_DIR/transient-seen ]]; then + : >"$MOCK_STATE_DIR/transient-seen" + exit 1 + elif [[ -f $MOCK_STATE_DIR/stopped && ${MOCK_IGNORE_STOP:-0} != 1 ]]; then + status=Stopped + exit_code=-1 + else + status=${MOCK_STATUS:-Success} + exit_code=${MOCK_EXIT_CODE:-0} + fi + printf '{"Invocation":{"InvocationStatus":"%s","ExitCode":"%s"}}\n' \ + "$status" "$exit_code" + ;; + 'ecs StopInvocation') + : >"$MOCK_STATE_DIR/stopped" + printf '{}\n' + ;; + *) + printf 'unexpected aliyun call: %s\n' "$*" >&2 + exit 2 + ;; +esac +MOCK_ALIYUN +cat >"$mock_bin/sleep" <<'MOCK_SLEEP' +#!/usr/bin/env bash +exit 0 +MOCK_SLEEP +chmod +x "$mock_bin/aliyun" "$mock_bin/sleep" + +common_env=( + PATH="$mock_bin:$PATH" + MOCK_STATE_DIR="$mock_state" + ACTION=gate + INSTANCE_ID=i-test123 + ARTIFACT_SHA256="$artifact" + MONDAY_ALLOW_SHORT_OPERATION_TEST=1 + MONDAY_OPERATION_TEST_POLLS=2 + MONDAY_OPERATION_TEST_CANCEL_POLLS=2 +) + +run_commands_before=$(grep -c 'ecs RunCommand' "$mock_state/calls.log" 2>/dev/null || true) +if env \ + PATH="$mock_bin:$PATH" \ + MOCK_STATE_DIR="$mock_state" \ + ACTION=cutover \ + INSTANCE_ID=i-test123 \ + ARTIFACT_SHA256="$artifact" \ + MONDAY_OPERATION_TEST_POLLS=invalid \ + "$INVOKE" >"$tmp_dir/preflight.out" 2>&1; then + printf 'operation wrapper accepted unauthorized test polling parameters\n' >&2 + exit 1 +fi +run_commands_after=$(grep -c 'ecs RunCommand' "$mock_state/calls.log" 2>/dev/null || true) +[[ $run_commands_after == "$run_commands_before" ]] || { + printf 'operation wrapper launched a remote command before validating test parameters\n' >&2 + exit 1 +} + +env "${common_env[@]}" MOCK_STATUS=Success MOCK_EXIT_CODE=0 "$INVOKE" \ + >"$tmp_dir/success.out" +grep -Fq 'gate completed successfully: mock-invoke' "$tmp_dir/success.out" + +rm -f "$mock_state/stopped" "$mock_state/transient-seen" +env "${common_env[@]}" MOCK_TRANSIENT_ONCE=1 MOCK_STATUS=Success MOCK_EXIT_CODE=0 \ + "$INVOKE" >"$tmp_dir/transient.out" +grep -Fq 'gate completed successfully: mock-invoke' "$tmp_dir/transient.out" + +rm -f "$mock_state/stopped" +if env "${common_env[@]}" MOCK_STATUS=PartialFailed MOCK_EXIT_CODE=1 "$INVOKE" \ + >"$tmp_dir/failed.out" 2>&1; then + printf 'operation wrapper accepted PartialFailed\n' >&2 + exit 1 +fi + +rm -f "$mock_state/stopped" +if env "${common_env[@]}" MOCK_STATUS=Running MOCK_EXIT_CODE=0 "$INVOKE" \ + >"$tmp_dir/timeout.out" 2>&1; then + printf 'operation wrapper accepted a locally timed-out invocation\n' >&2 + exit 1 +fi +grep -Fq 'ecs StopInvocation' "$mock_state/calls.log" +grep -Fq 'invocation reached terminal state after cancellation: Stopped' "$tmp_dir/timeout.out" + +rm -f "$mock_state/stopped" +if env "${common_env[@]}" MOCK_STATUS=Running MOCK_IGNORE_STOP=1 "$INVOKE" \ + >"$tmp_dir/unconfirmed.out" 2>&1; then + printf 'operation wrapper accepted an unconfirmed cancellation\n' >&2 + exit 1 +fi +grep -Fq 'invocation did not confirm cancellation' "$tmp_dir/unconfirmed.out" + +printf 'Rust collector control-plane contracts passed\n' diff --git a/deployment/aliyun/test_binance_lob_archiver.py b/deployment/aliyun/test_binance_lob_archiver.py deleted file mode 100644 index dc3e40b5b..000000000 --- a/deployment/aliyun/test_binance_lob_archiver.py +++ /dev/null @@ -1,518 +0,0 @@ -import importlib.util -import io -import tempfile -import unittest -from email.message import Message -from pathlib import Path -from types import SimpleNamespace -from urllib.error import HTTPError -from unittest.mock import patch - - -MODULE_PATH = Path(__file__).with_name("binance_lob_archiver.py") -SPEC = importlib.util.spec_from_file_location("binance_lob_archiver", MODULE_PATH) -ARCHIVER = importlib.util.module_from_spec(SPEC) -assert SPEC.loader is not None -SPEC.loader.exec_module(ARCHIVER) - - -def diff(first_update_id, final_update_id, bids=None, asks=None): - return { - "U": first_update_id, - "u": final_update_id, - "b": bids or [], - "a": asks or [], - } - - -class OrderBookStateTests(unittest.TestCase): - def test_snapshot_bridges_buffered_diff_and_deletes_zero_quantity(self): - state = ARCHIVER.OrderBookState("BTCUSDT") - state.apply_diff(diff(98, 100, bids=[["99", "3"]])) - state.apply_diff( - diff(101, 102, bids=[["100", "0"], ["101", "4"]]) - ) - - state.install_snapshot( - { - "lastUpdateId": 100, - "bids": [["100", "2"]], - "asks": [["102", "5"]], - } - ) - - self.assertTrue(state.synced) - self.assertEqual(state.last_update_id, 102) - self.assertNotIn("100", state.bids) - self.assertEqual(state.bids["101"], "4") - - def test_sequence_gap_is_rejected_after_sync(self): - state = ARCHIVER.OrderBookState("ETHUSDT") - state.install_snapshot( - {"lastUpdateId": 10, "bids": [["9", "1"]], "asks": [["11", "1"]]} - ) - state.apply_diff(diff(11, 11)) - - with self.assertRaises(ARCHIVER.SequenceGap) as caught: - state.apply_diff(diff(13, 13)) - - self.assertEqual(caught.exception.expected, 12) - self.assertEqual(caught.exception.first_update_id, 13) - self.assertFalse(state.synced) - - def test_checkpoint_round_trips_full_book(self): - state = ARCHIVER.OrderBookState("SOLUSDT") - state.install_snapshot( - { - "lastUpdateId": 20, - "bids": [["20", "2"], ["19", "1"]], - "asks": [["21", "3"], ["22", "4"]], - } - ) - state.apply_diff(diff(21, 21, asks=[["21", "0"], ["23", "5"]])) - - checkpoint = state.checkpoint("session-1") - - self.assertEqual(checkpoint["last_update_id"], 21) - self.assertEqual(checkpoint["bids"], [["20", "2"], ["19", "1"]]) - self.assertEqual(checkpoint["asks"], [["22", "4"], ["23", "5"]]) - - def test_usdm_uses_snapshot_bridge_then_previous_update_id(self): - state = ARCHIVER.OrderBookState("BTCUSDT", "usdm") - state.install_snapshot( - {"lastUpdateId": 100, "bids": [["99", "1"]], "asks": [["101", "1"]]} - ) - state.apply_diff({**diff(99, 101), "pu": 98}) - self.assertTrue(state.bridged) - - state.apply_diff({**diff(102, 103), "pu": 101}) - self.assertEqual(state.last_update_id, 103) - - with self.assertRaises(ARCHIVER.SequenceGap): - state.apply_diff({**diff(104, 104), "pu": 102}) - - def test_usdm_can_bridge_when_global_update_range_moved_past_snapshot(self): - state = ARCHIVER.OrderBookState("1000SHIBUSDT", "usdm") - state.install_snapshot( - {"lastUpdateId": 100, "bids": [["9", "1"]], "asks": [["11", "1"]]} - ) - - state.apply_diff({**diff(150, 175), "pu": 100}) - - self.assertTrue(state.bridged) - self.assertEqual(state.last_update_id, 175) - - def test_snapshot_fetch_retries_rate_limit_without_restarting_session(self): - rate_limit = HTTPError( - "https://example.test/depth", - 429, - "Too Many Requests", - Message(), - None, - ) - response = io.BytesIO( - b'{"lastUpdateId":1,"bids":[],"asks":[]}' - ) - with ( - patch.object( - ARCHIVER.urllib.request, - "urlopen", - side_effect=[rate_limit, response], - ), - patch.object(ARCHIVER.time, "sleep") as sleep, - ): - snapshot = ARCHIVER.fetch_snapshot_sync("btcusdt") - - self.assertEqual(snapshot["snapshot"]["lastUpdateId"], 1) - sleep.assert_called_once_with(1) - - def test_pending_budget_is_shared_across_symbols(self): - budget = ARCHIVER.PendingBudget(1) - btc = ARCHIVER.OrderBookState("BTCUSDT", pending_budget=budget) - eth = ARCHIVER.OrderBookState("ETHUSDT", pending_budget=budget) - btc.apply_diff(diff(1, 1)) - - with self.assertRaisesRegex(RuntimeError, "pending diff budget"): - eth.apply_diff(diff(1, 1)) - - btc.install_snapshot( - {"lastUpdateId": 1, "bids": [], "asks": []} - ) - eth.apply_diff(diff(1, 1)) - - -class RuntimeContractTests(unittest.IsolatedAsyncioTestCase): - async def asyncSetUp(self): - self.temporary = tempfile.TemporaryDirectory() - self.original_spool = ARCHIVER.SPOOL_DIR - ARCHIVER.SPOOL_DIR = Path(self.temporary.name) - - async def asyncTearDown(self): - ARCHIVER.SPOOL_DIR = self.original_spool - self.temporary.cleanup() - - async def test_unsafe_segment_does_not_emit_checkpoint(self): - runtime = ARCHIVER.ArchiveRuntime() - state = ARCHIVER.OrderBookState("BTCUSDT") - state.install_snapshot( - {"lastUpdateId": 10, "bids": [["9", "1"]], "asks": [["11", "1"]]} - ) - state.apply_diff(diff(11, 11)) - runtime.segment.mark_replay_unsafe() - - runtime.write_checkpoints({"BTCUSDT": state}, "session-1", "gap") - - self.assertEqual(runtime.segment.counts["checkpoint"], 0) - runtime.segment.close() - - async def test_finalize_keeps_source_until_manifest_is_committed(self): - path = ARCHIVER.SPOOL_DIR / "part-1.jsonl.part" - path.write_text('{"received_at_ns":1,"type":"diff"}\n') - output = path.with_suffix("").with_suffix(".jsonl.zst") - temporary_output = output.with_suffix(output.suffix + ".tmp") - - def compress(*args, **kwargs): - self.assertTrue(path.exists()) - temporary_output.write_bytes(b"compressed") - - with patch.object(ARCHIVER.subprocess, "run", side_effect=compress): - manifest = ARCHIVER.finalize_segment( - path, - ARCHIVER.Counter({"diff": 1}), - 1, - 1, - ) - - self.assertIsNotNone(manifest) - self.assertFalse(path.exists()) - self.assertTrue(manifest.exists()) - - async def test_recovery_drops_truncated_json_lines_before_compression(self): - path = ARCHIVER.SPOOL_DIR / "part-1.jsonl.part" - valid = b'{"received_at_ns":1,"type":"diff"}\n' - path.write_bytes(valid + b'{"received_at_ns":') - - with ( - patch.object(ARCHIVER, "finalize_segment") as finalize, - self.assertLogs("binance-lob-archiver", level="WARNING") as logs, - ): - ARCHIVER.recover_parts() - - self.assertEqual(path.read_bytes(), valid) - counts = finalize.call_args.args[1] - self.assertEqual(counts, ARCHIVER.Counter({"diff": 1})) - self.assertIn("invalid trailing", logs.output[0]) - - async def test_recovery_quarantines_invalid_middle_json(self): - path = ARCHIVER.SPOOL_DIR / "part-1.jsonl.part" - valid = b'{"received_at_ns":1,"type":"diff"}\n' - path.write_bytes(valid + b"not-json\n" + valid) - - with ( - patch.object(ARCHIVER, "finalize_segment") as finalize, - self.assertLogs("binance-lob-archiver", level="ERROR") as logs, - ): - ARCHIVER.recover_parts() - - self.assertFalse(path.exists()) - self.assertTrue(path.with_suffix(path.suffix + ".corrupt").exists()) - finalize.assert_not_called() - self.assertIn("invalid middle", logs.output[0]) - - async def test_recovery_quarantines_schema_invalid_middle_json(self): - path = ARCHIVER.SPOOL_DIR / "part-1.jsonl.part" - valid = b'{"received_at_ns":1,"type":"diff"}\n' - path.write_bytes(valid + b"null\n" + valid) - - with ( - patch.object(ARCHIVER, "finalize_segment") as finalize, - self.assertLogs("binance-lob-archiver", level="ERROR"), - ): - ARCHIVER.recover_parts() - - self.assertFalse(path.exists()) - self.assertTrue(path.with_suffix(path.suffix + ".corrupt").exists()) - finalize.assert_not_called() - - async def test_partial_market_state_cannot_emit_replay_safe_checkpoint(self): - runtime = ARCHIVER.ArchiveRuntime() - synced = ARCHIVER.OrderBookState("BTCUSDT") - synced.install_snapshot( - {"lastUpdateId": 10, "bids": [["9", "1"]], "asks": [["11", "1"]]} - ) - unsynced = ARCHIVER.OrderBookState("ETHUSDT") - - runtime.write_checkpoints( - {"BTCUSDT": synced, "ETHUSDT": unsynced}, - "session-1", - "scheduled", - ) - - self.assertFalse(runtime.segment.replay_safe) - self.assertEqual(runtime.segment.counts["checkpoint"], 0) - runtime.segment.close() - - async def test_gap_is_recorded_before_trailing_archive_only_diffs(self): - runtime = ARCHIVER.ArchiveRuntime() - state = ARCHIVER.OrderBookState("ETHUSDT") - state.install_snapshot( - {"lastUpdateId": 10, "bids": [["9", "1"]], "asks": [["11", "1"]]} - ) - state.apply_diff(diff(11, 11)) - - with self.assertRaises(ARCHIVER.SequenceGap): - ARCHIVER.archive_and_apply_diff( - runtime, - {"ETHUSDT": state}, - "session-1", - 123, - {"data": {"s": "ETHUSDT", **diff(13, 13)}}, - ) - - self.assertFalse(runtime.segment.replay_safe) - self.assertEqual(runtime.segment.counts["diff"], 1) - self.assertEqual(runtime.segment.counts["sequence_gap"], 1) - runtime.segment.close() - - async def test_pending_uploads_are_retried_without_restart(self): - runtime = ARCHIVER.ArchiveRuntime() - - with patch.object(ARCHIVER, "upload_pending") as pending: - await runtime.retry_uploads_if_due(force=True) - assert runtime.upload_thread is not None - runtime.upload_thread.join(timeout=1) - - pending.assert_called_once_with() - runtime.segment.close() - - async def test_upload_failure_is_exposed_to_health_checks(self): - manifest = ARCHIVER.SPOOL_DIR / "failed.manifest.json" - manifest.write_text("{}") - with ( - patch.object(ARCHIVER, "upload", side_effect=RuntimeError("oss down")), - self.assertLogs("binance-lob-archiver", level="ERROR"), - ): - ARCHIVER.upload_pending() - - self.assertIsNotNone(ARCHIVER.UPLOAD_STATUS["last_error_at"]) - self.assertEqual(ARCHIVER.UPLOAD_STATUS["last_error"], "oss down") - - async def test_collection_does_not_block_startup_on_pending_uploads(self): - stop = ARCHIVER.asyncio.Event() - - class Runtime: - total_gaps = 0 - - async def rotate(self, *args, **kwargs): - return None - - async def finish_uploads(self): - return None - - async def run_session(*args): - stop.set() - - with ( - patch.object(ARCHIVER, "recover_parts"), - patch.object(ARCHIVER, "upload_pending") as upload_pending, - patch.object(ARCHIVER, "ArchiveRuntime", return_value=Runtime()), - patch.object(ARCHIVER, "run_session", side_effect=run_session), - patch.object(ARCHIVER, "SYMBOLS", ("btcusdt",)), - ): - await ARCHIVER.collect(stop) - - upload_pending.assert_not_called() - - async def test_stall_watchdog_trips_only_after_timeout(self): - with patch.object(ARCHIVER, "STALL_TIMEOUT_SECONDS", 60): - self.assertFalse(ARCHIVER.is_stalled(100, 160)) - self.assertTrue(ARCHIVER.is_stalled(100, 160.1)) - - async def test_process_watchdog_has_independent_deadline(self): - with patch.object(ARCHIVER, "PROCESS_WATCHDOG_SECONDS", 180): - self.assertFalse(ARCHIVER.process_watchdog_expired(100, 280)) - self.assertTrue(ARCHIVER.process_watchdog_expired(100, 280.1)) - - async def test_task_cancellation_is_bounded(self): - self.assertEqual( - ARCHIVER.task_cancelling(SimpleNamespace()), "unsupported" - ) - release = ARCHIVER.asyncio.Event() - - async def stubborn(): - try: - await release.wait() - except ARCHIVER.asyncio.CancelledError: - await release.wait() - - task = ARCHIVER.asyncio.create_task(stubborn(), name="stubborn-test") - await ARCHIVER.asyncio.sleep(0) - with ( - patch.object(ARCHIVER, "TASK_CANCEL_TIMEOUT_SECONDS", 0.01), - self.assertLogs("binance-lob-archiver", level="ERROR") as logs, - ): - with self.assertRaises(ARCHIVER.TaskCancellationStuck): - await ARCHIVER.cancel_tasks_bounded((task,)) - - self.assertIn("name=stubborn-test", logs.output[0]) - self.assertIn("coro=RuntimeContractTests", logs.output[0]) - self.assertIn("cancel_requested=True", logs.output[0]) - self.assertIn("cancelling=1", logs.output[0]) - self.assertIn("location=stubborn:", logs.output[0]) - - release.set() - await task - - async def test_resync_does_not_reuse_expired_initial_deadline(self): - self.assertTrue(ARCHIVER.bridge_timed_out(False, 100, 101)) - self.assertFalse(ARCHIVER.bridge_timed_out(True, 100, 101)) - synced, deadline = ARCHIVER.begin_resync(200) - self.assertFalse(synced) - self.assertEqual(deadline, 200 + ARCHIVER.SYNC_TIMEOUT_SECONDS) - - async def test_disk_watermark_warns_without_stopping_collection(self): - with ( - patch.object(ARCHIVER, "MIN_FREE_GB", 20), - patch.object( - ARCHIVER.shutil, - "disk_usage", - return_value=SimpleNamespace(free=19 * 1024**3), - ), - self.assertLogs("binance-lob-archiver", level="WARNING") as logs, - ): - free_gb, warning = ARCHIVER.warn_if_disk_low() - - self.assertEqual(free_gb, 19.0) - self.assertTrue(warning) - self.assertIn("continuing collection", logs.output[0]) - - async def test_health_reports_disk_warning(self): - state = ARCHIVER.OrderBookState("BTCUSDT") - with ( - patch.object( - ARCHIVER, - "disk_headroom", - return_value=(19.0, True), - ), - patch.object(ARCHIVER, "pending_upload_count", return_value=3), - patch.dict( - ARCHIVER.UPLOAD_STATUS, - { - "last_success_at": "2026-07-13T12:00:00+00:00", - "last_error_at": "2026-07-13T12:01:00+00:00", - "last_error": "oss down", - }, - clear=True, - ), - ): - ARCHIVER.write_health( - {"BTCUSDT": state}, "session-1", "synced", 0 - ) - - health = ARCHIVER.json.loads( - (ARCHIVER.SPOOL_DIR / "health.json").read_text() - ) - self.assertEqual(health["disk_free_gb"], 19.0) - self.assertTrue(health["disk_warning"]) - self.assertEqual(health["disk_warning_threshold_gb"], 20) - self.assertEqual(health["pending_upload_segments"], 3) - self.assertTrue(health["upload_warning"]) - self.assertEqual(health["last_upload_error"], "oss down") - - async def test_streams_are_split_into_bounded_websocket_shards(self): - with ( - patch.object(ARCHIVER, "MARKET", "spot"), - patch.object(ARCHIVER, "SYMBOLS", ("a", "b", "c")), - patch.object(ARCHIVER, "WS_SHARD_SIZE", 2), - ): - urls = ARCHIVER.stream_urls() - - self.assertEqual(len(urls), 2) - self.assertIn("a@depth@100ms/b@depth@100ms", urls[0]) - self.assertTrue(urls[1].endswith("c@depth@100ms")) - - async def test_resync_snapshot_waits_without_blocking_queue_consumer(self): - queue = ARCHIVER.asyncio.Queue(maxsize=1) - await queue.put(("diff", 1, {})) - replacement = { - "symbol": "BTCUSDT", - "received_at_ns": 2, - "snapshot": {"lastUpdateId": 1, "bids": [], "asks": []}, - } - with patch.object(ARCHIVER, "fetch_snapshot_sync", return_value=replacement): - producer = ARCHIVER.asyncio.create_task( - ARCHIVER.produce_snapshot("btcusdt", queue) - ) - await ARCHIVER.asyncio.sleep(0) - self.assertFalse(producer.done()) - await queue.get() - await producer - - self.assertEqual((await queue.get())[0], "snapshot") - - async def test_initial_and_resync_snapshots_can_share_one_limiter(self): - calls = [] - - class Limiter: - async def fetch(self, symbol): - calls.append(symbol) - return { - "symbol": symbol.upper(), - "received_at_ns": len(calls), - "request_started_at_ns": len(calls), - "snapshot": {"lastUpdateId": len(calls), "bids": [], "asks": []}, - } - - queue = ARCHIVER.asyncio.Queue() - limiter = Limiter() - with patch.object(ARCHIVER, "SYMBOLS", ("btcusdt", "ethusdt")): - await ARCHIVER.produce_snapshots(queue, limiter) - await ARCHIVER.produce_snapshot("solusdt", queue, limiter) - - self.assertEqual(calls, ["btcusdt", "ethusdt", "solusdt"]) - - async def test_all_unavailable_symbols_stay_out_of_refreshed_catalog(self): - symbols, security_tokens = ARCHIVER.exclude_unavailable_symbols( - ("badusdt", "oldusdt"), - ("badusdt", "btcusdt", "oldusdt"), - ("badusdt", "oldusdt"), - ) - - self.assertEqual(symbols, ("btcusdt",)) - self.assertEqual(security_tokens, ()) - - async def test_only_binance_invalid_symbol_error_refreshes_catalog(self): - invalid_symbol = HTTPError( - "https://example.test/depth", - 400, - "Bad Request", - Message(), - io.BytesIO(b'{"code":-1121,"msg":"Invalid symbol."}'), - ) - with patch.object( - ARCHIVER.urllib.request, - "urlopen", - side_effect=invalid_symbol, - ): - with self.assertRaises(ARCHIVER.SnapshotUnavailable): - ARCHIVER.fetch_snapshot_sync("badusdt") - - async def test_host_wide_403_does_not_exclude_symbol(self): - forbidden = HTTPError( - "https://example.test/depth", - 403, - "Forbidden", - Message(), - io.BytesIO(b"forbidden"), - ) - with patch.object( - ARCHIVER.urllib.request, - "urlopen", - side_effect=forbidden, - ): - with self.assertRaises(HTTPError): - ARCHIVER.fetch_snapshot_sync("btcusdt") - - -if __name__ == "__main__": - unittest.main() diff --git a/rust_hft/Cargo.lock b/rust_hft/Cargo.lock index 44d1f72c5..2e4667205 100644 --- a/rust_hft/Cargo.lock +++ b/rust_hft/Cargo.lock @@ -3512,6 +3512,7 @@ dependencies = [ "clickhouse 0.12.2", "duckdb", "flate2", + "fs4", "futures", "hex", "hft-engine", diff --git a/rust_hft/tools/collector/Cargo.toml b/rust_hft/tools/collector/Cargo.toml index fd34bddd6..af7c9cb1a 100644 --- a/rust_hft/tools/collector/Cargo.toml +++ b/rust_hft/tools/collector/Cargo.toml @@ -27,6 +27,7 @@ hex = { workspace = true } anyhow = "1" async-trait = "0.1" chrono = { version = "0.4", features = ["serde"] } +fs4 = { workspace = true } rust_decimal = { version = "1", features = ["serde"] } ordered-float = "4" tracing = "0.1" diff --git a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs index a00ca087e..20bcb09de 100644 --- a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs +++ b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs @@ -26,6 +26,9 @@ use tracing::{error, info, warn}; struct Args { #[arg(long)] self_test: bool, + + #[arg(long, conflicts_with = "self_test")] + upload_only: bool, } #[derive(Debug, Clone)] @@ -56,6 +59,74 @@ struct Config { oss_copy_timeout: Duration, } +#[derive(Debug, Clone)] +struct UploadConfig { + spool_dir: PathBuf, + oss_bucket: String, + oss_endpoint: String, + oss_region: String, + aliyun_profile: String, + oss_copy_timeout: Duration, +} + +const UPLOADED_CLEANUP_SCHEMA: &str = "monday.binance_lob.uploaded_cleanup.v1"; +const UPLOADED_CLEANUP_SUFFIX: &str = ".uploaded-cleanup.json"; +const UPLOADED_CLEANUP_TMP_SUFFIX: &str = ".uploaded-cleanup.json.tmp"; +const SPOOL_LOCK_FILE: &str = ".binance-lob-archiver.lock"; + +#[derive(Debug)] +struct SpoolLock { + _file: std::fs::File, +} + +impl SpoolLock { + fn acquire(spool_dir: &Path) -> anyhow::Result { + let path = spool_dir.join(SPOOL_LOCK_FILE); + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&path) + .with_context(|| format!("failed to open spool lock {}", path.display()))?; + if let Err(error) = fs4::FileExt::try_lock(&file) { + let error = std::io::Error::from(error); + if error.kind() == std::io::ErrorKind::WouldBlock { + anyhow::bail!("spool is already locked: {}", spool_dir.display()); + } + return Err(error) + .with_context(|| format!("failed to lock spool {}", spool_dir.display())); + } + Ok(Self { _file: file }) + } +} + +impl UploadConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + spool_dir: PathBuf::from(env_string("SPOOL_DIR", "/data/monday/spool/binance-lob")), + oss_bucket: env_string("OSS_BUCKET", "monday-lob-apne1-1045353359"), + oss_endpoint: env_string("OSS_ENDPOINT", "oss-ap-northeast-1-internal.aliyuncs.com"), + oss_region: env_string("OSS_REGION", "ap-northeast-1"), + aliyun_profile: env_string("ALIYUN_PROFILE", "ecs-role"), + oss_copy_timeout: Duration::from_secs(env_parse("OSS_COPY_TIMEOUT_SECONDS", 300_u64)?), + }) + } +} + +impl From<&Config> for UploadConfig { + fn from(config: &Config) -> Self { + Self { + spool_dir: config.spool_dir.clone(), + oss_bucket: config.oss_bucket.clone(), + oss_endpoint: config.oss_endpoint.clone(), + oss_region: config.oss_region.clone(), + aliyun_profile: config.aliyun_profile.clone(), + oss_copy_timeout: config.oss_copy_timeout, + } + } +} + impl Config { async fn from_env() -> anyhow::Result { if env_string("DEPTH_MODE", "diff") != "diff" { @@ -315,12 +386,18 @@ async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt() .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .init(); - if Args::parse().self_test { + let args = Args::parse(); + if args.self_test { return self_test(); } + if args.upload_only { + return upload_only(&UploadConfig::from_env()?).await; + } + let spool_dir = PathBuf::from(env_string("SPOOL_DIR", "/data/monday/spool/binance-lob")); + std::fs::create_dir_all(&spool_dir)?; + let _spool_lock = SpoolLock::acquire(&spool_dir)?; let config = Arc::new(Config::from_env().await?); - std::fs::create_dir_all(&config.spool_dir)?; let recovered = recover_parts(&config.segment_config())?; if !recovered.is_empty() { info!(segments = recovered.len(), "recovered interrupted segments"); @@ -1016,25 +1093,10 @@ fn parse_symbol_catalog(market: Market, payload: &Value) -> anyhow::Result, mut shutdown: watch::Receiver) { + let upload_config = UploadConfig::from(config.as_ref()); loop { - let mut status = read_upload_status(&config.spool_dir); - match upload_pending(&config).await { - Ok(uploaded) => { - if uploaded > 0 { - status.last_success_at = Some(chrono::Utc::now().to_rfc3339()); - } - status.last_error_at = None; - status.last_error = None; - } - Err(error) => { - let now = chrono::Utc::now().to_rfc3339(); - status.last_error_at = Some(now); - status.last_error = Some(error.to_string().chars().take(500).collect()); - warn!(error = %error, "pending OSS upload failed; files retained"); - } - } - if let Err(error) = write_upload_status(&config.spool_dir, &status) { - error!(error = %error, "failed to persist OSS upload status"); + if let Err(error) = upload_pending_with_status(&upload_config).await { + warn!(error = %error, "pending OSS upload failed; files retained"); } tokio::select! { changed = shutdown.changed() => { @@ -1047,9 +1109,80 @@ async fn upload_loop(config: Arc, mut shutdown: watch::Receiver) { } } -async fn upload_pending(config: &Config) -> anyhow::Result { +async fn upload_only(config: &UploadConfig) -> anyhow::Result<()> { + if !config.spool_dir.is_dir() { + anyhow::bail!( + "SPOOL_DIR is not a directory: {}", + config.spool_dir.display() + ); + } + let _spool_lock = SpoolLock::acquire(&config.spool_dir)?; + let mut incomplete = Vec::new(); + for suffix in [".jsonl.part", ".zst.tmp", ".part.corrupt"] { + incomplete.extend(files_with_suffix(&config.spool_dir, suffix)?); + } + if !incomplete.is_empty() { + anyhow::bail!( + "upload-only drain blocked by {} incomplete segment artifacts; recover them with the collector release that created them", + incomplete.len() + ); + } + let uploaded = upload_pending_with_status(config).await?; + let pending = files_with_suffix(&config.spool_dir, ".manifest.json")?.len(); + if pending > 0 { + anyhow::bail!("upload-only drain incomplete: {pending} manifests remain"); + } + let mut residual = Vec::new(); + for suffix in [ + ".jsonl.part", + ".zst.tmp", + ".part.corrupt", + ".jsonl.zst", + "._SUCCESS", + UPLOADED_CLEANUP_SUFFIX, + UPLOADED_CLEANUP_TMP_SUFFIX, + ] { + residual.extend(files_with_suffix(&config.spool_dir, suffix)?); + } + if !residual.is_empty() { + anyhow::bail!( + "upload-only drain incomplete: {} local segment artifacts remain", + residual.len() + ); + } + println!("upload-only: uploaded={uploaded} pending=0"); + Ok(()) +} + +async fn upload_pending_with_status(config: &UploadConfig) -> anyhow::Result { + let mut status = read_upload_status(&config.spool_dir); + let result = upload_pending(config).await; + match &result { + Ok(uploaded) => { + if *uploaded > 0 { + status.last_success_at = Some(chrono::Utc::now().to_rfc3339()); + } + status.last_error_at = None; + status.last_error = None; + } + Err(error) => { + status.last_error_at = Some(chrono::Utc::now().to_rfc3339()); + status.last_error = Some(error.to_string().chars().take(500).collect()); + status.failure_count = status.failure_count.saturating_add(1); + } + } + write_upload_status(&config.spool_dir, &status) + .context("failed to persist OSS upload status")?; + result +} + +async fn upload_pending(config: &UploadConfig) -> anyhow::Result { let config = config.clone(); tokio::task::spawn_blocking(move || { + let recovered = recover_uploaded_cleanups(&config.spool_dir)?; + if recovered > 0 { + info!(recovered, "completed interrupted local upload cleanup"); + } let mut failures = 0_usize; let mut uploaded = 0_usize; for manifest in files_with_suffix(&config.spool_dir, ".manifest.json")? { @@ -1068,7 +1201,104 @@ async fn upload_pending(config: &Config) -> anyhow::Result { .await? } -fn upload_one(config: &Config, manifest: &Path) -> anyhow::Result<()> { +fn write_uploaded_cleanup_marker( + data: &Path, + manifest: &Path, + success: &Path, +) -> anyhow::Result { + let parent = manifest + .parent() + .ok_or_else(|| anyhow::anyhow!("manifest has no parent: {}", manifest.display()))?; + if data.parent() != Some(parent) || success.parent() != Some(parent) { + anyhow::bail!("uploaded cleanup artifacts must share one directory"); + } + let manifest_name = local_file_name(manifest)?; + let marker = manifest.with_file_name(format!("{manifest_name}{UPLOADED_CLEANUP_SUFFIX}")); + let temporary = marker.with_file_name(format!("{}.tmp", local_file_name(&marker)?)); + let mut bytes = serde_json::to_vec(&json!({ + "schema": UPLOADED_CLEANUP_SCHEMA, + "data": local_file_name(data)?, + "manifest": manifest_name, + "success": local_file_name(success)?, + }))?; + bytes.push(b'\n'); + let mut output = std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&temporary)?; + std::io::Write::write_all(&mut output, &bytes)?; + output.sync_all()?; + std::fs::rename(&temporary, &marker)?; + sync_parent_directory(&marker)?; + Ok(marker) +} + +fn recover_uploaded_cleanups(spool_dir: &Path) -> anyhow::Result { + let markers = files_with_suffix(spool_dir, UPLOADED_CLEANUP_SUFFIX)?; + for marker in &markers { + cleanup_uploaded_marker(marker)?; + } + Ok(markers.len()) +} + +fn cleanup_uploaded_marker(marker: &Path) -> anyhow::Result<()> { + let metadata: Value = serde_json::from_reader(std::fs::File::open(marker)?) + .with_context(|| format!("invalid uploaded cleanup marker {}", marker.display()))?; + if metadata["schema"] != UPLOADED_CLEANUP_SCHEMA { + anyhow::bail!( + "invalid uploaded cleanup marker schema: {}", + marker.display() + ); + } + for field in ["data", "manifest", "success"] { + let path = cleanup_artifact_path(marker, &metadata, field)?; + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!("failed to clean uploaded artifact {}", path.display()) + }); + } + } + } + sync_parent_directory(marker)?; + std::fs::remove_file(marker)?; + sync_parent_directory(marker) +} + +fn cleanup_artifact_path(marker: &Path, metadata: &Value, field: &str) -> anyhow::Result { + let name = metadata[field] + .as_str() + .ok_or_else(|| anyhow::anyhow!("uploaded cleanup marker missing {field}"))?; + if Path::new(name).file_name().and_then(|value| value.to_str()) != Some(name) { + anyhow::bail!("uploaded cleanup marker contains invalid {field} file name"); + } + let path = marker + .parent() + .ok_or_else(|| anyhow::anyhow!("cleanup marker has no parent: {}", marker.display()))? + .join(name); + if path == marker { + anyhow::bail!("uploaded cleanup marker cannot reference itself"); + } + Ok(path) +} + +fn local_file_name(path: &Path) -> anyhow::Result<&str> { + path.file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| anyhow::anyhow!("path has no UTF-8 file name: {}", path.display())) +} + +fn sync_parent_directory(path: &Path) -> anyhow::Result<()> { + if let Some(parent) = path.parent() { + std::fs::File::open(parent)?.sync_all()?; + } + Ok(()) +} + +fn upload_one(config: &UploadConfig, manifest: &Path) -> anyhow::Result<()> { let metadata: Value = serde_json::from_reader(std::fs::File::open(manifest)?)?; let data = manifest.with_file_name( metadata["file"] @@ -1114,10 +1344,8 @@ fn upload_one(config: &Config, manifest: &Path) -> anyhow::Result<()> { ); } } - std::fs::remove_file(data)?; - std::fs::remove_file(manifest)?; - std::fs::remove_file(success)?; - Ok(()) + let marker = write_uploaded_cleanup_marker(&data, manifest, &success)?; + cleanup_uploaded_marker(&marker) } async fn wait_for_signal(shutdown: watch::Sender) { @@ -1207,6 +1435,247 @@ mod tests { use std::io::{Read, Write}; use std::net::TcpListener; + #[test] + fn upload_only_cli_is_explicit_and_exclusive() { + let args = Args::try_parse_from(["binance-lob-archiver", "--upload-only"]).unwrap(); + assert!(args.upload_only); + assert!(!args.self_test); + assert!( + Args::try_parse_from(["binance-lob-archiver", "--upload-only", "--self-test",]) + .is_err() + ); + } + + #[tokio::test] + async fn upload_only_rejects_a_missing_spool() { + let spool_dir = + env::temp_dir().join(format!("monday-upload-only-missing-{}", now_ns().unwrap())); + let config = UploadConfig { + spool_dir: spool_dir.clone(), + oss_bucket: "unused".into(), + oss_endpoint: "unused".into(), + oss_region: "ap-northeast-1".into(), + aliyun_profile: "unused".into(), + oss_copy_timeout: Duration::from_secs(1), + }; + + let error = upload_only(&config).await.unwrap_err(); + assert!(error.to_string().contains("SPOOL_DIR is not a directory")); + assert!(!spool_dir.exists()); + } + + #[tokio::test] + async fn upload_only_accepts_an_empty_spool_without_network_access() { + let spool_dir = + env::temp_dir().join(format!("monday-upload-only-empty-{}", now_ns().unwrap())); + std::fs::create_dir_all(&spool_dir).unwrap(); + let config = UploadConfig { + spool_dir: spool_dir.clone(), + oss_bucket: "unused".into(), + oss_endpoint: "unused".into(), + oss_region: "ap-northeast-1".into(), + aliyun_profile: "unused".into(), + oss_copy_timeout: Duration::from_secs(1), + }; + + upload_only(&config).await.unwrap(); + assert!(spool_dir.join("upload-status.json").is_file()); + assert!(spool_dir.join(SPOOL_LOCK_FILE).is_file()); + assert!(files_with_suffix(&spool_dir, ".manifest.json") + .unwrap() + .is_empty()); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + + #[tokio::test] + async fn upload_only_fails_while_spool_lock_is_held_and_succeeds_after_release() { + let spool_dir = env::temp_dir().join(format!("monday-spool-lock-{}", now_ns().unwrap())); + std::fs::create_dir_all(&spool_dir).unwrap(); + let config = UploadConfig { + spool_dir: spool_dir.clone(), + oss_bucket: "unused".into(), + oss_endpoint: "unused".into(), + oss_region: "ap-northeast-1".into(), + aliyun_profile: "unused".into(), + oss_copy_timeout: Duration::from_secs(1), + }; + + let first = SpoolLock::acquire(&spool_dir).unwrap(); + let error = upload_only(&config).await.unwrap_err(); + assert!(error.to_string().contains("spool is already locked")); + assert!(!spool_dir.join("upload-status.json").exists()); + + drop(first); + upload_only(&config).await.unwrap(); + assert!(spool_dir.join(SPOOL_LOCK_FILE).is_file()); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + + #[tokio::test] + async fn upload_only_rejects_an_interrupted_segment() { + for suffix in ["jsonl.part", "jsonl.zst.tmp", "jsonl.part.corrupt"] { + let spool_dir = env::temp_dir().join(format!( + "monday-upload-only-interrupted-{suffix}-{}", + now_ns().unwrap() + )); + std::fs::create_dir_all(&spool_dir).unwrap(); + let artifact = spool_dir.join(format!("part-1700000000000000000.{suffix}")); + std::fs::write(&artifact, b"unfinished").unwrap(); + let config = UploadConfig { + spool_dir: spool_dir.clone(), + oss_bucket: "unused".into(), + oss_endpoint: "unused".into(), + oss_region: "ap-northeast-1".into(), + aliyun_profile: "unused".into(), + oss_copy_timeout: Duration::from_secs(1), + }; + + let error = upload_only(&config).await.unwrap_err(); + assert!(error.to_string().contains("incomplete segment artifacts")); + assert!(artifact.is_file()); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + } + + #[tokio::test] + async fn upload_only_rejects_an_orphaned_compressed_segment() { + let spool_dir = + env::temp_dir().join(format!("monday-upload-only-zst-{}", now_ns().unwrap())); + std::fs::create_dir_all(&spool_dir).unwrap(); + std::fs::write( + spool_dir.join("part-1700000000000000000.jsonl.zst"), + b"orphaned", + ) + .unwrap(); + let config = UploadConfig { + spool_dir: spool_dir.clone(), + oss_bucket: "unused".into(), + oss_endpoint: "unused".into(), + oss_region: "ap-northeast-1".into(), + aliyun_profile: "unused".into(), + oss_copy_timeout: Duration::from_secs(1), + }; + + let error = upload_only(&config).await.unwrap_err(); + assert!(error.to_string().contains("local segment artifacts remain")); + assert!(spool_dir + .join("part-1700000000000000000.jsonl.zst") + .is_file()); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + + #[test] + fn uploaded_cleanup_recovery_is_idempotent_after_every_interruption_point() { + for removed in 0..=3 { + let spool_dir = env::temp_dir().join(format!( + "monday-upload-cleanup-{removed}-{}", + now_ns().unwrap() + )); + std::fs::create_dir_all(&spool_dir).unwrap(); + let data = spool_dir.join("part-1700000000000000000.jsonl.zst"); + let manifest = spool_dir.join("part-1700000000000000000.jsonl.zst.manifest.json"); + let success = spool_dir.join("part-1700000000000000000.jsonl.zst._SUCCESS"); + for path in [&data, &manifest, &success] { + std::fs::write(path, b"artifact").unwrap(); + } + let marker = write_uploaded_cleanup_marker(&data, &manifest, &success).unwrap(); + + for path in [&data, &manifest, &success].into_iter().take(removed) { + std::fs::remove_file(path).unwrap(); + } + + assert_eq!(recover_uploaded_cleanups(&spool_dir).unwrap(), 1); + for path in [&data, &manifest, &success, &marker] { + assert!(!path.exists(), "{} was not cleaned", path.display()); + } + assert_eq!(recover_uploaded_cleanups(&spool_dir).unwrap(), 0); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + } + + #[test] + fn uploaded_cleanup_marker_is_removed_last() { + let spool_dir = + env::temp_dir().join(format!("monday-upload-cleanup-last-{}", now_ns().unwrap())); + std::fs::create_dir_all(&spool_dir).unwrap(); + let data = spool_dir.join("part-1700000000000000000.jsonl.zst"); + let manifest = spool_dir.join("part-1700000000000000000.jsonl.zst.manifest.json"); + let success = spool_dir.join("part-1700000000000000000.jsonl.zst._SUCCESS"); + for path in [&data, &manifest, &success] { + std::fs::write(path, b"artifact").unwrap(); + } + let marker = write_uploaded_cleanup_marker(&data, &manifest, &success).unwrap(); + std::fs::remove_file(&success).unwrap(); + std::fs::create_dir(&success).unwrap(); + + recover_uploaded_cleanups(&spool_dir).unwrap_err(); + assert!(marker.is_file()); + + std::fs::remove_dir(&success).unwrap(); + assert_eq!(recover_uploaded_cleanups(&spool_dir).unwrap(), 1); + assert!(!marker.exists()); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + + #[tokio::test] + async fn upload_pending_recovers_cleanup_before_reading_manifests() { + let spool_dir = env::temp_dir().join(format!( + "monday-upload-cleanup-before-manifests-{}", + now_ns().unwrap() + )); + std::fs::create_dir_all(&spool_dir).unwrap(); + let data = spool_dir.join("part-1700000000000000000.jsonl.zst"); + let manifest = spool_dir.join("part-1700000000000000000.jsonl.zst.manifest.json"); + let success = spool_dir.join("part-1700000000000000000.jsonl.zst._SUCCESS"); + for path in [&data, &manifest, &success] { + std::fs::write(path, b"not valid manifest JSON").unwrap(); + } + let marker = write_uploaded_cleanup_marker(&data, &manifest, &success).unwrap(); + let config = UploadConfig { + spool_dir: spool_dir.clone(), + oss_bucket: "unused".into(), + oss_endpoint: "unused".into(), + oss_region: "ap-northeast-1".into(), + aliyun_profile: "unused".into(), + oss_copy_timeout: Duration::from_secs(1), + }; + + assert_eq!(upload_pending(&config).await.unwrap(), 0); + assert!(!marker.exists()); + assert!(files_with_suffix(&spool_dir, ".manifest.json") + .unwrap() + .is_empty()); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + + #[tokio::test] + async fn upload_only_rejects_a_residual_cleanup_marker() { + let spool_dir = env::temp_dir().join(format!( + "monday-upload-cleanup-residual-{}", + now_ns().unwrap() + )); + std::fs::create_dir_all(&spool_dir).unwrap(); + let marker = spool_dir.join(format!( + "part-1700000000000000000.jsonl.zst.manifest.json{UPLOADED_CLEANUP_SUFFIX}" + )); + std::fs::write(&marker, b"{}\n").unwrap(); + let config = UploadConfig { + spool_dir: spool_dir.clone(), + oss_bucket: "unused".into(), + oss_endpoint: "unused".into(), + oss_region: "ap-northeast-1".into(), + aliyun_profile: "unused".into(), + oss_copy_timeout: Duration::from_secs(1), + }; + + let error = upload_only(&config).await.unwrap_err(); + assert!(error + .to_string() + .contains("invalid uploaded cleanup marker schema")); + assert!(marker.is_file()); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + fn test_config(rest_base: String) -> Config { Config { market: Market::Spot, diff --git a/rust_hft/tools/collector/src/lob_archiver.rs b/rust_hft/tools/collector/src/lob_archiver.rs index e514c41c0..ab0ea6391 100644 --- a/rust_hft/tools/collector/src/lob_archiver.rs +++ b/rust_hft/tools/collector/src/lob_archiver.rs @@ -721,14 +721,27 @@ pub fn recover_parts(config: &SegmentConfig) -> anyhow::Result anyhow::Result> { fn visit(path: &Path, suffix: &str, files: &mut Vec) -> anyhow::Result<()> { - if !path.exists() { - return Ok(()); + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + if metadata.file_type().is_symlink() { + anyhow::bail!("refusing symlink while scanning spool: {}", path.display()); + } + if !metadata.is_dir() { + anyhow::bail!("spool scan root is not a directory: {}", path.display()); } for entry in fs::read_dir(path)? { - let path = entry?.path(); - if path.is_dir() { + let entry = entry?; + let path = entry.path(); + let file_type = entry.file_type()?; + if file_type.is_symlink() { + anyhow::bail!("refusing symlink while scanning spool: {}", path.display()); + } + if file_type.is_dir() { visit(&path, suffix, files)?; - } else if path.to_string_lossy().ends_with(suffix) { + } else if file_type.is_file() && path.to_string_lossy().ends_with(suffix) { files.push(path); } } @@ -799,6 +812,7 @@ pub fn write_health( "disk_warning": disk_free_gb.is_some_and(|free| free < disk_warning_threshold_gb), "disk_warning_threshold_gb": disk_warning_threshold_gb, "upload_warning": upload.last_error_at.is_some(), + "upload_failure_count": upload.failure_count, "last_upload_success_at": upload.last_success_at, "last_upload_error_at": upload.last_error_at, "last_upload_error": upload.last_error, @@ -815,6 +829,8 @@ pub struct UploadStatus { pub last_success_at: Option, pub last_error_at: Option, pub last_error: Option, + #[serde(default)] + pub failure_count: u64, } pub fn read_upload_status(spool_dir: &Path) -> UploadStatus { @@ -1109,6 +1125,7 @@ mod tests { last_success_at: None, last_error_at: Some("2026-07-14T16:00:00Z".into()), last_error: Some("oss down".into()), + failure_count: 7, }, ) .unwrap(); @@ -1131,6 +1148,7 @@ mod tests { let health: Value = serde_json::from_reader(File::open(root.join("health.json")).unwrap()).unwrap(); assert_eq!(health["upload_warning"], true); + assert_eq!(health["upload_failure_count"], 7); assert_eq!(health["last_upload_error"], "oss down"); assert_eq!(health["snapshot_ready_count"], 0); assert_eq!(health["bridged_count"], 0); @@ -1139,6 +1157,40 @@ mod tests { fs::remove_dir_all(root).unwrap(); } + #[cfg(unix)] + #[test] + fn spool_scan_rejects_root_directory_and_file_symlinks() { + use std::os::unix::fs::symlink; + + let root = std::env::temp_dir().join(format!("monday-scan-root-{}", now_ns().unwrap())); + let outside = + std::env::temp_dir().join(format!("monday-scan-outside-{}", now_ns().unwrap())); + let root_link = + std::env::temp_dir().join(format!("monday-scan-root-link-{}", now_ns().unwrap())); + fs::create_dir_all(&root).unwrap(); + fs::create_dir_all(&outside).unwrap(); + let outside_file = outside.join("outside.manifest.json"); + fs::write(&outside_file, b"do-not-delete\n").unwrap(); + + symlink(&outside, &root_link).unwrap(); + let error = files_with_suffix(&root_link, ".manifest.json").unwrap_err(); + assert!(error.to_string().contains("refusing symlink")); + fs::remove_file(&root_link).unwrap(); + + symlink(&outside, root.join("escape")).unwrap(); + let error = files_with_suffix(&root, ".manifest.json").unwrap_err(); + assert!(error.to_string().contains("refusing symlink")); + fs::remove_file(root.join("escape")).unwrap(); + + symlink(&outside_file, root.join("linked.manifest.json")).unwrap(); + let error = files_with_suffix(&root, ".manifest.json").unwrap_err(); + assert!(error.to_string().contains("refusing symlink")); + assert_eq!(fs::read(&outside_file).unwrap(), b"do-not-delete\n"); + + fs::remove_dir_all(root).unwrap(); + fs::remove_dir_all(outside).unwrap(); + } + #[test] fn segment_emits_v2_manifest_digest_and_success_marker() { if Command::new("zstd").arg("--version").output().is_err() { From 88709aba4253384a55e30d0f7c4a5d4c11a5bc67 Mon Sep 17 00:00:00 2001 From: proerror Date: Wed, 15 Jul 2026 13:51:09 +0800 Subject: [PATCH 2/6] fix(collector): address release review findings --- .github/workflows/ci.yml | 4 +- deployment/aliyun/README.md | 55 ++-- .../binance-lob-archiver-rust-upload@.service | 2 +- .../binance-lob-archiver-upload@.service | 2 +- deployment/aliyun/deploy-rust-lob-release.sh | 93 +++++-- deployment/aliyun/host-rust-lob-cutover.sh | 150 ++++++++--- .../aliyun/host-rust-lob-shadow-gate.sh | 52 +++- .../aliyun/invoke-rust-lob-operation.sh | 2 +- .../aliyun/rust-lob-runtime-health-policy.jq | 8 +- .../aliyun/test-rust-lob-control-plane.sh | 224 ++++++++++++++-- .../collector/src/bin/binance-lob-archiver.rs | 247 ++++++++++++++++-- 11 files changed, 719 insertions(+), 120 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5daeaf68d..096447f6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,9 @@ jobs: - name: Verify collector release control plane run: | - bash -n ../deployment/aliyun/*.sh + for script in ../deployment/aliyun/*.sh; do + bash -n "$script" + done shellcheck ../deployment/aliyun/*.sh ../deployment/aliyun/test-rust-lob-control-plane.sh diff --git a/deployment/aliyun/README.md b/deployment/aliyun/README.md index fee7f9224..4e033481d 100644 --- a/deployment/aliyun/README.md +++ b/deployment/aliyun/README.md @@ -222,7 +222,12 @@ and the shadow symlink. Production unit/env files remain staged under: Candidate installation refuses an unmounted `/data`, an active shadow, a digest mismatch, or a concurrent release operation. It does not start any service and -does not overwrite production configuration or the production symlink. +does not overwrite production configuration or the production symlink. A +pre-existing artifact directory is reusable only when its binary, deployment +assets, artifact URI, bundle digest, bundle URI, and source revision all match +exactly; otherwise installation fails instead of rewriting historical release +evidence. First installation is assembled in a sibling directory and renamed +into place only after all identity checks pass. The committed shadow environments use `SYMBOLS=ALL`, ten-minute segments, the isolated spools below, and isolated OSS datasets: @@ -255,7 +260,8 @@ these are true for the entire candidate run: - every discovered symbol has a ready snapshot and sequence gaps remain zero; - neither session nor catalog membership changes, health never stops advancing for more than 90 seconds, and the persistent upload-failure count is unchanged; -- pending uploads are zero and queue, disk, and upload warnings are false; +- queue, disk, and upload warnings are false, while the persistent upload-failure + count does not increase during normal segment rotations; - CPU accounting and peak memory stay inside the systemd limits; - after stop, the candidate's `--upload-only` drain leaves no partial, temporary, corrupt, compressed, success-marker, or cleanup-marker artifact; @@ -265,14 +271,18 @@ these are true for the entire candidate run: A successful production gate writes: ```text -/data/monday/evidence/shadow-gates///gate.json -/data/monday/evidence/shadow-gates///PASSED.sha256 +/data/monday/evidence/shadow-gates///runs//run.json +/data/monday/evidence/shadow-gates///runs//gate.json +/data/monday/evidence/shadow-gates///runs//PASSED.sha256 ``` -The marker hashes exactly that `gate.json`. Evidence also binds the clean source -revision and deployment-bundle SHA-256, so unit or env changes cannot consume an -older gate for the same binary. A short test override is available -only for script testing; it writes `passed=false` and never creates +Every invocation gets a new append-only run directory; prior gate evidence is +never deleted or replaced. The marker hashes exactly that run's `gate.json`. +Evidence also binds the clean source revision and deployment-bundle SHA-256, so +unit or env changes cannot consume an older gate for the same binary. A second +production gate for an identity that already has a passing run is refused, and +cutover requires exactly one immutable passing run. A short test override is +available only for script testing; it writes `passed=false` and never creates `PASSED.sha256`, so it cannot authorize cutover. ### 3. Cut over or roll back @@ -291,7 +301,10 @@ ARTIFACT_SHA256=REPLACE_WITH_64_HEX_DIGEST \ The host cutover revalidates the binary, release metadata, staged deployment files, gate JSON, marker hash, duration, full-catalog counts, and OSS round trips. Only then does it disable and stop the current production units. After production -is stopped, it installs the target production unit/env files. +is stopped, it installs the target production unit/env files. Deleted legacy +Python instance units must be inactive and disabled before the transition; they +are included in the transition mask so they cannot become a second canonical +writer. The drain is bootstrap-safe: it runs the digest-pinned target binary directly against the canonical production env, so the first upgrade does not depend on @@ -303,12 +316,15 @@ and each process's `/proc//exe` resolving to the requested release; only a verified candidate is enabled for reboot. Any failure after production stops triggers a fail-closed Rust-to-Rust restore of -the previous digest-addressed binary and its staged production assets. Rollback -removes candidate health, starts the old units while disabled, requires health -written after that restart, and verifies full catalog, zero restarts, and the old +the previous digest-addressed binary. Before production stops, its deployment +assets are copied into the unique cutover evidence directory and covered by a +SHA-256 manifest; mutable `/etc` files are never written back into an old +digest-addressed release. Rollback verifies that snapshot before use, removes +candidate health, starts the old units while disabled, requires health written +after that restart, and verifies full catalog, zero restarts, and the old `/proc//exe` targets before enabling. If a safe restore cannot be proved, -both production units remain disabled and masked. Cutover evidence -is written under `/data/monday/evidence/cutovers/`. +both production units remain disabled and masked. Cutover evidence is written +under `/data/monday/evidence/cutovers/`. Rollback uses the same `ACTION=cutover` operation with a previously installed, previously gated artifact digest. There is no Python fallback and no manual @@ -318,8 +334,11 @@ symlink shortcut. After all three OSS objects upload successfully, the Rust collector atomically writes an uploaded-cleanup marker. Restart recovery consumes that marker first, -removes local data/manifest/success artifacts idempotently, fsyncs the directory, -and removes the marker last. An interrupted or invalid cleanup marker makes +derives the only permitted data/manifest/success names from the marker's segment +name, validates all three before deleting any file, removes them idempotently, +fsyncs the directory, and removes the marker last. Cleanup temp files use +exclusive creation and refuse symlinks or other non-regular stale paths. An +interrupted or invalid cleanup marker makes `--upload-only` fail closed. Normal collection and upload-only drain also share an exclusive per-spool process lock, so they cannot mutate one market spool concurrently even if an operator bypasses the systemd transition mask. Recursive @@ -331,3 +350,7 @@ canonical writer, or bypass `PASSED.sha256`. Do not open general SSH for a release. When a Cloud Assistant deadline expires, the local wrapper requests cancellation and waits for a terminal invocation state; host-side `flock` prevents a retry from racing an earlier operation. + +Full-catalog symbol discovery has a 15-second HTTP request timeout, so a stalled +Binance `exchangeInfo` response fails startup instead of leaving an active but +idle service until the systemd runtime limit. diff --git a/deployment/aliyun/binance-lob-archiver-rust-upload@.service b/deployment/aliyun/binance-lob-archiver-rust-upload@.service index 1de3121c0..b06bf47a9 100644 --- a/deployment/aliyun/binance-lob-archiver-rust-upload@.service +++ b/deployment/aliyun/binance-lob-archiver-rust-upload@.service @@ -13,7 +13,7 @@ Environment=RUST_LOG=info Environment=HOME=/var/lib/hft-collector EnvironmentFile=/etc/monday/binance-lob-archiver-rust-%i.env ExecStart=/opt/monday/bin/binance-lob-archiver-shadow --upload-only -TimeoutStartSec=600 +TimeoutStartSec=0 NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict diff --git a/deployment/aliyun/binance-lob-archiver-upload@.service b/deployment/aliyun/binance-lob-archiver-upload@.service index dbe0df553..fe9a8c348 100644 --- a/deployment/aliyun/binance-lob-archiver-upload@.service +++ b/deployment/aliyun/binance-lob-archiver-upload@.service @@ -13,7 +13,7 @@ Environment=RUST_LOG=info Environment=HOME=/var/lib/hft-collector EnvironmentFile=/etc/monday/binance-lob-archiver-production-%i.env ExecStart=/opt/monday/bin/binance-lob-archiver --upload-only -TimeoutStartSec=600 +TimeoutStartSec=0 NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict diff --git a/deployment/aliyun/deploy-rust-lob-release.sh b/deployment/aliyun/deploy-rust-lob-release.sh index 17516e802..1f9c15ea1 100755 --- a/deployment/aliyun/deploy-rust-lob-release.sh +++ b/deployment/aliyun/deploy-rust-lob-release.sh @@ -156,7 +156,14 @@ if systemctl is-active --quiet binance-lob-archiver-rust@spot.service \ fi work_dir=$(mktemp -d) -trap 'rm -rf "$work_dir"' EXIT +release_staging= +cleanup() { + rm -rf "$work_dir" + if [[ -n $release_staging && ( -e $release_staging || -L $release_staging ) ]]; then + rm -rf "$release_staging" + fi +} +trap cleanup EXIT artifact_tmp="$work_dir/binance-lob-archiver" bundle_tmp="$work_dir/deployment.tar" bundle_dir="$work_dir/deployment" @@ -182,8 +189,12 @@ if ! id hftcollector >/dev/null 2>&1; then --shell /usr/sbin/nologin hftcollector fi install -d -m 0755 /opt/monday/bin -install -d -m 0755 "/opt/monday/releases/binance-lob-archiver/$artifact_sha256" -install -d -m 0755 "/opt/monday/releases/binance-lob-archiver/$artifact_sha256/deployment" +release_root=/opt/monday/releases/binance-lob-archiver +release_dir="$release_root/$artifact_sha256" +release_binary="$release_dir/binance-lob-archiver" +release_deployment="$release_dir/deployment" +release_metadata="$release_dir/release.json" +install -d -m 0755 "$release_root" install -d -m 0755 /etc/monday install -d -m 0750 -o hftcollector -g hftcollector \ /data/monday/spool/binance-lob-rust-shadow/spot \ @@ -198,32 +209,78 @@ for path in \ fi done -release_binary="/opt/monday/releases/binance-lob-archiver/$artifact_sha256/binance-lob-archiver" -install -m 0755 "$artifact_tmp" "$release_binary" -printf '%s %s\n' "$artifact_sha256" "$release_binary" | sha256sum --check --strict -"$release_binary" --self-test +if [[ -e $release_dir || -L $release_dir ]]; then + [[ -d $release_dir && ! -L $release_dir && $(readlink -f "$release_dir") == "$release_dir" ]] \ + || { printf 'existing release path is indirect: %s\n' "$release_dir" >&2; exit 1; } + [[ -f $release_metadata && ! -L $release_metadata ]] \ + || { printf 'existing release metadata is not a regular file: %s\n' "$release_metadata" >&2; exit 1; } + jq -e \ + --arg artifact_uri "$artifact_uri" \ + --arg artifact_sha256 "$artifact_sha256" \ + --arg source_revision "$source_revision" \ + --arg bundle_uri "$bundle_uri" \ + --arg bundle_sha256 "$bundle_sha256" \ + '.artifact_uri == $artifact_uri + and .artifact_sha256 == $artifact_sha256 + and .deployment_source_revision == $source_revision + and .deployment_bundle_uri == $bundle_uri + and .deployment_bundle_sha256 == $bundle_sha256' \ + "$release_metadata" >/dev/null \ + || { printf 'existing release identity does not match requested artifact, bundle, and source\n' >&2; exit 1; } + [[ -f $release_binary && ! -L $release_binary && -x $release_binary ]] \ + || { printf 'existing release binary is not a regular executable\n' >&2; exit 1; } + printf '%s %s\n' "$artifact_sha256" "$release_binary" | sha256sum --check --strict + [[ -d $release_deployment && ! -L $release_deployment ]] \ + || { printf 'existing release deployment is not a direct directory\n' >&2; exit 1; } + expected_assets=0 + for source in "$bundle_dir"/*; do + [[ -f $source && ! -L $source ]] \ + || { printf 'deployment bundle contains a non-regular asset: %s\n' "$source" >&2; exit 1; } + asset=${source##*/} + destination="$release_deployment/$asset" + if [[ ! -f $destination || -L $destination ]] || ! cmp -s "$source" "$destination"; then + printf 'existing release deployment differs from the requested bundle: %s\n' "$asset" >&2 + exit 1 + fi + expected_assets=$((expected_assets + 1)) + done + actual_assets=$(find "$release_deployment" -mindepth 1 -maxdepth 1 -print | wc -l) + (( actual_assets == expected_assets )) \ + || { printf 'existing release deployment contains unexpected assets\n' >&2; exit 1; } +else + release_staging=$(mktemp -d "$release_root/.${artifact_sha256}.new.XXXXXX") + install -d -m 0755 "$release_staging/deployment" + install -m 0755 "$artifact_tmp" "$release_staging/binance-lob-archiver" + cp -a "$bundle_dir/." "$release_staging/deployment/" + printf '{"artifact_uri":"%s","artifact_sha256":"%s","deployment_source_revision":"%s","deployment_bundle_uri":"%s","deployment_bundle_sha256":"%s"}\n' \ + "$artifact_uri" "$artifact_sha256" "$source_revision" "$bundle_uri" "$bundle_sha256" \ + > "$release_staging/release.json" + chmod 0644 "$release_staging/release.json" + printf '%s %s\n' "$artifact_sha256" "$release_staging/binance-lob-archiver" \ + | sha256sum --check --strict + chmod 0755 "$release_staging" + mv -T "$release_staging" "$release_dir" + release_staging= +fi + +[[ $(stat -c %a "$release_dir") == 755 ]] \ + || { printf 'release directory must be traversable with mode 0755\n' >&2; exit 1; } +runuser -u hftcollector -- "$release_binary" --self-test "$release_binary" --help | grep -Fq -- '--upload-only' -cp -a "$bundle_dir/." \ - "/opt/monday/releases/binance-lob-archiver/$artifact_sha256/deployment/" -install -m 0644 "$bundle_dir/binance-lob-archiver-rust@.service" \ +install -m 0644 "$release_deployment/binance-lob-archiver-rust@.service" \ /etc/systemd/system/binance-lob-archiver-rust@.service -install -m 0644 "$bundle_dir/binance-lob-archiver-rust-upload@.service" \ +install -m 0644 "$release_deployment/binance-lob-archiver-rust-upload@.service" \ /etc/systemd/system/binance-lob-archiver-rust-upload@.service -install -m 0640 "$bundle_dir/binance-lob-archiver-rust-spot.env" \ +install -m 0640 "$release_deployment/binance-lob-archiver-rust-spot.env" \ /etc/monday/binance-lob-archiver-rust-spot.env -install -m 0640 "$bundle_dir/binance-lob-archiver-rust-usdm.env" \ +install -m 0640 "$release_deployment/binance-lob-archiver-rust-usdm.env" \ /etc/monday/binance-lob-archiver-rust-usdm.env ln -sfn "$release_binary" /opt/monday/bin/binance-lob-archiver-shadow printf '%s %s\n' "$artifact_sha256" /opt/monday/bin/binance-lob-archiver-shadow \ | sha256sum --check --strict -metadata_tmp="/opt/monday/releases/binance-lob-archiver/$artifact_sha256/release.json.tmp" -printf '{"artifact_uri":"%s","artifact_sha256":"%s","deployment_source_revision":"%s","deployment_bundle_uri":"%s","deployment_bundle_sha256":"%s"}\n' \ - "$artifact_uri" "$artifact_sha256" "$source_revision" "$bundle_uri" "$bundle_sha256" \ - > "$metadata_tmp" -mv "$metadata_tmp" "/opt/monday/releases/binance-lob-archiver/$artifact_sha256/release.json" systemctl daemon-reload printf 'installed Rust collector candidate %s from %s; no service was started\n' \ "$artifact_sha256" "$source_revision" diff --git a/deployment/aliyun/host-rust-lob-cutover.sh b/deployment/aliyun/host-rust-lob-cutover.sh index 778a14e49..d90b29a29 100755 --- a/deployment/aliyun/host-rust-lob-cutover.sh +++ b/deployment/aliyun/host-rust-lob-cutover.sh @@ -16,7 +16,7 @@ if [[ $# -ne 1 || ! $1 =~ ^[A-Fa-f0-9]{64}$ ]]; then exit 2 fi -for command in awk chmod date env find flock grep id install jq ln mountpoint mv readlink rm runuser sha256sum sleep stat systemctl tr wc; do +for command in awk chmod cmp date env find flock grep id install jq ln mkdir mountpoint mv readlink rm runuser sha256sum sleep stat systemctl tr wc; do if ! command -v "$command" >/dev/null 2>&1; then printf 'missing required command: %s\n' "$command" >&2 exit 2 @@ -31,6 +31,7 @@ CANDIDATE_DEPLOYMENT="$CANDIDATE_RELEASE/deployment" GATE_POLICY="$CANDIDATE_DEPLOYMENT/rust-lob-shadow-gate-policy.jq" RUNTIME_HEALTH_POLICY="$CANDIDATE_DEPLOYMENT/rust-lob-runtime-health-policy.jq" GATE_ROOT=/data/monday/evidence/shadow-gates +GATE_BUNDLE_DIR= GATE_DIR= GATE_JSON= GATE_MARKER= @@ -52,7 +53,15 @@ UPLOAD_UNITS=( binance-lob-archiver-upload@spot.service binance-lob-archiver-upload@usdm.service ) -TRANSITION_MASK_UNITS=("${PRODUCTION_UNITS[@]}" "${UPLOAD_UNITS[@]}") +LEGACY_UNITS=( + binance-lob-archiver@spot.service + binance-lob-archiver@usdm.service +) +TRANSITION_MASK_UNITS=( + "${PRODUCTION_UNITS[@]}" + "${UPLOAD_UNITS[@]}" + "${LEGACY_UNITS[@]}" +) QUIESCENT_UNITS=( binance-lob-archiver-rust@spot.service binance-lob-archiver-rust@usdm.service @@ -60,6 +69,7 @@ QUIESCENT_UNITS=( binance-lob-archiver-rust-upload@usdm.service binance-lob-archiver-upload@spot.service binance-lob-archiver-upload@usdm.service + "${LEGACY_UNITS[@]}" ) DEPLOYMENT_ASSETS=( binance-lob-archiver-production@.service @@ -106,12 +116,15 @@ for path in /data/monday /data/monday/evidence /data/monday/evidence/cutovers; d exit 1 fi done -install -d -m 0750 "$EVIDENCE_DIR" +install -d -m 0750 /data/monday/evidence/cutovers +mkdir -m 0750 -- "$EVIDENCE_DIR" \ + || { printf 'refusing to reuse cutover evidence directory: %s\n' "$EVIDENCE_DIR" >&2; exit 1; } STEP=preflight RESULT=preflight FAILURE_REASON= ROLLBACK_RESULT=not-needed +ROLLBACK_DEPLOYMENT_MANIFEST_SHA256= OLD_SHA256= OLD_BINARY= OLD_DEPLOYMENT= @@ -120,6 +133,7 @@ TRANSITION_STARTED=0 SUCCESS=0 OLD_SESSION_SPOT= OLD_SESSION_USDM= +CANDIDATE_STARTED_NS=0 fail() { FAILURE_REASON=$* @@ -298,35 +312,52 @@ run_candidate_drain() { } stage_existing_deployment_for_rollback() { - local existing=0 asset source mode + local existing=0 asset source installed_source mode source_kind + local release_deployment=$OLD_DEPLOYMENT + local snapshot="$EVIDENCE_DIR/rollback-deployment" + local manifest="$EVIDENCE_DIR/rollback-deployment.sha256" [[ ! -L $OLD_DEPLOYMENT ]] || fail "old staged deployment is a symlink: $OLD_DEPLOYMENT" for asset in "${DEPLOYMENT_ASSETS[@]}"; do - if [[ -e $OLD_DEPLOYMENT/$asset ]]; then + if [[ -e $release_deployment/$asset ]]; then ((existing += 1)) fi done if (( existing == ${#DEPLOYMENT_ASSETS[@]} )); then - validate_deployment "$OLD_DEPLOYMENT" false - return + validate_deployment "$release_deployment" false + source_kind=release + elif (( existing == 0 )); then + source_kind=installed + else + fail "old release has a partial staged deployment: $release_deployment" fi - (( existing == 0 )) || fail "old release has a partial staged deployment: $OLD_DEPLOYMENT" - install -d -m 0755 "$OLD_DEPLOYMENT" + [[ ! -e $snapshot && ! -L $snapshot ]] \ + || fail "rollback evidence snapshot already exists: $snapshot" + install -d -m 0750 "$snapshot" for asset in "${DEPLOYMENT_ASSETS[@]}"; do case "$asset" in - *.service) - source="/etc/systemd/system/$asset" - mode=0644 - ;; - *.env) - source="/etc/monday/$asset" - mode=0640 - ;; + *.service) installed_source="/etc/systemd/system/$asset"; mode=0644 ;; + *.env) installed_source="/etc/monday/$asset"; mode=0640 ;; esac - secure_regular_file "$source" - atomic_install "$mode" "$source" "$OLD_DEPLOYMENT/$asset" + secure_regular_file "$installed_source" + if [[ $source_kind == release ]]; then + source="$release_deployment/$asset" + secure_regular_file "$source" + cmp -s -- "$source" "$installed_source" \ + || fail "installed production asset drifted from the active immutable release: $installed_source" + else + source=$installed_source + fi + atomic_install "$mode" "$source" "$snapshot/$asset" done - validate_deployment "$OLD_DEPLOYMENT" false + validate_deployment "$snapshot" false + ( + cd "$snapshot" + sha256sum "${DEPLOYMENT_ASSETS[@]}" + ) >"$manifest" + chmod 0640 "$manifest" + ROLLBACK_DEPLOYMENT_MANIFEST_SHA256=$(sha256sum "$manifest" | awk '{print $1}') + OLD_DEPLOYMENT=$snapshot } unit_active_json() { @@ -349,10 +380,18 @@ copy_health_evidence() { } health_ready_for_release() { - local market=$1 minimum_symbols=$2 old_session=$3 minimum_updated_ns=${4:-0} health + local market=$1 minimum_symbols=$2 old_session=$3 minimum_updated_ns=${4:-0} + local health expected_dataset health="$CANONICAL_SPOOL/$market/health.json" [[ -f $health && ! -L $health ]] || return 1 + case "$market" in + spot) expected_dataset=spot_all ;; + usdm) expected_dataset=usdm_perpetual_all ;; + *) return 1 ;; + esac jq -e \ + --arg expected_market "$market" \ + --arg expected_dataset "$expected_dataset" \ --arg old_session "$old_session" \ --argjson minimum_symbols "$minimum_symbols" \ --argjson minimum_updated_ns "$minimum_updated_ns" \ @@ -405,11 +444,8 @@ clear_health_before_restart() { production_is_fail_closed() { local unit state - for unit in "${PRODUCTION_UNITS[@]}"; do - systemctl is-active --quiet "$unit" && return 1 - systemctl is-enabled --quiet "$unit" && return 1 - done for unit in "${TRANSITION_MASK_UNITS[@]}"; do + systemctl is-active --quiet "$unit" && return 1 state=$(systemctl is-enabled "$unit" 2>/dev/null || true) [[ $state == masked || $state == masked-runtime ]] || return 1 done @@ -428,6 +464,7 @@ write_evidence() { --arg step "$STEP" \ --arg failure_reason "$FAILURE_REASON" \ --arg rollback_result "$ROLLBACK_RESULT" \ + --arg rollback_deployment_manifest_sha256 "$ROLLBACK_DEPLOYMENT_MANIFEST_SHA256" \ --arg candidate_sha256 "$CANDIDATE_SHA256" \ --arg deployment_bundle_sha256 "$DEPLOYMENT_BUNDLE_SHA256" \ --arg previous_sha256 "$OLD_SHA256" \ @@ -442,6 +479,9 @@ write_evidence() { last_step: $step, failure_reason: (if $failure_reason == "" then null else $failure_reason end), rollback_result: $rollback_result, + rollback_deployment_manifest_sha256: + (if $rollback_deployment_manifest_sha256 == "" then null + else $rollback_deployment_manifest_sha256 end), candidate_sha256: $candidate_sha256, deployment_bundle_sha256: (if $deployment_bundle_sha256 == "" then null else $deployment_bundle_sha256 end), previous_sha256: (if $previous_sha256 == "" then null else $previous_sha256 end), @@ -480,7 +520,20 @@ rollback_after_failure() { fi if [[ $OLD_MODE == upgrade ]]; then - if ! install_deployment "$OLD_DEPLOYMENT"; then + if [[ -n $ROLLBACK_DEPLOYMENT_MANIFEST_SHA256 ]]; then + printf '%s %s\n' "$ROLLBACK_DEPLOYMENT_MANIFEST_SHA256" \ + "$EVIDENCE_DIR/rollback-deployment.sha256" | sha256sum --check --strict \ + || safe_to_restart=0 + ( + cd "$OLD_DEPLOYMENT" + sha256sum --check --strict "$EVIDENCE_DIR/rollback-deployment.sha256" + ) || safe_to_restart=0 + else + safe_to_restart=0 + fi + if (( safe_to_restart == 0 )); then + ROLLBACK_RESULT=rollback-evidence-unverified-disabled + elif ! install_deployment "$OLD_DEPLOYMENT"; then safe_to_restart=0 ROLLBACK_RESULT=restore-assets-failed-disabled elif ! atomic_symlink "$OLD_BINARY" "$PRODUCTION_LINK"; then @@ -535,7 +588,11 @@ rollback_after_failure() { if [[ $(readlink -f "$PRODUCTION_LINK" 2>/dev/null || true) == "$CANDIDATE_BINARY" ]]; then rm -f "$PRODUCTION_LINK" fi - ROLLBACK_RESULT=new-host-disabled + if production_is_fail_closed; then + ROLLBACK_RESULT=new-host-disabled + else + ROLLBACK_RESULT=new-host-containment-failed + fi fi copy_health_evidence rollback } @@ -590,9 +647,7 @@ jq -e --arg sha "$CANDIDATE_SHA256" --arg bundle "$DEPLOYMENT_BUNDLE_SHA256" \ '.artifact_sha256 == $sha and .deployment_bundle_sha256 == $bundle' \ "$CANDIDATE_RELEASE/release.json" >/dev/null \ || fail 'candidate release metadata does not match the requested identity' -GATE_DIR="$GATE_ROOT/$CANDIDATE_SHA256/$DEPLOYMENT_BUNDLE_SHA256" -GATE_JSON="$GATE_DIR/gate.json" -GATE_MARKER="$GATE_DIR/PASSED.sha256" +GATE_BUNDLE_DIR="$GATE_ROOT/$CANDIDATE_SHA256/$DEPLOYMENT_BUNDLE_SHA256" validate_deployment "$CANDIDATE_DEPLOYMENT" true id hftcollector >/dev/null 2>&1 || fail 'service account hftcollector is missing' runuser -u hftcollector -- "$CANDIDATE_BINARY" --self-test @@ -601,9 +656,20 @@ runuser -u hftcollector -- "$CANDIDATE_BINARY" --self-test || fail 'shadow symlink does not point to the gated candidate binary' STEP=validate-shadow-gate -for path in "$GATE_ROOT" "$GATE_ROOT/$CANDIDATE_SHA256" "$GATE_DIR"; do +for path in "$GATE_ROOT" "$GATE_ROOT/$CANDIDATE_SHA256" "$GATE_BUNDLE_DIR" \ + "$GATE_BUNDLE_DIR/runs"; do path_is_direct_or_absent "$path" || fail "shadow gate path contains a symlink: $path" done +shopt -s nullglob +gate_markers=("$GATE_BUNDLE_DIR"/runs/*/PASSED.sha256) +shopt -u nullglob +(( ${#gate_markers[@]} == 1 )) \ + || fail "expected exactly one immutable passed shadow gate, found ${#gate_markers[@]}" +GATE_MARKER=${gate_markers[0]} +GATE_DIR=${GATE_MARKER%/*} +GATE_JSON="$GATE_DIR/gate.json" +path_is_direct_or_absent "$GATE_DIR" \ + || fail "shadow gate run path contains a symlink: $GATE_DIR" secure_regular_file "$GATE_JSON" secure_regular_file "$GATE_MARKER" [[ $(wc -l < "$GATE_MARKER") -eq 1 ]] || fail 'PASSED.sha256 must contain exactly one entry' @@ -626,6 +692,10 @@ canonical_spool_paths_safe || fail 'canonical spool path contains a symlink or e for unit in "${QUIESCENT_UNITS[@]}"; do systemctl is-active --quiet "$unit" && fail "unit must be inactive before cutover: $unit" done +for unit in "${LEGACY_UNITS[@]}"; do + systemctl is-enabled --quiet "$unit" \ + && fail "legacy collector unit must be disabled before cutover: $unit" +done active_count=0 enabled_count=0 @@ -660,6 +730,11 @@ fi STEP=stop-production TRANSITION_STARTED=1 +systemctl disable --now "${LEGACY_UNITS[@]}" >/dev/null 2>&1 || true +for unit in "${LEGACY_UNITS[@]}"; do + systemctl is-active --quiet "$unit" && fail "legacy collector unit did not stop: $unit" + systemctl is-enabled --quiet "$unit" && fail "legacy collector unit remained enabled: $unit" +done if [[ $OLD_MODE == upgrade ]]; then systemctl disable --now "${PRODUCTION_UNITS[@]}" else @@ -690,13 +765,20 @@ STEP=switch-production-symlink atomic_symlink "$CANDIDATE_BINARY" "$PRODUCTION_LINK" printf '%s %s\n' "$CANDIDATE_SHA256" "$PRODUCTION_LINK" | sha256sum --check --strict +STEP=clear-stale-candidate-health +copy_health_evidence previous-production +clear_health_before_restart \ + || fail 'could not clear stale production health before starting the candidate' +CANDIDATE_STARTED_NS=$(date +%s%N) + STEP=start-candidate-production systemctl reset-failed "${PRODUCTION_UNITS[@]}" >/dev/null 2>&1 || true systemctl unmask --runtime "${PRODUCTION_UNITS[@]}" >/dev/null systemctl start "${PRODUCTION_UNITS[@]}" STEP=verify-candidate-production -wait_for_release_health "$CANDIDATE_BINARY" "$OLD_SESSION_SPOT" "$OLD_SESSION_USDM" \ +wait_for_release_health \ + "$CANDIDATE_BINARY" "$OLD_SESSION_SPOT" "$OLD_SESSION_USDM" "$CANDIDATE_STARTED_NS" \ || fail 'candidate production did not reach verified full-catalog health' copy_health_evidence production @@ -704,9 +786,9 @@ STEP=enable-verified-candidate systemctl enable "${PRODUCTION_UNITS[@]}" >/dev/null runtime_matches_release "$CANDIDATE_BINARY" true \ || fail 'candidate runtime identity changed while enabling production' -health_ready_for_release spot 1000 "$OLD_SESSION_SPOT" \ +health_ready_for_release spot 1000 "$OLD_SESSION_SPOT" "$CANDIDATE_STARTED_NS" \ || fail 'Spot health changed while enabling production' -health_ready_for_release usdm 400 "$OLD_SESSION_USDM" \ +health_ready_for_release usdm 400 "$OLD_SESSION_USDM" "$CANDIDATE_STARTED_NS" \ || fail 'USD-M health changed while enabling production' systemctl unmask --runtime "${UPLOAD_UNITS[@]}" >/dev/null diff --git a/deployment/aliyun/host-rust-lob-shadow-gate.sh b/deployment/aliyun/host-rust-lob-shadow-gate.sh index 9fceb6178..1c8eb88cc 100755 --- a/deployment/aliyun/host-rust-lob-shadow-gate.sh +++ b/deployment/aliyun/host-rust-lob-shadow-gate.sh @@ -35,7 +35,7 @@ usage() { exit 2 } -for command in aliyun awk chmod chown cmp date dirname find flock grep id install jq mktemp \ +for command in aliyun awk chmod chown cmp date dirname find flock grep id install jq mkdir mktemp \ mountpoint mv readlink rm runuser sed sha256sum sleep sort stat systemctl tr wc; do command -v "$command" >/dev/null 2>&1 || die "missing required command: $command" done @@ -187,7 +187,10 @@ min_symbols[spot]=1000 min_symbols[usdm]=400 binary_evidence_dir="$EVIDENCE_ROOT/$candidate_sha" -evidence_dir="$binary_evidence_dir/$deployment_bundle_sha256" +bundle_evidence_dir="$binary_evidence_dir/$deployment_bundle_sha256" +runs_dir="$bundle_evidence_dir/runs" +gate_run_id="$(date -u +%Y%m%dT%H%M%SZ)-$$" +evidence_dir="$runs_dir/$gate_run_id" gate_json="$evidence_dir/gate.json" passed_marker="$evidence_dir/PASSED.sha256" gate_tmp="$evidence_dir/.gate.json.tmp" @@ -203,9 +206,37 @@ done install -d -m 0750 "$binary_evidence_dir" direct_directory "$binary_evidence_dir" \ || die 'binary evidence directory is indirect or a symlink' -install -d -m 0750 "$evidence_dir" -direct_directory "$evidence_dir" || die 'bundle evidence directory is indirect or a symlink' -rm -f "$gate_json" "$passed_marker" "$gate_tmp" "$marker_tmp" +install -d -m 0750 "$bundle_evidence_dir" "$runs_dir" +direct_directory "$bundle_evidence_dir" \ + || die 'bundle evidence directory is indirect or a symlink' +direct_directory "$runs_dir" || die 'gate runs directory is indirect or a symlink' +if [[ $test_only != true ]]; then + shopt -s nullglob + existing_passes=("$runs_dir"/*/PASSED.sha256) + shopt -u nullglob + (( ${#existing_passes[@]} == 0 )) \ + || die 'an immutable production-eligible gate already exists for this release identity' +fi +mkdir -m 0750 -- "$evidence_dir" \ + || die 'gate run evidence directory already exists or could not be created atomically' +direct_directory "$evidence_dir" || die 'gate run evidence directory is indirect or a symlink' +run_created_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +jq -n \ + --arg schema monday.rust_lob_shadow_gate_run.v1 \ + --arg run_id "$gate_run_id" \ + --arg created_at "$run_created_at" \ + --arg candidate_sha256 "$candidate_sha" \ + --arg deployment_bundle_sha256 "$deployment_bundle_sha256" \ + --arg deployment_source_revision "$deployment_source_revision" \ + --argjson requested_duration_seconds "$gate_seconds" \ + --argjson test_only "$test_only" \ + '{schema:$schema,run_id:$run_id,created_at:$created_at, + candidate_sha256:$candidate_sha256, + deployment_bundle_sha256:$deployment_bundle_sha256, + deployment_source_revision:$deployment_source_revision, + requested_duration_seconds:$requested_duration_seconds,test_only:$test_only}' \ + >"$evidence_dir/run.json" +chmod 0640 "$evidence_dir/run.json" tmp_dir=$(mktemp -d) chown "$SERVICE_USER:$SERVICE_USER" "$tmp_dir" @@ -216,7 +247,6 @@ cleanup() { rm -rf "$tmp_dir" rm -f "$gate_tmp" "$marker_tmp" if [[ $gate_finished != true ]]; then - rm -f "$passed_marker" systemctl stop "${unit[spot]}" "${unit[usdm]}" >/dev/null 2>&1 || true fi exit "$status" @@ -234,7 +264,7 @@ assert_candidate() { assert_spool_drained() { local market=$1 local remaining - remaining=$(find "${spool_dir[$market]}" -type f \( \ + remaining=$(find "${spool_dir[$market]}" \( -type f -o -type l \) \( \ -name '*.manifest.json' -o -name '*.jsonl.part' -o \ -name '*.zst.tmp' -o -name '*.part.corrupt' -o \ -name '*.jsonl.zst' -o -name '*._SUCCESS' -o \ @@ -362,9 +392,12 @@ health_passes() { and .updated_at_ns >= $gate_started_ns and .status == "synced" and .sequence_gaps == 0 + and (.symbol_count | type) == "number" + and .symbol_count == (.symbol_count | floor) and .symbol_count >= $minimum_symbols + and (.snapshot_ready_count | type) == "number" + and .snapshot_ready_count == (.snapshot_ready_count | floor) and .snapshot_ready_count == .symbol_count - and .pending_upload_segments == 0 and .queue_saturated == false and .disk_warning == false and .upload_warning == false @@ -707,6 +740,7 @@ jq -n \ test_only:$test_only,checks_passed:$checks_passed, production_eligible:$production_eligible,passed:$passed,markets:$markets}' \ >"$gate_tmp" +[[ ! -e $gate_json && ! -L $gate_json ]] || die 'gate evidence path already exists' install -m 0640 "$gate_tmp" "$gate_json" rm -f "$gate_tmp" @@ -714,6 +748,8 @@ if [[ $production_eligible == true ]]; then gate_sha=$(sha256sum "$gate_json" | awk '{print $1}') printf '%s gate.json\n' "$gate_sha" >"$marker_tmp" chmod 0640 "$marker_tmp" + [[ ! -e $passed_marker && ! -L $passed_marker ]] \ + || die 'gate pass marker already exists' mv "$marker_tmp" "$passed_marker" printf 'production shadow gate passed: %s\nmarker: %s\n' "$gate_json" "$passed_marker" else diff --git a/deployment/aliyun/invoke-rust-lob-operation.sh b/deployment/aliyun/invoke-rust-lob-operation.sh index ff208eef9..cd79a5dbd 100755 --- a/deployment/aliyun/invoke-rust-lob-operation.sh +++ b/deployment/aliyun/invoke-rust-lob-operation.sh @@ -8,7 +8,7 @@ usage() { 'The command always targets ap-northeast-1 and uses Alibaba Cloud Assistant.' } -for command in aliyun base64 jq; do +for command in aliyun base64 jq seq sleep tr; do if ! command -v "$command" >/dev/null 2>&1; then printf 'missing required command: %s\n' "$command" >&2 exit 2 diff --git a/deployment/aliyun/rust-lob-runtime-health-policy.jq b/deployment/aliyun/rust-lob-runtime-health-policy.jq index bfb50e0a7..9889f7cb8 100644 --- a/deployment/aliyun/rust-lob-runtime-health-policy.jq +++ b/deployment/aliyun/rust-lob-runtime-health-policy.jq @@ -1,6 +1,12 @@ -.status == "synced" +.market == $expected_market +and .dataset == $expected_dataset +and .status == "synced" and .sequence_gaps == 0 +and (.symbol_count | type) == "number" +and .symbol_count == (.symbol_count | floor) and .symbol_count >= $minimum_symbols +and (.snapshot_ready_count | type) == "number" +and .snapshot_ready_count == (.snapshot_ready_count | floor) and .snapshot_ready_count == .symbol_count and .pending_upload_segments == 0 and .queue_saturated == false diff --git a/deployment/aliyun/test-rust-lob-control-plane.sh b/deployment/aliyun/test-rust-lob-control-plane.sh index ffdc9e8bc..529a76ef6 100755 --- a/deployment/aliyun/test-rust-lob-control-plane.sh +++ b/deployment/aliyun/test-rust-lob-control-plane.sh @@ -1,16 +1,19 @@ #!/usr/bin/env bash -# shellcheck disable=SC2016 +# Dynamically sourced production functions consume fixture globals and mocks. +# shellcheck disable=SC2016,SC2034,SC2329 set -euo pipefail SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd) CUTOVER="$SCRIPT_DIR/host-rust-lob-cutover.sh" +GATE="$SCRIPT_DIR/host-rust-lob-shadow-gate.sh" +INSTALL_RELEASE="$SCRIPT_DIR/deploy-rust-lob-release.sh" INVOKE="$SCRIPT_DIR/invoke-rust-lob-operation.sh" POLICY="$SCRIPT_DIR/rust-lob-shadow-gate-policy.jq" RUNTIME_POLICY="$SCRIPT_DIR/rust-lob-runtime-health-policy.jq" # shellcheck disable=SC1091 . "$SCRIPT_DIR/rust-lob-control-plane-lib.sh" -for command in awk base64 cut grep jq mktemp sed seq; do +for command in awk base64 cmp cut grep install jq mktemp sed seq sha256sum; do command -v "$command" >/dev/null 2>&1 || { printf 'missing test dependency: %s\n' "$command" >&2 exit 2 @@ -123,34 +126,61 @@ if jq -e \ exit 1 fi -jq -n '{status:"synced",sequence_gaps:0,symbol_count:1200, +jq -n '{market:"spot",dataset:"spot_all",status:"synced",sequence_gaps:0,symbol_count:1200, snapshot_ready_count:1200,pending_upload_segments:0,queue_saturated:false, disk_warning:false,upload_warning:false,updated_at_ns:200,session_id:"new-session"}' \ >"$tmp_dir/runtime-health.json" -jq -e \ - --arg old_session old-session \ - --argjson minimum_symbols 1000 \ - --argjson minimum_updated_ns 100 \ - -f "$RUNTIME_POLICY" "$tmp_dir/runtime-health.json" >/dev/null -if jq -e \ - --arg old_session old-session \ - --argjson minimum_symbols 1000 \ - --argjson minimum_updated_ns 200 \ - -f "$RUNTIME_POLICY" "$tmp_dir/runtime-health.json" >/dev/null; then +runtime_policy_accepts() { + local health=$1 old_session=$2 minimum_updated_ns=$3 + local expected_market=${4:-spot} expected_dataset=${5:-spot_all} + jq -e \ + --arg expected_market "$expected_market" \ + --arg expected_dataset "$expected_dataset" \ + --arg old_session "$old_session" \ + --argjson minimum_symbols 1000 \ + --argjson minimum_updated_ns "$minimum_updated_ns" \ + -f "$RUNTIME_POLICY" "$health" >/dev/null +} +runtime_policy_accepts "$tmp_dir/runtime-health.json" old-session 100 +if runtime_policy_accepts "$tmp_dir/runtime-health.json" old-session 200; then printf 'runtime policy accepted health that was not newer than restart\n' >&2 exit 1 fi -if jq -e \ - --arg old_session new-session \ - --argjson minimum_symbols 1000 \ - --argjson minimum_updated_ns 100 \ - -f "$RUNTIME_POLICY" "$tmp_dir/runtime-health.json" >/dev/null; then +if runtime_policy_accepts "$tmp_dir/runtime-health.json" new-session 100; then printf 'runtime policy accepted a stale session\n' >&2 exit 1 fi +for field in symbol_count snapshot_ready_count; do + jq --arg field "$field" '.[$field] = "1200"' \ + "$tmp_dir/runtime-health.json" >"$tmp_dir/quoted-count.json" + if runtime_policy_accepts "$tmp_dir/quoted-count.json" old-session 100; then + printf 'runtime policy accepted quoted %s\n' "$field" >&2 + exit 1 + fi + jq --arg field "$field" '.[$field] = 1200.5' \ + "$tmp_dir/runtime-health.json" >"$tmp_dir/fractional-count.json" + if runtime_policy_accepts "$tmp_dir/fractional-count.json" old-session 100; then + printf 'runtime policy accepted fractional %s\n' "$field" >&2 + exit 1 + fi +done +jq '.market = "usdm"' "$tmp_dir/runtime-health.json" >"$tmp_dir/cross-market.json" +if runtime_policy_accepts "$tmp_dir/cross-market.json" old-session 100; then + printf 'runtime policy accepted a cross-market health payload\n' >&2 + exit 1 +fi +jq '.dataset = "usdm_perpetual_all"' \ + "$tmp_dir/runtime-health.json" >"$tmp_dir/cross-dataset.json" +if runtime_policy_accepts "$tmp_dir/cross-dataset.json" old-session 100; then + printf 'runtime policy accepted a cross-dataset health payload\n' >&2 + exit 1 +fi rollback_body="$tmp_dir/rollback.sh" sed -n '/^rollback_after_failure()/,/^}/p' "$CUTOVER" >"$rollback_body" +production_predicate_body="$tmp_dir/production-is-fail-closed.sh" +sed -n '/^production_is_fail_closed()/,/^}/p' "$CUTOVER" \ + >"$production_predicate_body" start_line=$(grep -n 'systemctl start "${PRODUCTION_UNITS\[@\]}"' "$rollback_body" | tail -1 | cut -d: -f1) clear_line=$(grep -n 'clear_health_before_restart' "$rollback_body" | cut -d: -f1) health_line=$(grep -n 'wait_for_release_health' "$rollback_body" | cut -d: -f1) @@ -163,6 +193,164 @@ grep -Fq 'runtime_matches_release "$OLD_BINARY" true' "$rollback_body" grep -Fq '"$rollback_started_ns"' "$rollback_body" grep -Fq 'previous-release-health-unverified-disabled' "$rollback_body" grep -Fq 'systemctl mask --runtime "${TRANSITION_MASK_UNITS[@]}"' "$rollback_body" +grep -Fq 'ROLLBACK_RESULT=new-host-containment-failed' "$rollback_body" +grep -Fq 'binance-lob-archiver@spot.service' "$CUTOVER" +grep -Fq 'binance-lob-archiver@usdm.service' "$CUTOVER" +grep -Fq 'legacy collector unit must be disabled before cutover' "$CUTOVER" +grep -Fq 'release_staging=$(mktemp -d "$release_root/.${artifact_sha256}.new.XXXXXX")' \ + "$INSTALL_RELEASE" +grep -Fq 'chmod 0755 "$release_staging"' "$INSTALL_RELEASE" +grep -Fq 'release directory must be traversable with mode 0755' "$INSTALL_RELEASE" +grep -Fq 'runuser -u hftcollector -- "$release_binary" --self-test' "$INSTALL_RELEASE" +grep -Fq 'existing release identity does not match requested artifact, bundle, and source' \ + "$INSTALL_RELEASE" +grep -Fq 'existing release deployment differs from the requested bundle' "$INSTALL_RELEASE" +grep -Fq 'bundle_evidence_dir="$binary_evidence_dir/$deployment_bundle_sha256"' "$GATE" +grep -Fq 'evidence_dir="$runs_dir/$gate_run_id"' "$GATE" +grep -Fq 'an immutable production-eligible gate already exists' "$GATE" +if grep -Fq 'rm -f "$gate_json"' "$GATE"; then + printf 'shadow gate still deletes immutable gate evidence\n' >&2 + exit 1 +fi +grep -Fq 'gate_markers=("$GATE_BUNDLE_DIR"/runs/*/PASSED.sha256)' "$CUTOVER" +grep -Fq 'rollback-deployment.sha256' "$CUTOVER" +grep -Fq 'ROLLBACK_DEPLOYMENT_MANIFEST_SHA256' "$rollback_body" +grep -Fq 'installed production asset drifted from the active immutable release' "$CUTOVER" +grep -Fq 'cmp -s -- "$source" "$installed_source"' "$CUTOVER" +grep -Fq 'mkdir -m 0750 -- "$EVIDENCE_DIR"' "$CUTOVER" +grep -Fq 'mkdir -m 0750 -- "$evidence_dir"' "$GATE" +grep -Fq '\( -type f -o -type l \)' "$GATE" +grep -Fxq 'TimeoutStartSec=0' "$SCRIPT_DIR/binance-lob-archiver-upload@.service" +grep -Fxq 'TimeoutStartSec=0' "$SCRIPT_DIR/binance-lob-archiver-rust-upload@.service" + +candidate_start_body="$tmp_dir/candidate-start.sh" +sed -n '/^STEP=clear-stale-candidate-health/,/^STEP=write-cutover-evidence/p' \ + "$CUTOVER" >"$candidate_start_body" +candidate_clear_line=$(grep -n '^clear_health_before_restart' "$candidate_start_body" | cut -d: -f1) +candidate_timestamp_line=$(grep -n '^CANDIDATE_STARTED_NS=' "$candidate_start_body" | cut -d: -f1) +candidate_start_line=$(grep -n 'systemctl start "${PRODUCTION_UNITS\[@\]}"' \ + "$candidate_start_body" | cut -d: -f1) +candidate_health_line=$(grep -n '^wait_for_release_health' "$candidate_start_body" | cut -d: -f1) +candidate_enable_line=$(grep -n 'systemctl enable "${PRODUCTION_UNITS\[@\]}"' \ + "$candidate_start_body" | cut -d: -f1) +((candidate_clear_line < candidate_timestamp_line \ + && candidate_timestamp_line < candidate_start_line \ + && candidate_start_line < candidate_health_line \ + && candidate_health_line < candidate_enable_line)) || { + printf 'candidate no longer follows clear stale health -> timestamp -> start -> verify -> enable\n' >&2 + exit 1 +} +grep -Fq '"$CANDIDATE_STARTED_NS"' "$candidate_start_body" + +# Execute the rollback snapshot logic against isolated fixture roots. This catches +# content drift and manifest-tamper regressions that static contract greps miss. +installed_root="$tmp_dir/installed" +release_deployment="$tmp_dir/old-release/deployment" +stage_body="$tmp_dir/stage-existing-deployment.sh" +mkdir -p "$installed_root/systemd" "$installed_root/monday" "$release_deployment" +sed -n '/^stage_existing_deployment_for_rollback()/,/^}/p' "$CUTOVER" \ + | sed \ + -e "s#/etc/systemd/system#$installed_root/systemd#g" \ + -e "s#/etc/monday#$installed_root/monday#g" \ + >"$stage_body" +deployment_assets=( + binance-lob-archiver-production@.service + binance-lob-archiver-upload@.service + binance-lob-archiver-production-spot.env + binance-lob-archiver-production-usdm.env +) +for asset in "${deployment_assets[@]}"; do + case "$asset" in + *.service) installed="$installed_root/systemd/$asset" ;; + *.env) installed="$installed_root/monday/$asset" ;; + esac + printf 'fixture:%s\n' "$asset" >"$release_deployment/$asset" + install -m 0644 "$release_deployment/$asset" "$installed" +done + +run_stage_fixture() ( + DEPLOYMENT_ASSETS=("${deployment_assets[@]}") + OLD_DEPLOYMENT="$release_deployment" + EVIDENCE_DIR=$1 + ROLLBACK_DEPLOYMENT_MANIFEST_SHA256= + fail() { printf '%s\n' "$*" >&2; exit 1; } + validate_deployment() { return 0; } + secure_regular_file() { [[ -f $1 && ! -L $1 ]]; } + atomic_install() { install -m "$1" "$2" "$3"; } + # shellcheck disable=SC1090 + . "$stage_body" + stage_existing_deployment_for_rollback +) + +snapshot_evidence="$tmp_dir/snapshot-evidence" +mkdir -p "$snapshot_evidence" +run_stage_fixture "$snapshot_evidence" +( + cd "$snapshot_evidence/rollback-deployment" + sha256sum --check --strict "$snapshot_evidence/rollback-deployment.sha256" >/dev/null +) +printf 'tampered\n' >> \ + "$snapshot_evidence/rollback-deployment/binance-lob-archiver-production@.service" +if ( + cd "$snapshot_evidence/rollback-deployment" + sha256sum --check --strict "$snapshot_evidence/rollback-deployment.sha256" >/dev/null 2>&1 +); then + printf 'rollback manifest accepted a tampered snapshot\n' >&2 + exit 1 +fi + +printf 'drifted\n' >>"$installed_root/monday/binance-lob-archiver-production-spot.env" +drift_evidence="$tmp_dir/drift-evidence" +mkdir -p "$drift_evidence" +if run_stage_fixture "$drift_evidence" >"$tmp_dir/drift.out" 2>&1; then + printf 'rollback snapshot accepted installed configuration drift\n' >&2 + exit 1 +fi +grep -Fq 'installed production asset drifted from the active immutable release' \ + "$tmp_dir/drift.out" + +run_new_host_rollback_fixture() ( + local active_unit=${1:-} unit + PRODUCTION_UNITS=(production-spot production-usdm) + UPLOAD_UNITS=(upload-spot upload-usdm) + LEGACY_UNITS=(legacy-spot legacy-usdm) + TRANSITION_MASK_UNITS=("${PRODUCTION_UNITS[@]}" "${UPLOAD_UNITS[@]}" "${LEGACY_UNITS[@]}") + CANONICAL_SPOOL="$tmp_dir/nonexistent-spool" + CANDIDATE_DEPLOYMENT="$tmp_dir/candidate-deployment" + CANDIDATE_BINARY="$tmp_dir/candidate-binary" + PRODUCTION_LINK="$tmp_dir/nonexistent-production-link" + OLD_MODE=new-host + ROLLBACK_RESULT= + systemctl() { + case "$1" in + is-active) + unit=${!#} + [[ -n $active_unit && $unit == "$active_unit" ]] + ;; + is-enabled) + unit=${!#} + if [[ ${2:-} == --quiet ]]; then + return 1 + fi + printf 'masked-runtime\n' + return 1 + ;; + *) return 0 ;; + esac + } + copy_health_evidence() { return 0; } + run_candidate_drain() { return 0; } + # shellcheck disable=SC1090 + . "$production_predicate_body" + # shellcheck disable=SC1090 + . "$rollback_body" + rollback_after_failure + printf '%s\n' "$ROLLBACK_RESULT" +) + +[[ $(run_new_host_rollback_fixture) == new-host-disabled ]] +[[ $(run_new_host_rollback_fixture legacy-spot) == new-host-containment-failed ]] +[[ $(run_new_host_rollback_fixture upload-usdm) == new-host-containment-failed ]] mock_bin="$tmp_dir/bin" mock_state="$tmp_dir/mock-state" diff --git a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs index 20bcb09de..6fe14f4c0 100644 --- a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs +++ b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs @@ -1037,11 +1037,21 @@ fn snapshot_retry_delay(retry_after: Option<&str>, attempt: usize) -> Duration { } async fn discover_symbols(market: Market, rest_base: &str) -> anyhow::Result { + discover_symbols_with_timeout(market, rest_base, Duration::from_secs(15)).await +} + +async fn discover_symbols_with_timeout( + market: Market, + rest_base: &str, + timeout: Duration, +) -> anyhow::Result { let path = match market { Market::Spot => "/api/v3/exchangeInfo", Market::Usdm => "/fapi/v1/exchangeInfo", }; - let payload: Value = reqwest::Client::new() + let payload: Value = reqwest::Client::builder() + .timeout(timeout) + .build()? .get(format!("{rest_base}{path}")) .send() .await? @@ -1215,6 +1225,16 @@ fn write_uploaded_cleanup_marker( let manifest_name = local_file_name(manifest)?; let marker = manifest.with_file_name(format!("{manifest_name}{UPLOADED_CLEANUP_SUFFIX}")); let temporary = marker.with_file_name(format!("{}.tmp", local_file_name(&marker)?)); + match std::fs::symlink_metadata(&marker) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => anyhow::bail!( + "uploaded cleanup marker already exists: {}", + marker.display() + ), + Err(error) => { + return Err(error).with_context(|| format!("failed to inspect {}", marker.display())) + } + } let mut bytes = serde_json::to_vec(&json!({ "schema": UPLOADED_CLEANUP_SCHEMA, "data": local_file_name(data)?, @@ -1222,11 +1242,7 @@ fn write_uploaded_cleanup_marker( "success": local_file_name(success)?, }))?; bytes.push(b'\n'); - let mut output = std::fs::OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&temporary)?; + let mut output = create_cleanup_temporary(&temporary)?; std::io::Write::write_all(&mut output, &bytes)?; output.sync_all()?; std::fs::rename(&temporary, &marker)?; @@ -1234,6 +1250,34 @@ fn write_uploaded_cleanup_marker( Ok(marker) } +fn create_cleanup_temporary(path: &Path) -> anyhow::Result { + for attempt in 0..2 { + match std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(path) + { + Ok(file) => return Ok(file), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists && attempt == 0 => { + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("failed to inspect stale cleanup temp {}", path.display()) + })?; + if !metadata.file_type().is_file() { + anyhow::bail!("refusing non-regular cleanup temp path: {}", path.display()); + } + std::fs::remove_file(path).with_context(|| { + format!("failed to remove stale cleanup temp {}", path.display()) + })?; + } + Err(error) => { + return Err(error) + .with_context(|| format!("failed to create cleanup temp {}", path.display())); + } + } + } + unreachable!("cleanup temp creation retries are bounded") +} + fn recover_uploaded_cleanups(spool_dir: &Path) -> anyhow::Result { let markers = files_with_suffix(spool_dir, UPLOADED_CLEANUP_SUFFIX)?; for marker in &markers { @@ -1251,8 +1295,8 @@ fn cleanup_uploaded_marker(marker: &Path) -> anyhow::Result<()> { marker.display() ); } - for field in ["data", "manifest", "success"] { - let path = cleanup_artifact_path(marker, &metadata, field)?; + let paths = cleanup_artifact_paths(marker, &metadata)?; + for path in paths { match std::fs::remove_file(&path) { Ok(()) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} @@ -1268,21 +1312,47 @@ fn cleanup_uploaded_marker(marker: &Path) -> anyhow::Result<()> { sync_parent_directory(marker) } -fn cleanup_artifact_path(marker: &Path, metadata: &Value, field: &str) -> anyhow::Result { - let name = metadata[field] - .as_str() - .ok_or_else(|| anyhow::anyhow!("uploaded cleanup marker missing {field}"))?; - if Path::new(name).file_name().and_then(|value| value.to_str()) != Some(name) { - anyhow::bail!("uploaded cleanup marker contains invalid {field} file name"); +fn cleanup_artifact_paths(marker: &Path, metadata: &Value) -> anyhow::Result<[PathBuf; 3]> { + let marker_name = local_file_name(marker)?; + let manifest_name = marker_name + .strip_suffix(UPLOADED_CLEANUP_SUFFIX) + .ok_or_else(|| anyhow::anyhow!("invalid uploaded cleanup marker name: {marker_name}"))?; + let data_name = manifest_name + .strip_suffix(".manifest.json") + .ok_or_else(|| { + anyhow::anyhow!("cleanup marker is not bound to a manifest: {marker_name}") + })?; + let _segment_id = data_name + .strip_prefix("part-") + .and_then(|value| value.strip_suffix(".jsonl.zst")) + .filter(|value| !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())) + .ok_or_else(|| { + anyhow::anyhow!("cleanup marker has an invalid segment name: {marker_name}") + })?; + let success_name = format!("{data_name}._SUCCESS"); + let expected = [ + ("data", data_name), + ("manifest", manifest_name), + ("success", success_name.as_str()), + ]; + for (field, expected_name) in expected { + let actual = metadata[field] + .as_str() + .ok_or_else(|| anyhow::anyhow!("uploaded cleanup marker missing {field}"))?; + if actual != expected_name { + anyhow::bail!( + "uploaded cleanup marker {field} does not match its segment: expected {expected_name}, got {actual}" + ); + } } - let path = marker + let parent = marker .parent() - .ok_or_else(|| anyhow::anyhow!("cleanup marker has no parent: {}", marker.display()))? - .join(name); - if path == marker { - anyhow::bail!("uploaded cleanup marker cannot reference itself"); - } - Ok(path) + .ok_or_else(|| anyhow::anyhow!("cleanup marker has no parent: {}", marker.display()))?; + Ok([ + parent.join(data_name), + parent.join(manifest_name), + parent.join(success_name), + ]) } fn local_file_name(path: &Path) -> anyhow::Result<&str> { @@ -1593,6 +1663,113 @@ mod tests { } } + #[test] + fn uploaded_cleanup_marker_replaces_only_a_stale_regular_temp() { + let spool_dir = env::temp_dir().join(format!( + "monday-upload-cleanup-stale-temp-{}", + now_ns().unwrap() + )); + std::fs::create_dir_all(&spool_dir).unwrap(); + let data = spool_dir.join("part-1700000000000000000.jsonl.zst"); + let manifest = spool_dir.join("part-1700000000000000000.jsonl.zst.manifest.json"); + let success = spool_dir.join("part-1700000000000000000.jsonl.zst._SUCCESS"); + for path in [&data, &manifest, &success] { + std::fs::write(path, b"artifact").unwrap(); + } + let temporary = spool_dir.join(format!( + "{}{}.tmp", + local_file_name(&manifest).unwrap(), + UPLOADED_CLEANUP_SUFFIX + )); + std::fs::write(&temporary, b"stale").unwrap(); + + let marker = write_uploaded_cleanup_marker(&data, &manifest, &success).unwrap(); + assert!(marker.is_file()); + assert!(!temporary.exists()); + cleanup_uploaded_marker(&marker).unwrap(); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + + #[cfg(unix)] + #[test] + fn uploaded_cleanup_marker_refuses_a_symlink_temp_without_touching_its_target() { + use std::os::unix::fs::symlink; + + let spool_dir = env::temp_dir().join(format!( + "monday-upload-cleanup-symlink-temp-{}", + now_ns().unwrap() + )); + std::fs::create_dir_all(&spool_dir).unwrap(); + let data = spool_dir.join("part-1700000000000000000.jsonl.zst"); + let manifest = spool_dir.join("part-1700000000000000000.jsonl.zst.manifest.json"); + let success = spool_dir.join("part-1700000000000000000.jsonl.zst._SUCCESS"); + for path in [&data, &manifest, &success] { + std::fs::write(path, b"artifact").unwrap(); + } + let victim = spool_dir.join("victim"); + std::fs::write(&victim, b"keep-me").unwrap(); + let temporary = spool_dir.join(format!( + "{}{}.tmp", + local_file_name(&manifest).unwrap(), + UPLOADED_CLEANUP_SUFFIX + )); + symlink(&victim, &temporary).unwrap(); + + let error = write_uploaded_cleanup_marker(&data, &manifest, &success).unwrap_err(); + assert!(error.to_string().contains("non-regular cleanup temp path")); + assert_eq!(std::fs::read(&victim).unwrap(), b"keep-me"); + assert!(std::fs::symlink_metadata(&temporary) + .unwrap() + .file_type() + .is_symlink()); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + + #[test] + fn uploaded_cleanup_marker_cannot_delete_a_sibling_segment() { + let spool_dir = env::temp_dir().join(format!( + "monday-upload-cleanup-cross-segment-{}", + now_ns().unwrap() + )); + std::fs::create_dir_all(&spool_dir).unwrap(); + let data = spool_dir.join("part-1700000000000000000.jsonl.zst"); + let manifest = spool_dir.join("part-1700000000000000000.jsonl.zst.manifest.json"); + let success = spool_dir.join("part-1700000000000000000.jsonl.zst._SUCCESS"); + let sibling_data = spool_dir.join("part-1700000000000000001.jsonl.zst"); + let sibling_manifest = spool_dir.join("part-1700000000000000001.jsonl.zst.manifest.json"); + let sibling_success = spool_dir.join("part-1700000000000000001.jsonl.zst._SUCCESS"); + for path in [ + &data, + &manifest, + &success, + &sibling_data, + &sibling_manifest, + &sibling_success, + ] { + std::fs::write(path, b"artifact").unwrap(); + } + let marker = write_uploaded_cleanup_marker(&data, &manifest, &success).unwrap(); + let mut metadata: Value = + serde_json::from_reader(std::fs::File::open(&marker).unwrap()).unwrap(); + metadata["manifest"] = json!(local_file_name(&sibling_manifest).unwrap()); + std::fs::write(&marker, serde_json::to_vec(&metadata).unwrap()).unwrap(); + + let error = cleanup_uploaded_marker(&marker).unwrap_err(); + assert!(error.to_string().contains("does not match its segment")); + for path in [ + &data, + &manifest, + &success, + &sibling_data, + &sibling_manifest, + &sibling_success, + &marker, + ] { + assert!(path.exists(), "{} was removed", path.display()); + } + std::fs::remove_dir_all(spool_dir).unwrap(); + } + #[test] fn uploaded_cleanup_marker_is_removed_last() { let spool_dir = @@ -1731,6 +1908,34 @@ mod tests { assert_eq!(manifest.excluded_symbols, ["BLOCKEDUSDT"]); } + #[tokio::test] + async fn symbol_discovery_times_out_when_exchange_info_stalls() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 2048]; + let _ = stream.read(&mut request); + thread::sleep(Duration::from_millis(250)); + }); + + let started = Instant::now(); + let error = discover_symbols_with_timeout( + Market::Spot, + &format!("http://{address}"), + Duration::from_millis(50), + ) + .await + .unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(1)); + assert!(error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(reqwest::Error::is_timeout) + })); + server.join().unwrap(); + } + #[tokio::test] async fn snapshot_retries_429_and_5xx_then_succeeds() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); From b9d7a7172c30fbc0a3003affa8e96ceb2cbe2fb7 Mon Sep 17 00:00:00 2001 From: proerror Date: Wed, 15 Jul 2026 14:08:48 +0800 Subject: [PATCH 3/6] fix(ci): allow sourced control-plane fixtures --- deployment/aliyun/test-rust-lob-control-plane.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/aliyun/test-rust-lob-control-plane.sh b/deployment/aliyun/test-rust-lob-control-plane.sh index 529a76ef6..3ef8c4f9f 100755 --- a/deployment/aliyun/test-rust-lob-control-plane.sh +++ b/deployment/aliyun/test-rust-lob-control-plane.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Dynamically sourced production functions consume fixture globals and mocks. -# shellcheck disable=SC2016,SC2034,SC2329 +# shellcheck disable=SC2016,SC2034,SC2317,SC2329 set -euo pipefail SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd) From 51457c822b962c8dd5c6fa1e43f46f6f40da2783 Mon Sep 17 00:00:00 2001 From: proerror Date: Wed, 15 Jul 2026 14:24:10 +0800 Subject: [PATCH 4/6] fix(collector): harden spool cleanup paths --- rust_hft/Cargo.lock | 1 + rust_hft/tools/collector/Cargo.toml | 1 + .../collector/src/bin/binance-lob-archiver.rs | 99 ++++++++++++++++++- 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/rust_hft/Cargo.lock b/rust_hft/Cargo.lock index 2e4667205..f35084a94 100644 --- a/rust_hft/Cargo.lock +++ b/rust_hft/Cargo.lock @@ -3517,6 +3517,7 @@ dependencies = [ "hex", "hft-engine", "http 0.2.12", + "libc", "lz4", "once_cell", "ordered-float 4.6.0", diff --git a/rust_hft/tools/collector/Cargo.toml b/rust_hft/tools/collector/Cargo.toml index af7c9cb1a..e1793939a 100644 --- a/rust_hft/tools/collector/Cargo.toml +++ b/rust_hft/tools/collector/Cargo.toml @@ -28,6 +28,7 @@ anyhow = "1" async-trait = "0.1" chrono = { version = "0.4", features = ["serde"] } fs4 = { workspace = true } +libc = "0.2" rust_decimal = { version = "1", features = ["serde"] } ordered-float = "4" tracing = "0.1" diff --git a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs index 6fe14f4c0..0405e74e3 100644 --- a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs +++ b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs @@ -10,6 +10,7 @@ use hft_collector::lob_archiver::{ use serde_json::{json, Value}; use std::collections::{BTreeSet, HashMap}; use std::env; +use std::os::unix::fs::OpenOptionsExt; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -87,8 +88,15 @@ impl SpoolLock { .truncate(false) .read(true) .write(true) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) .open(&path) .with_context(|| format!("failed to open spool lock {}", path.display()))?; + let metadata = file + .metadata() + .with_context(|| format!("failed to inspect spool lock {}", path.display()))?; + if !metadata.file_type().is_file() { + anyhow::bail!("spool lock is not a regular file: {}", path.display()); + } if let Err(error) = fs4::FileExt::try_lock(&file) { let error = std::io::Error::from(error); if error.kind() == std::io::ErrorKind::WouldBlock { @@ -1287,7 +1295,33 @@ fn recover_uploaded_cleanups(spool_dir: &Path) -> anyhow::Result { } fn cleanup_uploaded_marker(marker: &Path) -> anyhow::Result<()> { - let metadata: Value = serde_json::from_reader(std::fs::File::open(marker)?) + let marker_file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .open(marker) + .with_context(|| { + format!( + "failed to open uploaded cleanup marker {}", + marker.display() + ) + })?; + if !marker_file + .metadata() + .with_context(|| { + format!( + "failed to inspect uploaded cleanup marker {}", + marker.display() + ) + })? + .file_type() + .is_file() + { + anyhow::bail!( + "uploaded cleanup marker is not a regular file: {}", + marker.display() + ); + } + let metadata: Value = serde_json::from_reader(marker_file) .with_context(|| format!("invalid uploaded cleanup marker {}", marker.display()))?; if metadata["schema"] != UPLOADED_CLEANUP_SCHEMA { anyhow::bail!( @@ -1296,6 +1330,35 @@ fn cleanup_uploaded_marker(marker: &Path) -> anyhow::Result<()> { ); } let paths = cleanup_artifact_paths(marker, &metadata)?; + match std::fs::symlink_metadata(marker) { + Ok(metadata) if metadata.file_type().is_file() => {} + Ok(_) => anyhow::bail!( + "refusing non-regular uploaded cleanup marker: {}", + marker.display() + ), + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to inspect uploaded cleanup marker {}", + marker.display() + ) + }); + } + } + for path in &paths { + match std::fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Ok(metadata) if metadata.file_type().is_file() => {} + Ok(_) => { + anyhow::bail!("refusing non-regular uploaded artifact: {}", path.display()) + } + Err(error) => { + return Err(error).with_context(|| { + format!("failed to inspect uploaded artifact {}", path.display()) + }); + } + } + } for path in paths { match std::fs::remove_file(&path) { Ok(()) => {} @@ -1581,6 +1644,37 @@ mod tests { std::fs::remove_dir_all(spool_dir).unwrap(); } + #[tokio::test] + async fn upload_only_rejects_a_symlink_spool_lock_without_touching_its_target() { + use std::os::unix::fs::symlink; + + let spool_dir = + env::temp_dir().join(format!("monday-spool-lock-symlink-{}", now_ns().unwrap())); + std::fs::create_dir_all(&spool_dir).unwrap(); + let victim = spool_dir.join("victim"); + std::fs::write(&victim, b"keep-me").unwrap(); + let lock_path = spool_dir.join(SPOOL_LOCK_FILE); + symlink(&victim, &lock_path).unwrap(); + let config = UploadConfig { + spool_dir: spool_dir.clone(), + oss_bucket: "unused".into(), + oss_endpoint: "unused".into(), + oss_region: "ap-northeast-1".into(), + aliyun_profile: "unused".into(), + oss_copy_timeout: Duration::from_secs(1), + }; + + let error = upload_only(&config).await.unwrap_err(); + assert!(error.to_string().contains("failed to open spool lock")); + assert_eq!(std::fs::read(&victim).unwrap(), b"keep-me"); + assert!(std::fs::symlink_metadata(&lock_path) + .unwrap() + .file_type() + .is_symlink()); + assert!(!spool_dir.join("upload-status.json").exists()); + std::fs::remove_dir_all(spool_dir).unwrap(); + } + #[tokio::test] async fn upload_only_rejects_an_interrupted_segment() { for suffix in ["jsonl.part", "jsonl.zst.tmp", "jsonl.part.corrupt"] { @@ -1787,6 +1881,9 @@ mod tests { recover_uploaded_cleanups(&spool_dir).unwrap_err(); assert!(marker.is_file()); + assert!(data.is_file()); + assert!(manifest.is_file()); + assert!(success.is_dir()); std::fs::remove_dir(&success).unwrap(); assert_eq!(recover_uploaded_cleanups(&spool_dir).unwrap(), 1); From a9fd835d915f9ed7a0a03b6a0471e50382a925a3 Mon Sep 17 00:00:00 2001 From: proerror Date: Wed, 15 Jul 2026 14:26:44 +0800 Subject: [PATCH 5/6] fix(collector): reject special cleanup paths --- .../collector/src/bin/binance-lob-archiver.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs index 0405e74e3..9cb0d28e6 100644 --- a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs +++ b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs @@ -88,7 +88,7 @@ impl SpoolLock { .truncate(false) .read(true) .write(true) - .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK) .open(&path) .with_context(|| format!("failed to open spool lock {}", path.display()))?; let metadata = file @@ -1295,9 +1295,24 @@ fn recover_uploaded_cleanups(spool_dir: &Path) -> anyhow::Result { } fn cleanup_uploaded_marker(marker: &Path) -> anyhow::Result<()> { + match std::fs::symlink_metadata(marker) { + Ok(metadata) if metadata.file_type().is_file() => {} + Ok(_) => anyhow::bail!( + "refusing non-regular uploaded cleanup marker: {}", + marker.display() + ), + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to inspect uploaded cleanup marker {}", + marker.display() + ) + }); + } + } let marker_file = std::fs::OpenOptions::new() .read(true) - .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK) .open(marker) .with_context(|| { format!( From c2a63d3c33f51bcd69eaaac578f314d855abf33c Mon Sep 17 00:00:00 2001 From: proerror Date: Wed, 15 Jul 2026 14:32:37 +0800 Subject: [PATCH 6/6] test(collector): isolate spool lock fixture --- rust_hft/Cargo.lock | 1 + rust_hft/tools/collector/Cargo.toml | 3 +++ rust_hft/tools/collector/src/bin/binance-lob-archiver.rs | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rust_hft/Cargo.lock b/rust_hft/Cargo.lock index f35084a94..8d9693bd7 100644 --- a/rust_hft/Cargo.lock +++ b/rust_hft/Cargo.lock @@ -3528,6 +3528,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "tempfile", "tokio", "tokio-tungstenite 0.26.2", "tracing", diff --git a/rust_hft/tools/collector/Cargo.toml b/rust_hft/tools/collector/Cargo.toml index e1793939a..e35c53bb1 100644 --- a/rust_hft/tools/collector/Cargo.toml +++ b/rust_hft/tools/collector/Cargo.toml @@ -68,6 +68,9 @@ collector-okx = [] collector-asterdex = [] duckdb = ["dep:duckdb"] +[dev-dependencies] +tempfile = "3.10" + [[bin]] name = "hft-collector" path = "src/main.rs" diff --git a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs index 9cb0d28e6..3c454d915 100644 --- a/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs +++ b/rust_hft/tools/collector/src/bin/binance-lob-archiver.rs @@ -1663,8 +1663,8 @@ mod tests { async fn upload_only_rejects_a_symlink_spool_lock_without_touching_its_target() { use std::os::unix::fs::symlink; - let spool_dir = - env::temp_dir().join(format!("monday-spool-lock-symlink-{}", now_ns().unwrap())); + let temp_dir = tempfile::tempdir().unwrap(); + let spool_dir = temp_dir.path().join("spool"); std::fs::create_dir_all(&spool_dir).unwrap(); let victim = spool_dir.join("victim"); std::fs::write(&victim, b"keep-me").unwrap();