diff --git a/.github/scripts/fixtures/rust-ci-scope/collector.txt b/.github/scripts/fixtures/rust-ci-scope/collector.txt new file mode 100644 index 000000000..9163f6cee --- /dev/null +++ b/.github/scripts/fixtures/rust-ci-scope/collector.txt @@ -0,0 +1 @@ +rust_hft/tools/collector/src/polymarket/reference.rs diff --git a/.github/scripts/fixtures/rust-ci-scope/control.txt b/.github/scripts/fixtures/rust-ci-scope/control.txt new file mode 100644 index 000000000..b5e0e5c41 --- /dev/null +++ b/.github/scripts/fixtures/rust-ci-scope/control.txt @@ -0,0 +1 @@ +deployment/aliyun/polymarket-raw-ops-cutover.sh diff --git a/.github/scripts/fixtures/rust-ci-scope/docs.txt b/.github/scripts/fixtures/rust-ci-scope/docs.txt new file mode 100644 index 000000000..7efd40e02 --- /dev/null +++ b/.github/scripts/fixtures/rust-ci-scope/docs.txt @@ -0,0 +1 @@ +docs/reports/example.md diff --git a/.github/scripts/fixtures/rust-ci-scope/live.txt b/.github/scripts/fixtures/rust-ci-scope/live.txt new file mode 100644 index 000000000..01cca5318 --- /dev/null +++ b/.github/scripts/fixtures/rust-ci-scope/live.txt @@ -0,0 +1 @@ +rust_hft/apps/live/src/main.rs diff --git a/.github/scripts/fixtures/rust-ci-scope/metadata.fixture b/.github/scripts/fixtures/rust-ci-scope/metadata.fixture new file mode 100644 index 000000000..686a69b90 --- /dev/null +++ b/.github/scripts/fixtures/rust-ci-scope/metadata.fixture @@ -0,0 +1,14 @@ +{ + "packages": [ + { "name": "hft-collector", "manifest_path": "rust_hft/tools/collector/Cargo.toml", "dependencies": [] }, + { + "name": "alpha-harness", + "manifest_path": "rust_hft/alpha-harness/app/Cargo.toml", + "dependencies": [{ "name": "hft-collector", "path": "rust_hft/tools/collector" }] + }, + { "name": "hft-live", "manifest_path": "rust_hft/apps/live/Cargo.toml", "dependencies": [] }, + { "name": "hft-integration", "manifest_path": "rust_hft/market-core/integration/Cargo.toml", "dependencies": [] }, + { "name": "hft-data-adapter-ondo-perps", "manifest_path": "rust_hft/data-pipelines/adapters/adapter-ondo-perps/Cargo.toml", "dependencies": [] }, + { "name": "hft-paper", "manifest_path": "rust_hft/apps/paper/Cargo.toml", "dependencies": [] } + ] +} diff --git a/.github/scripts/select-rust-ci-scope.sh b/.github/scripts/select-rust-ci-scope.sh new file mode 100755 index 000000000..6c9495579 --- /dev/null +++ b/.github/scripts/select-rust-ci-scope.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +set -euo pipefail + +event=${GITHUB_EVENT_NAME:-} +base= +head=HEAD +changed_files= +metadata= +output=${GITHUB_OUTPUT:-/dev/stdout} + +while (($#)); do + case "$1" in + --event) event=$2; shift 2 ;; + --base) base=$2; shift 2 ;; + --head) head=$2; shift 2 ;; + --changed-files) changed_files=$2; shift 2 ;; + --metadata) metadata=$2; shift 2 ;; + --output) output=$2; shift 2 ;; + *) printf 'unknown argument: %s\n' "$1" >&2; exit 2 ;; + esac +done + +loop=false +handoff=false +json=false +ondo=false +collector=false +control=false +focused=false +toolchain=false + +select_all() { + loop=true + handoff=true + json=true + ondo=true + collector=true + control=true + focused=true + toolchain=true +} + +emit() { + printf '%s\n' \ + "loop=$loop" \ + "handoff=$handoff" \ + "json=$json" \ + "ondo=$ondo" \ + "collector=$collector" \ + "control=$control" \ + "focused=$focused" \ + "toolchain=$toolchain" >>"$output" +} + +# Branch pushes and manual runs retain the complete build/package contract. +if [[ $event != pull_request ]]; then + select_all + emit + exit 0 +fi + +repo_root=$(git rev-parse --show-toplevel) +declare -a paths=() +if [[ -n $changed_files ]]; then + while IFS= read -r path; do paths+=("$path"); done <"$changed_files" +else + [[ -n $base ]] || { printf '%s\n' '--base is required for pull requests' >&2; exit 2; } + while IFS= read -r -d '' path; do paths+=("$path"); done \ + < <(git diff --no-renames --name-only --diff-filter=ACMRD -z "$base...$head") +fi + +needs_metadata=false +for path in "${paths[@]}"; do + case "$path" in + rust_hft/Cargo.toml|rust_hft/Cargo.lock|rust_hft/rust-toolchain*|rust_hft/.cargo/*|.cargo/*|.github/workflows/ci.yml|.github/scripts/select-rust-ci-scope.sh|.github/scripts/test-select-rust-ci-scope.sh|.github/scripts/fixtures/rust-ci-scope/*) + select_all + emit + exit 0 + ;; + deployment/aliyun/*) + control=true + toolchain=true + ;; + rust_hft/*) + needs_metadata=true + ;; + esac +done + +if [[ $needs_metadata == false ]]; then + emit + exit 0 +fi + +if [[ -z $metadata ]]; then + metadata=$(mktemp) + trap 'rm -f "$metadata"' EXIT + (cd "$repo_root/rust_hft" && cargo metadata --format-version 1 --no-deps --locked) >"$metadata" +fi + +declare -a package_names=() package_dirs=() package_dependencies=() +while IFS=$'\t' read -r name manifest dependencies; do + if [[ $manifest == "$repo_root/"* ]]; then manifest=${manifest#"$repo_root/"}; fi + package_names+=("$name") + package_dirs+=("${manifest%/Cargo.toml}") + package_dependencies+=("$dependencies") +done < <(jq -r '.packages[] | [.name, .manifest_path, ([.dependencies[] | select(.path != null) | .name] | join(","))] | @tsv' "$metadata") + +affected=$'\n' +is_affected() { [[ $affected == *$'\n'"$1"$'\n'* ]]; } +mark_affected() { is_affected "$1" || affected+="$1"$'\n'; } + +for path in "${paths[@]}"; do + [[ $path == rust_hft/* ]] || continue + owner= + owner_length=0 + for ((index = 0; index < ${#package_names[@]}; index++)); do + name=${package_names[$index]} + directory=${package_dirs[$index]} + if [[ $path == "$directory"/* && ${#directory} -gt $owner_length ]]; then + owner=$name + owner_length=${#directory} + fi + done + if [[ -z $owner || $owner == rust-hft-workspace ]]; then + select_all + emit + exit 0 + fi + mark_affected "$owner" +done + +# Cargo.toml remains the source of truth for downstream package impact. +changed=true +while [[ $changed == true ]]; do + changed=false + for ((index = 0; index < ${#package_names[@]}; index++)); do + name=${package_names[$index]} + is_affected "$name" && continue + dependency_list=${package_dependencies[$index]} + [[ -n $dependency_list ]] || continue + IFS=',' read -ra dependencies <<<"$dependency_list" + for dependency in "${dependencies[@]}"; do + if [[ -n $dependency ]] && is_affected "$dependency"; then + mark_affected "$name" + changed=true + break + fi + done + done +done + +select_if_affected() { + local flag=$1 + shift + local package + for package in "$@"; do + if is_affected "$package"; then + case "$flag" in + loop) loop=true ;; + handoff) handoff=true ;; + json) json=true ;; + ondo) ondo=true ;; + collector) collector=true ;; + focused) focused=true ;; + esac + toolchain=true + return + fi + done +} + +select_if_affected loop alpha-domain alpha-store alpha-engine alpha-onnx-evaluator \ + alpha-harness hft-harnessctl hft-research-ml +select_if_affected handoff hft-live +select_if_affected json hft-integration hft-data-adapter-binance hft-infra-redis hft-live +select_if_affected ondo hft-data-adapter-ondo-perps hft-execution-adapter-ondo-perps hft-live +select_if_affected collector hft-collector +select_if_affected focused hft-live hft-paper hft-all-in-one + +# Collector source and its host controls are one release boundary. +if [[ $collector == true ]]; then control=true; fi + +emit diff --git a/.github/scripts/test-select-rust-ci-scope.sh b/.github/scripts/test-select-rust-ci-scope.sh new file mode 100755 index 000000000..0a4e45b3e --- /dev/null +++ b/.github/scripts/test-select-rust-ci-scope.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +selector="$script_dir/select-rust-ci-scope.sh" +fixtures="$script_dir/fixtures/rust-ci-scope" +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT + +run_case() { + local name=$1 event=$2 changed=$3 + local output="$tmp_dir/$name.out" + "$selector" --event "$event" --changed-files "$fixtures/$changed" \ + --metadata "$fixtures/metadata.fixture" --output "$output" + printf '%s\n' "$output" +} + +assert_flag() { + local output=$1 flag=$2 expected=$3 + grep -qx "$flag=$expected" "$output" || { + printf '%s: expected %s=%s\n' "$output" "$flag" "$expected" >&2 + cat "$output" >&2 + exit 1 + } +} + +collector=$(run_case collector pull_request collector.txt) +for flag in loop collector control toolchain; do assert_flag "$collector" "$flag" true; done +for flag in handoff json ondo focused; do assert_flag "$collector" "$flag" false; done + +live=$(run_case live pull_request live.txt) +for flag in handoff json ondo focused toolchain; do assert_flag "$live" "$flag" true; done +for flag in loop collector control; do assert_flag "$live" "$flag" false; done + +control=$(run_case control pull_request control.txt) +for flag in control toolchain; do assert_flag "$control" "$flag" true; done +for flag in loop handoff json ondo collector focused; do assert_flag "$control" "$flag" false; done + +docs=$(run_case docs pull_request docs.txt) +for flag in loop handoff json ondo collector control focused toolchain; do + assert_flag "$docs" "$flag" false +done + +full=$(run_case full push collector.txt) +for flag in loop handoff json ondo collector control focused toolchain; do + assert_flag "$full" "$flag" true +done + +deletion_repo="$tmp_dir/deletion-repo" +mkdir -p "$deletion_repo/rust_hft/tools/collector/src" +git -C "$deletion_repo" init -q +touch "$deletion_repo/rust_hft/tools/collector/src/removed.rs" +git -C "$deletion_repo" add . +git -C "$deletion_repo" -c user.name=CI -c user.email=ci@example.invalid commit -qm base +deletion_base=$(git -C "$deletion_repo" rev-parse HEAD) +rm "$deletion_repo/rust_hft/tools/collector/src/removed.rs" +git -C "$deletion_repo" add -u +git -C "$deletion_repo" -c user.name=CI -c user.email=ci@example.invalid commit -qm deletion +deletion="$tmp_dir/deletion.out" +(cd "$deletion_repo" && "$selector" --event pull_request --base "$deletion_base" \ + --head HEAD --metadata "$fixtures/metadata.fixture" --output "$deletion") +for flag in loop collector control toolchain; do assert_flag "$deletion" "$flag" true; done + +rename_repo="$tmp_dir/rename-repo" +mkdir -p "$rename_repo/rust_hft/tools/collector/src" "$rename_repo/docs" +git -C "$rename_repo" init -q +touch "$rename_repo/rust_hft/tools/collector/src/moved.rs" +git -C "$rename_repo" add . +git -C "$rename_repo" -c user.name=CI -c user.email=ci@example.invalid commit -qm base +rename_base=$(git -C "$rename_repo" rev-parse HEAD) +git -C "$rename_repo" mv rust_hft/tools/collector/src/moved.rs docs/moved.rs +git -C "$rename_repo" -c user.name=CI -c user.email=ci@example.invalid commit -qm rename +rename="$tmp_dir/rename.out" +(cd "$rename_repo" && "$selector" --event pull_request --base "$rename_base" \ + --head HEAD --metadata "$fixtures/metadata.fixture" --output "$rename") +for flag in loop collector control toolchain; do assert_flag "$rename" "$flag" true; done + +printf 'rust CI scope selector tests passed\n' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1711b3cfc..50f06bc27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: Monorepo CI on: + workflow_dispatch: push: branches: [main, develop] paths-ignore: @@ -32,6 +33,21 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Select Rust validation scope + id: scope + working-directory: . + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + .github/scripts/test-select-rust-ci-scope.sh + .github/scripts/select-rust-ci-scope.sh \ + --event "${{ github.event_name }}" \ + --base "${BASE_SHA}" \ + --head "${HEAD_SHA}" - name: Enforce Monday module layout working-directory: . @@ -185,17 +201,21 @@ jobs: fi - name: Setup Rust toolchain + if: steps.scope.outputs.toolchain == 'true' uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy - name: Install Linux build tools + if: steps.scope.outputs.toolchain == 'true' run: sudo apt-get update && sudo apt-get install -y clang mold protobuf-compiler shellcheck - name: Setup sccache + if: steps.scope.outputs.toolchain == 'true' uses: mozilla-actions/sccache-action@v0.0.10 - name: Record cache dimensions + if: steps.scope.outputs.toolchain == 'true' id: cache-info shell: bash run: | @@ -203,6 +223,7 @@ jobs: echo "sccache=$(${SCCACHE_PATH:-sccache} --version | awk '{print $2}')" >> "$GITHUB_OUTPUT" - name: Cache Rust + if: steps.scope.outputs.toolchain == 'true' uses: Swatinem/rust-cache@v2 with: workspaces: | @@ -212,6 +233,7 @@ jobs: save-if: ${{ github.event_name != 'pull_request' }} - name: Format Loop Engineer packages + if: steps.scope.outputs.loop == 'true' run: | cargo fmt --check --package alpha-domain cargo fmt --check --package alpha-store @@ -222,9 +244,11 @@ jobs: cargo fmt --check --package hft-research-ml - name: Test Loop Engineer packages + if: steps.scope.outputs.loop == 'true' run: cargo test -p alpha-domain -p alpha-store -p alpha-engine -p alpha-onnx-evaluator -p alpha-harness -p hft-harnessctl -p hft-research-ml --locked - name: Verify signed runtime handoff + if: steps.scope.outputs.handoff == 'true' run: | cargo test -p hft-live --no-default-features --test deployment_envelope --locked cargo test -p hft-live --no-default-features --test deployment_artifacts --locked @@ -232,6 +256,7 @@ jobs: cargo test -p hft-live --no-default-features --features formula-strategy,bitget --test deployment_artifacts --locked - name: Check JSON feature matrix + if: steps.scope.outputs.json == 'true' run: | cargo test -p hft-integration --no-default-features --locked cargo test -p hft-integration --no-default-features --features json-simd --locked @@ -241,18 +266,21 @@ jobs: cargo check -p hft-live --features clickhouse,redis,grpc --locked - name: Verify Ondo Perps feature boundary + if: steps.scope.outputs.ondo == 'true' run: | cargo test -p hft-data-adapter-ondo-perps -p hft-execution-adapter-ondo-perps --locked cargo clippy -p hft-data-adapter-ondo-perps -p hft-execution-adapter-ondo-perps --all-targets --no-deps --locked -- -D warnings cargo check -p hft-live --features ondo-perps --locked - name: Verify governed data collector + if: steps.scope.outputs.collector == 'true' run: | cargo check -p hft-collector --locked cargo clippy -p hft-collector --all-targets --features collector-binance --no-deps --locked -- -D warnings cargo test -p hft-collector --features collector-binance --locked - name: Verify collector release control plane + if: steps.scope.outputs.control == 'true' run: | for script in ../deployment/aliyun/*.sh; do bash -n "$script" @@ -265,6 +293,7 @@ jobs: cargo test -p hft-collector systemd_memory_envelope_covers_measured_polymarket_cold_start --locked - name: Build focused packages + if: steps.scope.outputs.focused == 'true' run: cargo build -p hft-live -p hft-paper -p hft-all-in-one -p alpha-harness -p hft-harnessctl --locked - name: Package binaries (if present)