Claims pipeline (#3)

.buildkite/fetch-snapshot.yml

@@ -0,0 +1,60 @@
+agents:
+  queue: "snapshots"
+
+steps:
+  - command: echo "--> Start of concurrency gate"
+    concurrency_group: 'validator-bonds/fetch'
+    concurrency: 1
+
+  - input: "Which epoch to fetch?"
+    fields:
+      - text: "Epoch"
+        key: "epoch"
+        format: "[0-9]+"
+    if: "build.env('EPOCH') == null"
+
+  - wait: ~
+
+  - label: ":file_folder: Prepare snapshot directory"
+    env:
+      target_dir: /mnt/storage-1/snapshots
+    commands:
+    - 'mkdir -p /mnt/storage-1/snapshots'
+    - 'epoch=${EPOCH:-$(buildkite-agent meta-data get epoch)}'
+    - 'snapshot_dir=$(mktemp --directory -p "$$target_dir" "snapshot-$$epoch-$(date +%s)-XXXXXX")'
+    - 'buildkite-agent meta-data set snapshot_dir "$$snapshot_dir"'
+
+  - wait: ~
+
+  - label: ":cloud: :arrow_right: :floppy_disk: Fetch genesis"
+    commands:
+    - 'snapshot_dir=$(buildkite-agent meta-data get snapshot_dir)'
+    - './scripts/fetch-genesis.bash "$$snapshot_dir"'
+
+  - label: ":cloud: :arrow_right: :floppy_disk: Fetch snapshot"
+    commands:
+    - 'snapshot_dir=$(buildkite-agent meta-data get snapshot_dir)'
+    - 'epoch=${EPOCH:-$(buildkite-agent meta-data get epoch)}'
+    - './scripts/fetch-jito-snapshot.bash "$$epoch" "$$snapshot_dir"'
+
+  - wait: ~
+
+  - label: ":gear: Setup trigger"
+    commands:
+    - 'snapshot_dir=$(buildkite-agent meta-data get snapshot_dir)'
+    - |
+      cat <<EOF | buildkite-agent pipeline upload
+      steps:
+        - trigger: "prepare-claims"
+          label: ":rocket: Trigger: Prepare claims ($$snapshot_dir)"
+          async: false
+          build:
+            env:
+              SNAPSHOT_DIR: $$snapshot_dir
+      EOF
+
+  - wait: ~
+
+  - command: echo "End of concurrency gate <--"
+    concurrency_group: 'validator-bonds/fetch'
+    concurrency: 1

.buildkite/prepare-claims.yml

@@ -0,0 +1,102 @@
+agents:
+  queue: "snapshots"
+
+steps:
+  - label: ":hammer_and_wrench: :rust: Build"
+    commands:
+    - '. "$HOME/.cargo/env"'
+    - 'cargo build --release --bin snapshot-parser-cli --bin insurance-engine-cli'
+    artifact_paths:
+      - target/release/snapshot-parser-cli
+      - target/release/insurance-engine-cli
+
+  - input: "What is the snapshot directory?"
+    fields:
+      - text: "Snapshot directory"
+        key: "snapshot_dir"
+    if: "build.env('SNAPSHOT_DIR') == null"
+
+  - wait: ~
+
+  - label: ":gear: Setup"
+    commands:
+    - '. "$HOME/.cargo/env"'
+    - 'snapshot_dir=${SNAPSHOT_DIR:-$(buildkite-agent meta-data get snapshot_dir)}'
+    - 'echo "Snapshot dir: $$snapshot_dir"'
+    - 'epoch=$(<<<"$$snapshot_dir" grep -oP "(?<=snapshot-)\d+" || exit 1)'
+    - 'echo "Epoch: $$epoch"'
+    - 'buildkite-agent meta-data set snapshot_dir "$$snapshot_dir"'
+    - 'buildkite-agent meta-data set epoch "$$epoch"'
+
+  - wait: ~
+
+  - command: echo "--> Start of concurrency gate"
+    concurrency_group: 'validator-bonds/prepare-claims'
+    concurrency: 1
+
+  - wait: ~
+
+  - label: ":microscope: Parse Snapshot"
+    commands:
+    - 'snapshot_dir=$(buildkite-agent meta-data get snapshot_dir)'
+    - 'buildkite-agent artifact download --include-retried-jobs target/release/snapshot-parser-cli .'
+    - 'chmod +x target/release/snapshot-parser-cli'
+    - |
+      ./target/release/snapshot-parser-cli \
+        --ledger-path "$$snapshot_dir" \
+        --output-validator-meta-collection "$$snapshot_dir/validators.json" \
+        --output-stake-meta-collection "$$snapshot_dir/stakes.json"
+
+  - wait: ~
+
+  - label: ":scales: Evaluate Marinade Insured Events"
+    env:
+      WHITELIST_STAKE_AUTHORITY: stWirqFCf2Uts1JBL1Jsd3r6VBWhgnpdPxCTe1MFjrq,4bZ6o3eUUNXhKuqjdCnCoPAoLgWiuLYixKaxoa8PpiKk,ex9CfkBZZd6Nv9XdnoDmmB45ymbu4arXVk7g5pWnt3N
+    commands:
+    - 'snapshot_dir=$(buildkite-agent meta-data get snapshot_dir)'
+    - 'buildkite-agent artifact download --include-retried-jobs target/release/insurance-engine-cli .'
+    - 'chmod +x target/release/insurance-engine-cli'
+    - |
+      ./target/release/insurance-engine-cli \
+        --validator-meta-collection "$$snapshot_dir/validators.json" \
+        --stake-meta-collection "$$snapshot_dir/stakes.json" \
+        --output-insured-event-collection "$$snapshot_dir/events.json" \
+        --output-insurance-claim-collection "$$snapshot_dir/claims.json" \
+        --output-merkle-tree-collection "$$snapshot_dir/claims_merkle.json" \
+        --low-rewards-threshold-pct 0.9
+
+  - label: ":scales: Evaluate All Insured Events"
+    commands:
+    - 'snapshot_dir=$(buildkite-agent meta-data get snapshot_dir)'
+    - 'buildkite-agent artifact download --include-retried-jobs target/release/insurance-engine-cli .'
+    - 'chmod +x target/release/insurance-engine-cli'
+    - |
+      ./target/release/insurance-engine-cli \
+        --validator-meta-collection "$$snapshot_dir/validators.json" \
+        --stake-meta-collection "$$snapshot_dir/stakes.json" \
+        --output-insured-event-collection "$$snapshot_dir/events_all.json" \
+        --output-insurance-claim-collection "$$snapshot_dir/claims_all.json" \
+        --output-merkle-tree-collection "$$snapshot_dir/claims_merkle_all.json" \
+        --low-rewards-threshold-pct 0.9
+
+  - block: ":rocket: Continue?"
+
+  - label: ":floppy_disk: :arrow_right: :cloud: Upload artifacts"
+    env:
+      gs_bucket: gs://marinade-validator-bonds-mainnet
+    commands:
+    - 'snapshot_dir=$(buildkite-agent meta-data get snapshot_dir)'
+    - 'epoch=$(buildkite-agent meta-data get epoch)'
+    - 'gcloud storage cp "$$snapshot_dir/validators.json" "$$gs_bucket/$$epoch/"'
+    - 'gcloud storage cp "$$snapshot_dir/stakes.json" "$$gs_bucket/$$epoch/"'
+    - 'gcloud storage cp "$$snapshot_dir/claims.json" "$$gs_bucket/$$epoch/"'
+    - 'gcloud storage cp "$$snapshot_dir/claims_merkle.json" "$$gs_bucket/$$epoch/"'
+    - 'gcloud storage cp "$$snapshot_dir/events_all.json" "$$gs_bucket/$$epoch/"'
+    - 'gcloud storage cp "$$snapshot_dir/claims_all.json" "$$gs_bucket/$$epoch/"'
+    - 'gcloud storage cp "$$snapshot_dir/claims_merkle_all.json" "$$gs_bucket/$$epoch/"'
+
+  - wait: ~
+
+  - command: echo "End of concurrency gate <--"
+    concurrency_group: 'validator-bonds/prepare-claims'
+    concurrency: 1

.buildkite/scheduler.yml

@@ -0,0 +1,44 @@
+agents:
+  queue: "snapshots"
+
+steps:
+  - command: echo "--> Start of concurrency gate"
+    concurrency_group: 'validator-bonds/scheduler'
+    concurrency: 1
+
+  - wait: ~
+
+  - label: ":recycle: Pruning old snapshot directories"
+    concurrency_group: 'validator-bonds/fetch'
+    concurrency: 1
+    env:
+      snapshots_dir: /mnt/storage-1/snapshots
+    commands:
+    - 'ls -td -- /mnt/storage-1/snapshots/*/ | awk "NR>1" | tee /dev/stderr | xargs rm -rf'
+
+  - label: ":calendar: Schedule"
+    concurrency_group: 'validator-bonds/fetch'
+    concurrency: 1
+    commands:
+    - |
+      max_processed_epoch=$(gcloud storage ls "gs://marinade-validator-bonds-mainnet" | awk -F / '{print $$4}' | sort -nr | head -n 1 || exit 1)
+      max_available_epoch=$(gcloud storage ls "gs://jito-mainnet/**/*.tar.zst" | awk -F / '{print $$4}' | sort -nr | head -n 1)
+      echo max_processed_epoch: $$max_processed_epoch
+      echo max_available_epoch: $$max_available_epoch
+      (( $$max_processed_epoch < $$max_available_epoch )) && cat <<EOF | buildkite-agent pipeline upload
+      steps:
+        - trigger: "fetch-solana-snapshot"
+          label: ":rocket: Trigger: Fetch Solana snapshot ($$max_available_epoch)"
+          async: false
+          build:
+            env:
+              EPOCH: $$max_available_epoch
+      EOF
+
+      echo Scheduling complete
+
+  - wait: ~
+
+  - command: echo "End of concurrency gate <--"
+    concurrency_group: 'validator-bonds/scheduler'
+    concurrency: 1

.github/workflows/typescript-lint.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   lint:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 30
     steps:
       - name: Checkout project
         uses: actions/checkout@v4