Refactor reporting init claim (#56)

* [cli] squashed from main * [settlement-pipeline] rent payer refactor fix * [settlement-pipeline] claim settlement reporting and refactoring * [settlement-pipeline] init settlement refactoring * [readme] basic readme for settlement pipelines * review fixes * pipeline colors adjustments * Update .buildkite/claim-settlements.yml Co-authored-by: janlegner <32453746+janlegner@users.noreply.github.com> Update .buildkite/claim-settlements.yml Co-authored-by: janlegner <32453746+janlegner@users.noreply.github.com> Update .buildkite/close-settlements.yml Co-authored-by: janlegner <32453746+janlegner@users.noreply.github.com> Update .buildkite/init-settlements.yml Co-authored-by: janlegner <32453746+janlegner@users.noreply.github.com> Update settlement-pipelines/README.md Co-authored-by: janlegner <32453746+janlegner@users.noreply.github.com> Update settlement-pipelines/README.md Co-authored-by: janlegner <32453746+janlegner@users.noreply.github.com> * review updates * review updates: fixing test output * pipeline: blocked state means running * speed-up claiming with more parallel tx execution --------- Co-authored-by: janlegner <32453746+janlegner@users.noreply.github.com>
marinade-finance · May 27, 2024 · 5422fca · 5422fca
1 parent 3b83407
commit 5422fca
Show file tree

Hide file tree

Showing 19 changed files with 1,623 additions and 864 deletions.
diff --git a/.buildkite/claim-settlements.yml b/.buildkite/claim-settlements.yml
@@ -60,14 +60,22 @@ steps:
       RUST_LOG: info,solana_transaction_builder_executor=debug,solana_transaction_builder=debug,builder_executor=debug,solana_transaction_executor=debug,settlement_pipelines=debug,claim_settlement=debug
       # RUST_BACKTRACE: full
     commands:
+    - '. "$HOME/.cargo/env"'
     - |
       claimable_epochs_num=$(buildkite-agent meta-data get claimable_epochs_num)
-      [ "$$claimable_epochs_num" -eq 0 ] && echo 'No settlement to claim from' && exit 0
+      if [[ $$claimable_epochs_num -eq 0 ]]; then
+        echo 'No settlement to claim from' | tee ./claiming-report.txt
+        exit 0
+      fi
+    - 'prior_build_number=$(($$BUILDKITE_RETRY_COUNT - 1))'
+    - 'buildkite-agent artifact download --include-retried-jobs build-report.$$prior_build_number . || true'
+    - |
+      cp "build-report.$$prior_build_number" "build-report.$$BUILDKITE_RETRY_COUNT" || true
+      rm -f "build-report.$$prior_build_number"
+      echo "#ATTEMPT $$BUILDKITE_RETRY_COUNT" | tee -a "./build-report.$$BUILDKITE_RETRY_COUNT"
     - 'buildkite-agent artifact download --include-retried-jobs "merkle-trees/*" .'
-    - '. "$HOME/.cargo/env"'
     - 'buildkite-agent artifact download --include-retried-jobs target/release/claim-settlement .'
     - 'chmod +x target/release/claim-settlement'
-    - 'echo "UNKNOWN ERROR" > ./claiming-report.txt'
     - |
       set -o pipefail
       ./target/release/claim-settlement \
@@ -76,26 +84,70 @@ steps:
         --operator-authority "$$VALIDATOR_BONDS_OPERATOR_AUTHORITY" \
         --fee-payer "$$VALIDATOR_BONDS_CLAIMING_WALLET" \
         --rent-payer "$$VALIDATOR_BONDS_CLAIMING_WALLET" \
-        | tee ./claiming-report.txt
+        | tee -a "./build-report.$$BUILDKITE_RETRY_COUNT"
+    key: 'claim-settlement'
+    artifact_paths:
+    - "./build-report.*"
+    retry:
+      automatic:
+        - exit_status: 100
+          limit: 5
+
+  - label: "Notification setup"
+    env:
+      gs_bucket: gs://marinade-validator-bonds-mainnet
+    commands:
+     - |
+       build_result=$(buildkite-agent step get "outcome" --step "claim-settlement")
+        if [[ -z "$$build_result" || "$$build_result" =~ "failed" ]]; then
+          notification_result="finished with WARNINGS"
+          notification_color="16355909"
+        else
+          notification_result="SUCCEEDED"
+          notification_color="52224"
+        fi
+     - 'buildkite-agent meta-data set notification_color "$$notification_color"'
+     - 'buildkite-agent meta-data set notification_result "$$notification_result"'
+     - 'echo "Notification result: $$notification_result, color: $$notification_color"'
+     - 'mkdir ./reports'
+     - 'buildkite-agent artifact download --include-retried-jobs "build-report.*" ./reports/'
+     - 'report_path="./reports/$(ls -v1 reports/ | tail -n 1)"'
+     - '[[ -f "$$report_path" ]] || echo "No report generated; UNKNOWN ERROR" >> "$$report_path"'
+     - 'cp "$$report_path" ./claiming-report.txt'
+     - 'claimable_epochs_json=$(buildkite-agent meta-data get claimable_epochs_json)'
+     - 'first_epoch=$(echo "$$claimable_epochs_json" | jq ".[0]")'
+     - '[[ -n "$$first_epoch" && "$$first_epoch" != "null" ]] && gcloud storage cp ./claiming-report.txt "$$gs_bucket/$$first_epoch/claiming-report.$(date +%s).txt"'
+     - 'buildkite-agent meta-data set attempts_count "$(grep -c ATTEMPT ./claiming-report.txt)"'
     artifact_paths:
     - "./claiming-report.txt"
+    key: 'notification'
+    depends_on: "claim-settlement"
+    allow_dependency_failure: true
 
   - wait: ~
-    continue_on_failure: true
 
   - label: ":mega: Notification settlements claiming"
     commands:
-    - 'buildkite-agent artifact download --include-retried-jobs claiming-report.txt .'
+    - 'notification_result=$(buildkite-agent meta-data get notification_result || echo "UNKNOWN")'
+    - 'notification_color=$(buildkite-agent meta-data get notification_color || echo "15158332")'
+    - 'attempts_count=$(buildkite-agent meta-data get attempts_count || echo "UNKNOWN")'
+    - 'buildkite-agent artifact download --include-retried-jobs claiming-report.txt . || echo "UNKNOWN ERROR" > claiming-report.txt'
     - 'claimable_epochs_json=$(buildkite-agent meta-data get claimable_epochs_json)'
     - |
       claimable_epochs=$(echo "$$claimable_epochs_json" | jq ". | join(\",\")" | tr -d '"')
       curl "$$DISCORD_WEBHOOK_VALIDATOR_BONDS" \
         -F 'payload_json={
-            "content": "Claim Settlements for epochs: '"$$claimable_epochs"'",
-            "url": "'"$$BUILDKITE_BUILD_URL"'",
-            "color": "8388863"
+            "embeds":[{
+              "title": "Claim Settlements '"$$notification_result"' for epochs: '"$$claimable_epochs"' after '"$$attempts_count"' attempts",
+              "url": "'"$$BUILDKITE_BUILD_URL"'",
+              "color": "'"$$notification_color"'"
+            }]
         }' \
         -F "file1=@./claiming-report.txt"
+    - |
+      build_result=$(buildkite-agent step get "outcome" --step "claim-settlement")
+    depends_on: "notification"
+    allow_dependency_failure: true
 
   - wait: ~
 

diff --git a/.buildkite/close-settlements.yml b/.buildkite/close-settlements.yml
@@ -59,12 +59,17 @@ steps:
   - wait: ~
 
   - label: ":campfire: Close settlements"
-    key: 'close-settlement'
     env:
       RUST_LOG: info,solana_transaction_builder_executor=debug,solana_transaction_builder=debug,solana_transaction_executor=debug,close_settlement=debug
       # RUST_BACKTRACE: full
     commands:
     - '. "$HOME/.cargo/env"'
+    - 'prior_build_number=$(($$BUILDKITE_RETRY_COUNT - 1))'
+    - 'buildkite-agent artifact download --include-retried-jobs build-report.$$prior_build_number . || true'
+    - |
+      cp "build-report.$$prior_build_number" "build-report.$$BUILDKITE_RETRY_COUNT" || true
+      rm -f "build-report.$$prior_build_number"
+      echo "#ATTEMPT $$BUILDKITE_RETRY_COUNT" | tee -a "./build-report.$$BUILDKITE_RETRY_COUNT"
     - 'buildkite-agent artifact download --include-retried-jobs target/release/close-settlement .'
     - 'buildkite-agent artifact download --include-retried-jobs past-settlements.json .'
     - 'chmod +x target/release/close-settlement'
@@ -75,29 +80,57 @@ steps:
         --operator-authority "$$VALIDATOR_BONDS_OPERATOR_AUTHORITY" \
         --fee-payer "$$VALIDATOR_BONDS_SETTLEMENTS_WALLET" \
         --marinade-wallet "$$VALIDATOR_BONDS_FUNDING_WALLET" \
-        --past-settlements ./past-settlements.json | tee close-settlement-report.txt
+        --past-settlements ./past-settlements.json | tee -a "./build-report.$$BUILDKITE_RETRY_COUNT"
+    key: 'close-settlement'
     artifact_paths:
-    - "./close-settlement-report.txt"
+      - "./build-report.*"
+    retry:
+      automatic:
+        - exit_status: 100
+          limit: 5
+
+  - label: "Notification setup"
+    env:
+      gs_bucket: gs://marinade-validator-bonds-mainnet
+    commands:
+     - 'mkdir ./reports'
+     - 'buildkite-agent artifact download --include-retried-jobs "build-report.*" ./reports/'
+     - 'report_path="./reports/$(ls -v1 reports/ | tail -n 1)"'
+     - '[[ -f "$$report_path" ]] || echo "No report generated; UNKNOWN ERROR" >> "$$report_path"'
+     - 'cp "$$report_path" ./close-report.txt'
+     - 'buildkite-agent meta-data set attempts_count "$(grep -c ATTEMPT ./close-report.txt)"'
+    artifact_paths:
+    - "./close-report.txt"
+    key: 'notification'
+    depends_on: "close-settlement"
+    allow_dependency_failure: true
+
+  - wait: ~
 
-  # discord reporting only in case of failure => manual intervention is needed
   - label: ":mega: Notification settlements closing"
     commands:
     - 'build_result=$(buildkite-agent step get "outcome" --step "close-settlement")'
     - |
-        if [[ "$$build_result" =~ "failed" ]]; then
-          buildkite-agent artifact download --include-retried-jobs close-settlement-report.txt . || echo 'UNKNOWN ERROR' > './close-settlement-report.txt'
-          curl "$$DISCORD_WEBHOOK_VALIDATOR_BONDS" \
-            -F 'payload_json={
-                "content": "Closed Settlements FAILURE, epoch: '"$$claimable_epochs"'",
-                "url": "'"$$BUILDKITE_BUILD_URL"'",
-                "color": "3158271"
-            }' \
-            -F "file1=@./close-settlement-report.txt"
-        else
-          echo "No error, no need to notify about closing settlements, all good"
-        fi
-
-    depends_on: "close-settlement"
+      if [[ -z "$$build_result" || "$$build_result" =~ "failed" ]]; then
+        notification_result="FAILED"
+        notification_color="15158332"
+      else
+        notification_result="SUCCEEDED"
+        notification_color="52224"
+      fi
+    - |
+      attempts_count=$(buildkite-agent meta-data get attempts_count || echo "UNKNOWN")
+      buildkite-agent artifact download --include-retried-jobs close-report.txt . || echo 'UNKNOWN ERROR' > './close-report.txt'
+      curl "$$DISCORD_WEBHOOK_VALIDATOR_BONDS" \
+        -F 'payload_json={
+            "embeds":[{
+              "title": "Close Settlements '"$$notification_result"', '"$$attempts_count"' attempts",
+              "url": "'"$$BUILDKITE_BUILD_URL"'",
+              "color": "'"$$notification_color"'"
+            }]
+        }' \
+        -F "file1=@./close-report.txt"
+    depends_on: "notification"
     allow_dependency_failure: true
 
   - wait: ~

diff --git a/.buildkite/init-settlements.yml b/.buildkite/init-settlements.yml
@@ -64,6 +64,7 @@ steps:
 
   - block: "Confirm to get processed init settlements"
     prompt: "Please review the 'Unblock' if ready to proceed."
+    blocked_state: running
 
   - wait: ~
 
@@ -83,60 +84,58 @@ steps:
     - 'prior_build_number=$(($$BUILDKITE_RETRY_COUNT - 1))'
     - 'buildkite-agent artifact download --include-retried-jobs build-report.$$prior_build_number . || true'
     - |
-      echo "This is attempt number $$BUILDKITE_RETRY_COUNT"
       cp "build-report.$$prior_build_number" "build-report.$$BUILDKITE_RETRY_COUNT" || true
       rm -f "build-report.$$prior_build_number"
+      echo "#ATTEMPT $$BUILDKITE_RETRY_COUNT" | tee -a "./build-report.$$BUILDKITE_RETRY_COUNT"
     - 'buildkite-agent artifact download --include-retried-jobs target/release/init-settlement .'
     - 'buildkite-agent artifact download --include-retried-jobs settlement-merkle-trees.json .'
     - 'buildkite-agent artifact download --include-retried-jobs settlements.json .'
     - 'chmod +x target/release/init-settlement'
-    - 'echo "#ATTEMPT $$BUILDKITE_RETRY_COUNT" | tee -a "./build-report.$$BUILDKITE_RETRY_COUNT"'
     - |
-      report=$(./target/release/init-settlement \
+      set -o pipefail
+      ./target/release/init-settlement \
         --rpc-url $$RPC_URL \
         --input-merkle-tree-collection "./settlement-merkle-trees.json" \
         --input-settlement-collection "./settlements.json" \
         --operator-authority "$$VALIDATOR_BONDS_OPERATOR_AUTHORITY" \
         --fee-payer "$$VALIDATOR_BONDS_SETTLEMENTS_WALLET" \
         --rent-payer "$$VALIDATOR_BONDS_SETTLEMENTS_WALLET" \
-        --marinade-wallet "$$VALIDATOR_BONDS_FUNDING_WALLET" \
-        --epoch "$$epoch" || true)
-    - 'echo "$$report" | tee -a "./build-report.$$BUILDKITE_RETRY_COUNT"'
-    - '[ -n "$$result_code" ] && echo "init-settlement command FAILED" && exit $$result_code'
+        --epoch "$$epoch" | tee -a "./build-report.$$BUILDKITE_RETRY_COUNT"
     key: 'init-settlement'
     artifact_paths:
     - "./build-report.*"
     retry:
       automatic:
-        - exit_status: '*'
-          limit: 10
+        - exit_status: 100
+          limit: 5
 
   - label: "Notification setup"
     env:
       gs_bucket: gs://marinade-validator-bonds-mainnet
     commands:
      - 'epoch=$(buildkite-agent meta-data get epoch)'
-     - 'build_result=$(buildkite-agent step get "outcome" --step "init-settlement")'
      - |
-        if [[ "$$build_result" =~ "failed" ]]; then
+       build_result=$(buildkite-agent step get "outcome" --step "init-settlement")
+        if [[ -z "$$build_result" || "$$build_result" =~ "failed" ]]; then
           notification_result="FAILED"
           notification_color="15158332"
         else
           notification_result="SUCCEEDED"
-          notification_color="8388863"
+          notification_color="52224"
         fi
      - 'buildkite-agent meta-data set notification_color "$$notification_color"'
      - 'buildkite-agent meta-data set notification_result "$$notification_result"'
      - 'echo "Notification result: $$notification_result, color: $$notification_color"'
      - 'mkdir ./reports'
      - 'buildkite-agent artifact download --include-retried-jobs "build-report.*" ./reports/'
      - 'report_path="./reports/$(ls -v1 reports/ | tail -n 1)"'
-     - '[ -f "$$report_path" ] || echo "UNKNOWN ERROR" >> "$$report_path"'
-     - 'cp "$$report_path" ./init-settlement-report.txt'
-     - 'gcloud storage cp ./init-settlement-report.txt "$$gs_bucket/$$epoch/init-settlement-report.$(date +%s).txt"'
-     - 'buildkite-agent meta-data set attempts_count "$(grep -c "#ATTEMPT" "$$report_path")"'
+     - '[[ -f "$$report_path" ]] || echo "No report generated; UNKNOWN ERROR" >> "$$report_path"'
+     - 'cp "$$report_path" ./init-report.txt'
+     - 'gcloud storage cp ./init-report.txt "$$gs_bucket/$$epoch/init-settlement-report.$(date +%s).txt"'
+     - 'buildkite-agent meta-data set attempts_count "$(grep -c ATTEMPT ./init-report.txt)"'
     artifact_paths:
-    - "./init-settlement-report.txt"
+    - "./init-report.txt"
+    key: 'notification'
     depends_on: "init-settlement"
     allow_dependency_failure: true
 
@@ -146,18 +145,22 @@ steps:
   - label: ":mega: Notification settlements initialized"
     commands:
     - 'epoch=$(buildkite-agent meta-data get epoch)'
-    - 'notification_result=$(buildkite-agent meta-data get notification_result)'
-    - 'notification_color=$(buildkite-agent meta-data get notification_color)'
-    - 'attempts_count=$(buildkite-agent meta-data get attempts_count)'
-    - 'buildkite-agent artifact download --include-retried-jobs init-settlement-report.txt .'
+    - 'notification_result=$(buildkite-agent meta-data get notification_result || echo "UNKNOWN")'
+    - 'notification_color=$(buildkite-agent meta-data get notification_color || echo "15158332")'
+    - 'attempts_count=$(buildkite-agent meta-data get attempts_count || echo "UNKNOWN")'
+    - 'buildkite-agent artifact download --include-retried-jobs init-report.txt . || echo "UNKNOWN ERROR" > init-report.txt'
     - |
         curl "$$DISCORD_WEBHOOK_VALIDATOR_BONDS" \
         -F 'payload_json={
-            "content": "Init Settlements for Validator Bonds ('"$$epoch"') after '"$$attempts_count"' attempts.",
-            "url": "'"$$BUILDKITE_BUILD_URL"'",
-            "color": "8388863"
+            "embeds":[{
+              "title": "Init Settlements '"$notification_result"' for Validator Bonds ('"$$epoch"') after '"$$attempts_count"' attempts",
+              "url": "'"$$BUILDKITE_BUILD_URL"'",
+              "color": "'"$$notification_color"'"
+            }]
         }' \
-        -F "file1=@./init-settlement-report.txt"
+        -F "file1=@./init-report.txt"
+    depends_on: "notification"
+    allow_dependency_failure: true
 
   - command: echo "End of concurrency gate <--"
     concurrency_group: 'validator-bonds/init-settlements'

diff --git a/common-rs/src/stake_accounts.rs b/common-rs/src/stake_accounts.rs
@@ -1,5 +1,5 @@
 use crate::utils::get_sysvar_clock;
-use log::info;
+use log::{info, warn};
 use solana_account_decoder::{UiAccountEncoding, UiDataSliceConfig};
 use solana_client::{
     nonblocking::rpc_client::RpcClient,
@@ -88,8 +88,8 @@ pub async fn collect_stake_accounts(
 // i.e., to the vote account that the stake account is delegated to
 // returns Map<voter_pubkey, Vec<stake_account_data>>
 pub async fn obtain_delegated_stake_accounts(
-    stake_accounts: CollectedStakeAccounts,
     rpc_client: Arc<RpcClient>,
+    stake_accounts: CollectedStakeAccounts,
 ) -> anyhow::Result<HashMap<Pubkey, CollectedStakeAccounts>> {
     let clock: Clock = get_sysvar_clock(rpc_client).await?;
     let mut vote_account_map: HashMap<Pubkey, CollectedStakeAccounts> = HashMap::new();
@@ -126,9 +126,19 @@ pub async fn obtain_claimable_stake_accounts_for_settlement(
     let stake_history = get_stake_history(rpc_client.clone()).await?;
     let filtered_deactivated_stake_accounts: CollectedStakeAccounts = stake_accounts
         .into_iter()
-        .filter(|(_, _, stake)| {
+        .filter(|(pubkey, _, stake)| {
             if is_locked(stake, &clock) {
                 // cannot use locked stake account
+                warn!(
+                    "Locked stake account {} found (withdrawer {}/staker {})",
+                    pubkey,
+                    stake
+                        .authorized()
+                        .map_or("None".to_string(), |a| a.withdrawer.to_string()),
+                    stake
+                        .authorized()
+                        .map_or("None".to_string(), |a| a.staker.to_string()),
+                );
                 false
             } else if let Some(delegation) = stake.delegation() {
                 // stake has got delegation but is fully deactivated
@@ -154,10 +164,10 @@ pub async fn obtain_claimable_stake_accounts_for_settlement(
 // All non locked stake accounts that are funded to the Settlement
 // Stake accounts are good to be claimed in near future (i.e., in next epoch, deactivated)
 pub async fn obtain_funded_stake_accounts_for_settlement(
+    rpc_client: Arc<RpcClient>,
     stake_accounts: CollectedStakeAccounts,
     config_address: &Pubkey,
     settlement_addresses: Vec<Pubkey>,
-    rpc_client: Arc<RpcClient>,
 ) -> anyhow::Result<HashMap<Pubkey, (u64, CollectedStakeAccounts)>> {
     let clock = get_sysvar_clock(rpc_client.clone()).await?;
     let stake_history = get_stake_history(rpc_client.clone()).await?;