[settlement-pipelines] better reporting (#53)

* [settlement-pipelines] better reporting

* Update scripts/settlement-json-listing.sh

* dependency update + fixes

---------

Co-authored-by: janlegner <32453746+janlegner@users.noreply.github.com>

.buildkite/claim-settlements.yml

@@ -21,8 +21,7 @@ steps:
 
   - label: ":campfire: List claimable epochs"
     env:
-      # RUST_LOG: trace
-      # RUST_BACKTRACE: full
+      # RUST_LOG: debug
     commands:
     - '. "$HOME/.cargo/env"'
     - 'buildkite-agent artifact download --include-retried-jobs target/release/list-claimable-epoch .'
@@ -58,7 +57,7 @@ steps:
 
   - label: ":campfire: Claim settlements"
     env:
-      # RUST_LOG: trace
+      RUST_LOG: solana_transaction_builder_executor=debug,solana_transaction_builder=debug,builder_executor=debug,solana_transaction_executor=debug,settlement_pipelines=debug,claim_settlement=debug
       # RUST_BACKTRACE: full
     commands:
     - |
@@ -73,7 +72,28 @@ steps:
         --rpc-url $$RPC_URL \
         --merkle-trees-dir "./merkle-trees/" \
         --operator-authority "$$VALIDATOR_BONDS_OPERATOR_AUTHORITY" \
-        --fee-payer "$$VALIDATOR_BONDS_FUNDING_WALLET"
+        --fee-payer "$$VALIDATOR_BONDS_FUNDING_WALLET" | tee ./claiming-report.txt
+    artifact_paths:
+    - "./claiming-report.txt"
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":mega: Notification settlements claiming"
+    env:
+      DISCORD_WEBHOOK: $$DISCORD_WEBHOOK_VALIDATOR_BONDS
+    commands:
+    - 'buildkite-agent artifact download --include-retried-jobs claiming-report.txt .'
+    - 'claimable_epochs_json=$(buildkite-agent meta-data get claimable_epochs_json)'
+    - 'claimable_epochs=$(echo "$$claimable_epochs_json" | jq ".[]")'
+    - |
+      curl "$$DISCORD_WEBHOOK" \
+        -F 'payload_json={
+            "content": "Claimed Settlements for epochs: '"$$claimable_epochs"'",
+            "url": "'"$$BUILDKITE_BUILD_URL"'",
+            "color": "8388863"
+        }' \
+        -F "file1=@./claiming-report.txt"
 
   - wait: ~
 

.buildkite/init-settlements.yml

@@ -16,12 +16,14 @@ steps:
   - wait: ~
 
   - label: ":mega: Notification initializing settlements"
+    env:
+      DISCORD_WEBHOOK: $$DISCORD_WEBHOOK_VALIDATOR_BONDS
     commands:
     - 'epoch=${EPOCH:-$(buildkite-agent meta-data get epoch)}'
     - 'buildkite-agent meta-data set epoch "$$epoch"'
     - 'echo "Epoch: $$epoch"'
     - |
-      curl "$$DISCORD_WEBHOOK_VALIDATOR_BONDS" -H "Content-Type: application/json" -d '{
+      curl "$$DISCORD_WEBHOOK" -H "Content-Type: application/json" -d '{
         "embeds": [
           {
             "title": "Confirm Settlements Initialization for Validator Bonds ('"$$epoch"').",
@@ -31,28 +33,14 @@ steps:
           }
         ]
       }'
-
-  - block: "Confirm to get processed init settlements"
-    prompt: "Please review the 'Unblock' if ready to proceed."
-    if: "build.env('EPOCH') != null"
-
-  - wait: ~
-
+    
   - label: ":hammer_and_wrench: :rust: Build"
     commands:
     - '. "$HOME/.cargo/env"'
     - 'cargo build --release --bin init-settlement'
     artifact_paths:
       - target/release/init-settlement
-
-  - wait: ~
-
-  - command: echo "--> Start of concurrency gate"
-    concurrency_group: 'validator-bonds/init-settlements'
-    concurrency: 1
-
-  - wait: ~
-
+
   - label: ":floppy_disk: :arrow_left: :cloud: Downloading merkle trees"
     env:
       gs_bucket: gs://marinade-validator-bonds-mainnet
@@ -61,16 +49,35 @@ steps:
     - |
       gcloud storage cp "$$gs_bucket/$$epoch/settlement-merkle-trees.json" "."
       gcloud storage cp "$$gs_bucket/$$epoch/settlements.json" "."
+    key: 'download-json'
     artifact_paths:
       - "./settlement-merkle-trees.json"
-      - "./settlements.json"
+      - "./settlements.json"  
+
+  - label: "Settlement JSON listing"
+    commands:
+     - 'buildkite-agent artifact download --include-retried-jobs settlement-merkle-trees.json .'
+     - 'chmod uog+x ./scripts/settlement-json-listing.sh'
+     - './scripts/settlement-json-listing.sh settlement-merkle-trees.json'
+    depends_on: "download-json"
+
+  - wait: ~
+
+  - block: "Confirm to get processed init settlements"
+    prompt: "Please review the 'Unblock' if ready to proceed."
+
+  - wait: ~
+
+  - command: echo "--> Start of concurrency gate"
+    concurrency_group: 'validator-bonds/init-settlements'
+    concurrency: 1
 
   - wait: ~
 
   - label: ":campfire: Create settlements"
-    # env:
-    #   RUST_LOG: trace
-    #   RUST_BACKTRACE: full
+    env:
+      RUST_LOG: solana_transaction_builder_executor=debug,solana_transaction_builder=debug,builder_executor=debug,solana_transaction_executor=debug,settlement_pipelines=debug,init_settlement=debug
+      #  RUST_BACKTRACE: full
     commands:
     - '. "$HOME/.cargo/env"'
     - 'epoch=$(buildkite-agent meta-data get epoch)'
@@ -124,35 +131,34 @@ steps:
      - 'buildkite-agent artifact download --include-retried-jobs "build-report.*" ./reports/'
      - 'report_path="./reports/$(ls -v1 reports/ | tail -n 1)"'
      - '[ -f "$$report_path" ] || echo "UNKNOWN ERROR" >> "$$report_path"'
-     - 'gcloud storage cp "$$report_path" "$$gs_bucket/$$epoch/init-settlement-report.txt"'
+     - 'cp "$$report_path" ./init-settlement-report.txt'
+     - 'gcloud storage cp ./init-settlement-report.txt "$$gs_bucket/$$epoch/init-settlement-report.txt"'
      - 'buildkite-agent meta-data set attempts_count "$(grep -c "#ATTEMPT" "$$report_path")"'
-     - 'buildkite-agent meta-data set report "$(cat "$$report_path")"'
+    artifact_paths:
+    - "./init-settlement-report.txt"
     depends_on: "init-settlement"
     allow_dependency_failure: true
 
   - wait: ~
     continue_on_failure: true
 
   - label: ":mega: Notification settlements initialized"
+    env:
+      DISCORD_WEBHOOK: $$DISCORD_WEBHOOK_VALIDATOR_BONDS
     commands:
     - 'epoch=$(buildkite-agent meta-data get epoch)'
     - 'notification_result=$(buildkite-agent meta-data get notification_result)'
     - 'notification_color=$(buildkite-agent meta-data get notification_color)'
     - 'attempts_count=$(buildkite-agent meta-data get attempts_count)'
-    - 'report=$(buildkite-agent meta-data get report)'
+    - 'buildkite-agent artifact download --include-retried-jobs init-settlement-report.txt .'
     - |
-        escaped_report=$(jq -n --arg report "$$report" '"```"+$$report+"```"')
-        curl "$$DISCORD_WEBHOOK_VALIDATOR_BONDS" -H "Content-Type: application/json" -d '{
-          "embeds": [
-            {
-              "title": "Initializing Settlements for Validator Bonds ('"$$epoch"') '"$$notification_result"' after '"$$attempts_count"' attempts.",
-              "url": "'"$$BUILDKITE_BUILD_URL"'",
-              "color": "'"$$notification_color"'"
-            }
-          ],
-          "content": '"$$escaped_report"',
-          "attachments": []
-        }'
+        curl "$$DISCORD_WEBHOOK" \
+        -F 'payload_json={
+            "content": "Initializing Settlements for Validator Bonds ('"$$epoch"') after '"$$attempts_count"' attempts.",
+            "url": "'"$$BUILDKITE_BUILD_URL"'",
+            "color": "8388863"
+        }' \
+        -F "file1=@./init-settlement-report.txt"
 
   - command: echo "End of concurrency gate <--"
     concurrency_group: 'validator-bonds/init-settlements'

Cargo.toml

@@ -57,8 +57,8 @@ solana-runtime = "=1.17.22"
 solana-sdk = "=1.17.22"
 solana-security-txt = "1.1.1"
 solana-transaction-executor = { git = "https://github.com/marinade-finance/solana-transaction-executor", branch = "master" }
-solana-transaction-builder = { git = "https://github.com/marinade-finance/solana-transaction-builder", branch = "logging-adjust" }
-solana-transaction-builder-executor = { git = "https://github.com/marinade-finance/solana-transaction-builder", branch = "logging-adjust" }
+solana-transaction-builder = { git = "https://github.com/marinade-finance/solana-transaction-builder", branch = "main" }
+solana-transaction-builder-executor = { git = "https://github.com/marinade-finance/solana-transaction-builder", branch = "main" }
 structopt = "0.3.21"
 tokio = { version = "1", features = ["full"] }
 tokio-postgres = { version = "0.7.7", features = ["with-chrono-0_4"] }

common-rs/src/bonds.rs

@@ -1,11 +1,10 @@
-use anchor_client::anchor_lang::AccountDeserialize;
-use log::error;
 use solana_client::nonblocking::rpc_client::RpcClient;
 use solana_sdk::pubkey::Pubkey;
 use std::sync::Arc;
 use validator_bonds::state::bond::Bond;
 
 use crate::get_validator_bonds_program;
+use crate::utils::get_accounts_for_pubkeys;
 
 pub async fn get_bonds(rpc_client: Arc<RpcClient>) -> anyhow::Result<Vec<(Pubkey, Bond)>> {
     let program = get_validator_bonds_program(rpc_client, None)?;
@@ -16,25 +15,5 @@ pub async fn get_bonds_for_pubkeys(
     rpc_client: Arc<RpcClient>,
     pubkeys: &[Pubkey],
 ) -> anyhow::Result<Vec<(Pubkey, Option<Bond>)>> {
-    let bond_accounts = rpc_client.get_multiple_accounts(pubkeys).await?;
-    pubkeys
-        .iter()
-        .zip(bond_accounts.iter())
-        .map(|(pubkey, account)| {
-            let account = account.as_ref().and_then(|account| {
-                let mut data: &[u8] = &account.data;
-                Bond::try_deserialize(&mut data).map_or_else(
-                    |e| {
-                        error!(
-                            "Cannot deserialize account data for bond account {}: {}",
-                            pubkey, e
-                        );
-                        None
-                    },
-                    Some,
-                )
-            });
-            Ok((*pubkey, account))
-        })
-        .collect()
+    get_accounts_for_pubkeys(rpc_client, pubkeys).await
 }

common-rs/src/config.rs

@@ -0,0 +1,21 @@
+use crate::get_validator_bonds_program;
+use anyhow::anyhow;
+use solana_client::nonblocking::rpc_client::RpcClient;
+use solana_program::pubkey::Pubkey;
+use std::sync::Arc;
+use validator_bonds::state::config::Config;
+
+pub async fn get_config(
+    rpc_client: Arc<RpcClient>,
+    config_address: Pubkey,
+) -> anyhow::Result<Config> {
+    let program = get_validator_bonds_program(rpc_client, None)?;
+    let config = program.account(config_address).await.map_err(|e| {
+        anyhow!(
+            "Cannot load validator-bonds config account {}: {:?}",
+            config_address,
+            e
+        )
+    })?;
+    Ok(config)
+}

common-rs/src/lib.rs

@@ -5,6 +5,7 @@ use std::rc::Rc;
 use std::{str::FromStr, sync::Arc};
 
 pub mod bonds;
+pub mod config;
 pub mod constants;
 pub mod funded_bonds;
 pub mod settlement_claims;

common-rs/src/settlement_claims.rs

@@ -1,9 +1,12 @@
 use crate::get_validator_bonds_program;
+
 use anyhow::anyhow;
 use solana_account_decoder::UiDataSliceConfig;
 use solana_client::nonblocking::rpc_client::RpcClient;
 use solana_client::rpc_config::RpcAccountInfoConfig;
+use solana_client::rpc_filter::{Memcmp, RpcFilterType};
 use solana_program::pubkey::Pubkey;
+
 use std::sync::Arc;
 use validator_bonds::state::settlement_claim::SettlementClaim;
 
@@ -14,6 +17,19 @@ pub async fn get_settlement_claims(
     Ok(program.accounts(Default::default()).await?)
 }
 
+pub async fn get_settlement_claims_for_settlement(
+    rpc_client: Arc<RpcClient>,
+    settlement_address: &Pubkey,
+) -> anyhow::Result<Vec<(Pubkey, SettlementClaim)>> {
+    let program = get_validator_bonds_program(rpc_client, None)?;
+    Ok(program
+        .accounts(vec![RpcFilterType::Memcmp(Memcmp::new(
+            8,
+            solana_client::rpc_filter::MemcmpEncodedBytes::Base58(settlement_address.to_string()),
+        ))])
+        .await?)
+}
+
 pub async fn collect_existence_settlement_claims_from_addresses(
     rpc_client: Arc<RpcClient>,
     settlement_claim_addresses: &[Pubkey],

common-rs/src/settlements.rs

@@ -1,11 +1,12 @@
-use anchor_client::anchor_lang::AccountDeserialize;
-use log::error;
 use solana_client::nonblocking::rpc_client::RpcClient;
 use solana_sdk::pubkey::Pubkey;
+
 use std::sync::Arc;
+
 use validator_bonds::state::settlement::Settlement;
 
 use crate::get_validator_bonds_program;
+use crate::utils::get_accounts_for_pubkeys;
 
 pub async fn get_settlements(
     rpc_client: Arc<RpcClient>,
@@ -18,25 +19,5 @@ pub async fn get_settlements_for_pubkeys(
     rpc_client: Arc<RpcClient>,
     pubkeys: &[Pubkey],
 ) -> anyhow::Result<Vec<(Pubkey, Option<Settlement>)>> {
-    let settlement_accounts = rpc_client.get_multiple_accounts(pubkeys).await?;
-    pubkeys
-        .iter()
-        .zip(settlement_accounts.iter())
-        .map(|(pubkey, account)| {
-            let account = account.as_ref().and_then(|account| {
-                let mut data: &[u8] = &account.data;
-                Settlement::try_deserialize(&mut data).map_or_else(
-                    |e| {
-                        error!(
-                            "Cannot deserialize account data for settlement account {}: {}",
-                            pubkey, e
-                        );
-                        None
-                    },
-                    Some,
-                )
-            });
-            Ok((*pubkey, account))
-        })
-        .collect()
+    get_accounts_for_pubkeys(rpc_client, pubkeys).await
 }