[contract] using bitmap data structure instead of PDA

README.md

@@ -89,6 +89,8 @@ pnpm build
 
 # testing the SDK+CLI against the bankrun and local validator
 pnpm test
+# running single cargo test
+cargo test --package settlement-engine ts_cross_check_hash_generate
 # bankrun part of the tests
 pnpm test:bankrun
 # local validator part of the tests

common-rs/src/settlement_claims.rs

@@ -1,65 +1,61 @@
-use crate::get_validator_bonds_program;
-
+use anchor_client::anchor_lang::AccountDeserialize;
 use anyhow::anyhow;
-use solana_account_decoder::UiDataSliceConfig;
-use solana_client::nonblocking::rpc_client::RpcClient;
-use solana_client::rpc_config::RpcAccountInfoConfig;
-use solana_client::rpc_filter::{Memcmp, RpcFilterType};
-use solana_program::pubkey::Pubkey;
-
-use std::sync::Arc;
-use validator_bonds::state::settlement_claim::SettlementClaim;
+use solana_sdk::account::Account;
+use std::fmt::Debug;
+use validator_bonds::constants::SETTLEMENT_CLAIMS_ANCHOR_HEADER_SIZE;
+use validator_bonds::state::settlement_claims::SettlementClaims;
+use validator_bonds::utils::BitmapProjection;
 
-pub async fn get_settlement_claims(
-    rpc_client: Arc<RpcClient>,
-) -> anyhow::Result<Vec<(Pubkey, SettlementClaim)>> {
-    let program = get_validator_bonds_program(rpc_client, None)?;
-    Ok(program.accounts(Default::default()).await?)
+/// Off-chain handler for accessing bitmap from SettlementClaims account
+/// The struct stores memory "copied" from the loaded Solana account.
+/// The struct provides methods redirecting to [BitmapProjection] to access the bitmap data.
+pub struct SettlementClaimsBitmap {
+    pub data: Vec<u8>,
+    bitmap_projection: BitmapProjection,
 }
 
-pub async fn get_settlement_claims_for_settlement(
-    rpc_client: Arc<RpcClient>,
-    settlement_address: &Pubkey,
-) -> anyhow::Result<Vec<(Pubkey, SettlementClaim)>> {
-    let program = get_validator_bonds_program(rpc_client, None)?;
-    Ok(program
-        .accounts(vec![RpcFilterType::Memcmp(Memcmp::new(
-            8,
-            solana_client::rpc_filter::MemcmpEncodedBytes::Base58(settlement_address.to_string()),
-        ))])
-        .await?)
+impl SettlementClaimsBitmap {
+    pub fn new(account: Account) -> anyhow::Result<Self> {
+        let mut data = account.data.to_vec();
+        let settlement_claims = SettlementClaims::try_deserialize(&mut data.as_slice())
+            .map_or_else(
+                |e| Err(anyhow!("Cannot deserialize SettlementClaims data: {}", e)),
+                Ok,
+            )?;
+        data.drain(0..SETTLEMENT_CLAIMS_ANCHOR_HEADER_SIZE as usize);
+        BitmapProjection::check_size(settlement_claims.max_records, &data)?;
+        let bitmap_projection = BitmapProjection(settlement_claims.max_records);
+        Ok(Self {
+            data,
+            bitmap_projection,
+        })
+    }
+
+    pub fn max_records(&self) -> u64 {
+        self.bitmap_projection.0
+    }
+
+    pub fn is_set(&self, index: u64) -> bool {
+        self.bitmap_projection
+            .is_set(index, &self.data)
+            .expect("BitmapProjection should be initialized, checked in new()")
+    }
+
+    pub fn number_of_set_bits(&self) -> u64 {
+        self.bitmap_projection
+            .number_of_bits(&self.data)
+            .expect("SettlementClaimsBitmap should be initialized, checked in new()")
+    }
 }
 
-pub async fn collect_existence_settlement_claims_from_addresses(
-    rpc_client: Arc<RpcClient>,
-    settlement_claim_addresses: &[Pubkey],
-) -> anyhow::Result<Vec<(Pubkey, bool)>> {
-    let settlement_claim_addresses_chunked = settlement_claim_addresses
-        // permitted to fetch 100 accounts at once; https://solana.com/docs/rpc/http/getmultipleaccounts
-        .chunks(100)
-        .collect::<Vec<&[Pubkey]>>();
-    let mut settlement_claims: Vec<(Pubkey, bool)> = vec![];
-    for address_chunk in settlement_claim_addresses_chunked.iter() {
-        let accounts = rpc_client
-            .get_multiple_accounts_with_config(
-                address_chunk,
-                RpcAccountInfoConfig {
-                    data_slice: Some(UiDataSliceConfig {
-                        offset: 0,
-                        length: 0,
-                    }),
-                    ..RpcAccountInfoConfig::default()
-                },
+impl Debug for SettlementClaimsBitmap {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SettlementClaimsBitmap")
+            .field("set_bits", &self.number_of_set_bits())
+            .field(
+                "bitmap_projection",
+                &self.bitmap_projection.debug_string(&self.data),
             )
-            .await
-            .map_err(|e| anyhow!("Error fetching settlement claim accounts: {:?}", e))?;
-        accounts
-            .value
-            .iter()
-            .zip(address_chunk.iter())
-            .for_each(|(a, p)| {
-                settlement_claims.push((*p, a.is_some()));
-            });
+            .finish()
     }
-    Ok(settlement_claims)
 }

common-rs/src/settlements.rs

@@ -1,14 +1,15 @@
 use crate::bonds::get_bonds_for_pubkeys;
 use crate::get_validator_bonds_program;
-use crate::utils::get_accounts_for_pubkeys;
+use crate::settlement_claims::SettlementClaimsBitmap;
+use crate::utils::{get_account_infos_for_pubkeys, get_accounts_for_pubkeys};
 use anyhow::anyhow;
 use log::{debug, error};
 use solana_client::nonblocking::rpc_client::RpcClient;
 use solana_sdk::pubkey::Pubkey;
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use validator_bonds::state::bond::Bond;
-use validator_bonds::state::settlement::Settlement;
+use validator_bonds::state::settlement::{find_settlement_claims_address, Settlement};
 
 pub async fn get_settlements(
     rpc_client: Arc<RpcClient>,
@@ -82,6 +83,46 @@ pub async fn get_settlements_for_pubkeys(
     get_accounts_for_pubkeys(rpc_client, pubkeys).await
 }
 
+pub async fn get_settlement_claims_for_settlement_pubkeys(
+    rpc_client: Arc<RpcClient>,
+    settlement_pubkeys: &[Pubkey],
+) -> anyhow::Result<Vec<(Pubkey, Pubkey, Option<SettlementClaimsBitmap>)>> {
+    let settlement_claims_pubkeys = settlement_pubkeys
+        .iter()
+        .map(|settlement_pubkey| find_settlement_claims_address(settlement_pubkey).0)
+        .collect::<Vec<Pubkey>>();
+    let settlement_claims = get_account_infos_for_pubkeys(rpc_client, &settlement_claims_pubkeys)
+        .await?
+        .into_iter()
+        .map(|(pubkey, account)| {
+            if let Some(account) = account {
+                Ok((pubkey, Some(SettlementClaimsBitmap::new(account)?)))
+            } else {
+                Ok((pubkey, None))
+            }
+        })
+        .collect::<anyhow::Result<
+            Vec<(
+                Pubkey,
+                Option<crate::settlement_claims::SettlementClaimsBitmap>,
+            )>,
+        >>()?;
+    let result = settlement_pubkeys
+        .iter()
+        .zip(settlement_claims.into_iter())
+        .map(
+            |(settlement_pubkey, (settlement_claims_pubkey, settlement_claims))| {
+                (
+                    *settlement_pubkey,
+                    settlement_claims_pubkey,
+                    settlement_claims,
+                )
+            },
+        )
+        .collect::<Vec<(Pubkey, Pubkey, Option<SettlementClaimsBitmap>)>>();
+    Ok(result)
+}
+
 pub async fn get_bonds_for_settlements(
     rpc_client: Arc<RpcClient>,
     settlements: &[(Pubkey, Settlement)],

common-rs/src/utils.rs

@@ -6,43 +6,57 @@ use solana_client::rpc_config::RpcAccountInfoConfig;
 
 use solana_program::pubkey::Pubkey;
 
+use solana_sdk::account::Account;
 use std::sync::Arc;
 
-pub async fn get_accounts_for_pubkeys<T: AccountDeserialize>(
+pub async fn get_account_infos_for_pubkeys(
     rpc_client: Arc<RpcClient>,
     pubkeys: &[Pubkey],
-) -> anyhow::Result<Vec<(Pubkey, Option<T>)>> {
-    let settlement_addresses = pubkeys
+) -> anyhow::Result<Vec<(Pubkey, Option<Account>)>> {
+    let addresses = pubkeys
         // permitted to fetch 100 accounts at once; https://solana.com/docs/rpc/http/getmultipleaccounts
         .chunks(100)
         .collect::<Vec<&[Pubkey]>>();
 
-    let mut settlement_accounts: Vec<(Pubkey, Option<T>)> = vec![];
-    for address_chunk in settlement_addresses.iter() {
+    let mut accounts_result: Vec<(Pubkey, Option<Account>)> = vec![];
+    for address_chunk in addresses.iter() {
         let accounts = rpc_client
             .get_multiple_accounts_with_config(address_chunk, RpcAccountInfoConfig::default())
             .await
             .map_err(|e| anyhow!("Error fetching settlement accounts: {:?}", e))?;
         accounts
             .value
-            .iter()
+            .into_iter()
             .zip(address_chunk.iter())
             .for_each(|(account, pubkey)| {
-                let account = account.as_ref().and_then(|account| {
-                    let mut data: &[u8] = &account.data;
-                    T::try_deserialize(&mut data).map_or_else(
-                        |e| {
-                            error!(
-                                "Cannot deserialize account data for settlement account {}: {}",
-                                pubkey, e
-                            );
-                            None
-                        },
-                        Some,
-                    )
-                });
-                settlement_accounts.push((*pubkey, account));
+                accounts_result.push((*pubkey, account));
             });
     }
-    Ok(settlement_accounts)
+    Ok(accounts_result)
+}
+
+pub async fn get_accounts_for_pubkeys<T: AccountDeserialize>(
+    rpc_client: Arc<RpcClient>,
+    pubkeys: &[Pubkey],
+) -> anyhow::Result<Vec<(Pubkey, Option<T>)>> {
+    let accounts = get_account_infos_for_pubkeys(rpc_client, pubkeys).await?;
+    Ok(accounts
+        .iter()
+        .map(|(pubkey, account)| {
+            let account = account.as_ref().and_then(|account| {
+                let mut data: &[u8] = &account.data;
+                T::try_deserialize(&mut data).map_or_else(
+                    |e| {
+                        error!(
+                            "Cannot deserialize account data for account {}: {}",
+                            pubkey, e
+                        );
+                        None
+                    },
+                    Some,
+                )
+            });
+            (*pubkey, account)
+        })
+        .collect())
 }

fixtures/accounts-v1/4wQELTA1RMEM3cKN7gjbiNN247e3GY9Sga7MKpNV38kL.json

@@ -0,0 +1,14 @@
+{
+  "pubkey": "4wQELTA1RMEM3cKN7gjbiNN247e3GY9Sga7MKpNV38kL",
+  "account": {
+    "lamports": 5178240,
+    "data": [
+      "mwyq4B76zII4HpgLhYEuc6g74EtwoEc1alwqp7Tw1KFGeARb9I+QvPMAweQ1Z/OyzZlMvPorcha+ntSchewZJIjcBOkIOq4/CgAAAAAAAAAJAAAAAAAAAADKmjsAAAAA/jgemAuFgS5zqDvgS3CgRzVqXCqntPDUoUZ4BFv0j5C8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
+      "base64"
+    ],
+    "owner": "vBoNdEvzMrSai7is21XgVYik65mqtaKXuSdMBJ1xkW4",
+    "executable": false,
+    "rentEpoch": 18446744073709551615,
+    "space": 616
+  }
+}

fixtures/accounts-v1/6WMubUXAaYgQGSKFC31H4NRkriwaP7VJqukTAyMhytzZ.json

@@ -0,0 +1,14 @@
+{
+  "pubkey": "6WMubUXAaYgQGSKFC31H4NRkriwaP7VJqukTAyMhytzZ",
+  "account": {
+    "lamports": 2784000,
+    "data": [
+      "2Gfn9qtjfIWeoVf+zLnmbeySdJFZJLayhq5qv2QDkyXlggriGJ1irO230S9k8OkxIn2wXOnvML8csPgEwJK+ZHfYuB56QW7YaG5JX0/ztlrZ/Onlaz/FYUwAaZHEeDf5fWavAA428kcrXRv4W26Lqv7HhVwnAJjW+j4ZRfudkL7bF2EP4FBAN9IEAAAAAAAA/Z455gcNUjxhL+VysvN0gZ7fz3PtP/F7CKZayixfqc3rAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+      "base64"
+    ],
+    "owner": "vBoNdEvzMrSai7is21XgVYik65mqtaKXuSdMBJ1xkW4",
+    "executable": false,
+    "rentEpoch": 18446744073709551615,
+    "space": 272
+  }
+}

fixtures/accounts-v1/7M9TkgxM8DHWgZnJNJif4Jyfyqb5NJuCgzvfgc9xjRE1.json

@@ -0,0 +1,14 @@
+{
+  "pubkey": "7M9TkgxM8DHWgZnJNJif4Jyfyqb5NJuCgzvfgc9xjRE1",
+  "account": {
+    "lamports": 44004877880,
+    "data": [
+      "AgAAAIDVIgAAAAAA/RbuBMPFlaQSRwOct/1l50OdRQDK4dfWJwuO2NVOz3ZKTA3mf/3M9yGIkX3F0YM2BS68S2d4wGoa+WOHUa6Q0QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANQ8dq8eNLNfR+Pa0LVpAHbXUVqBgwcAcBDDNqXFhJRjilXCPgoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA0D8AAAAAAAAAAAAAAAA=",
+      "base64"
+    ],
+    "owner": "Stake11111111111111111111111111111111111111",
+    "executable": false,
+    "rentEpoch": 18446744073709551615,
+    "space": 200
+  }
+}

fixtures/accounts-v1/BBUdhr6vsEoFsQn6csMCpaiFo5rYXeDv1E7jZidLYfEx.json

@@ -0,0 +1,14 @@
+{
+  "pubkey": "BBUdhr6vsEoFsQn6csMCpaiFo5rYXeDv1E7jZidLYfEx",
+  "account": {
+    "lamports": 554998997717120,
+    "data": [
+      "AgAAAIDVIgAAAAAASkwN5n/9zPchiJF9xdGDNgUuvEtneMBqGvljh1GukNFKTA3mf/3M9yGIkX3F0YM2BS68S2d4wGoa+WOHUa6Q0QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANQ8dq8eNLNfR+Pa0LVpAHbXUVqBgwcAcBDDNqXFhJRjADv5xcT4AQAAAAAAAAAAAP//////////AAAAAAAA0D8AAAAAAAAAAAAAAAA=",
+      "base64"
+    ],
+    "owner": "Stake11111111111111111111111111111111111111",
+    "executable": false,
+    "rentEpoch": 18446744073709551615,
+    "space": 200
+  }
+}

fixtures/accounts-v1/BgE7ME4DrJC5okjHtcoZuuMYLMsHMfip1nJVS5NNvPQf.json

@@ -0,0 +1,14 @@
+{
+  "pubkey": "BgE7ME4DrJC5okjHtcoZuuMYLMsHMfip1nJVS5NNvPQf",
+  "account": {
+    "lamports": 3173760,
+    "data": [
+      "NwvbISSIKLakvA+/7hrKayRamilmrQNlNqgg1yVyYkkVUVbyMi66KP0W7gTDxZWkEkcDnLf9ZedDnUUAyuHX1icLjtjVTs92TmlWMOkXUzFXs3VEaZ2EXBBBg+fcJE8jR7lu2690etgKtgQDCgAAAAUAAAAAAAAACrYEAwoAAADSBAAAAAAAAAEAAAAAAAAAgN6AAgAAAAARAAAAAAAAAL22n18zjA+Ev1y56zMpQ17oqBTbd3y+aCWX/OaEpMA9AaqWa1Z9/ht1lR3yhrKGdRHmww7+L6BFOD3eC2LhHRRugNUiAAAAAAD7+wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
+      "base64"
+    ],
+    "owner": "vBoNdEvzMrSai7is21XgVYik65mqtaKXuSdMBJ1xkW4",
+    "executable": false,
+    "rentEpoch": 18446744073709551615,
+    "space": 328
+  }
+}

fixtures/accounts-v1/C64BjL47V5r26xnw8yYwfVRgu7mhkkMabBQPT8CeCzsh.json

@@ -0,0 +1,14 @@
+{
+  "pubkey": "C64BjL47V5r26xnw8yYwfVRgu7mhkkMabBQPT8CeCzsh",
+  "account": {
+    "lamports": 2672640,
+    "data": [
+      "4IAw+7b2b8Q6grDMsjkP5Fwq+q7tnKRujAmG8Ava9PZ3nuLOZr0svdQ8dq8eNLNfR+Pa0LVpAHbXUVqBgwcAcBDDNqXFhJRjc3kQo6mg5RJIzmDEolHZ+6qDL7D9vCad/DLJdgDtUUMUAAAAAAAAAP4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
+      "base64"
+    ],
+    "owner": "vBoNdEvzMrSai7is21XgVYik65mqtaKXuSdMBJ1xkW4",
+    "executable": false,
+    "rentEpoch": 18446744073709551615,
+    "space": 256
+  }
+}