feat: clone accounts on demand

Cargo.toml

@@ -7,6 +7,7 @@ split-debuginfo = "packed"
 members = [
   "geyser-grpc-proto",
   "programs/sleipnir",
+  "sleipnir-accounts",
   "sleipnir-bank",
   "sleipnir-geyser-plugin",
   "sleipnir-ledger",
@@ -55,6 +56,7 @@ bincode = "1.3.3"
 bs58 = "0.4.0"
 byteorder = "1.5.0"
 cargo-lock = "9.0.0"
+conjunto-transwise = { path = "../../conjunto/conjunto/transwise" }
 console-subscriber = "0.2.0"
 crossbeam-channel = "0.5.11"
 csv = "1.3.0"
@@ -94,11 +96,13 @@ semver = "1.0.22"
 serde = "1.0.196"
 serde_derive = "1.0.103"
 serde_json = "1.0.113"
+sleipnir-accounts = { path = "./sleipnir-accounts" }
 sleipnir-bank = { path = "./sleipnir-bank" }
 sleipnir-geyser-plugin = { path = "./sleipnir-geyser-plugin" }
 sleipnir-ledger = { path = "./sleipnir-ledger" }
 sleipnir-messaging = { path = "./sleipnir-messaging" }
 sleipnir-mutator = { path = "./sleipnir-mutator" }
+sleipnir-perf-service = { path = "./sleipnir-perf-service" }
 sleipnir-processor = { path = "./sleipnir-processor" }
 sleipnir-program = { path = "./programs/sleipnir" }
 sleipnir-pubsub = { path = "./sleipnir-pubsub" }
@@ -112,28 +116,27 @@ sleipnir-tokens = { path = "./sleipnir-tokens" }
 sleipnir-streamer = { path = "./sleipnir-streamer" }
 sleipnir-transaction-status = { path = "./sleipnir-transaction-status" }
 sleipnir-version = { path = "./sleipnir-version" }
-solana-accounts-db = { path = "./solana/accounts-db" }
-solana-account-decoder = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-account-decoder" }
-solana-bpf-loader-program = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-bpf-loader-program" }
-solana-cost-model = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-cost-model" }
-solana-frozen-abi = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-frozen-abi" }
-solana-frozen-abi-macro = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-frozen-abi-macro" }
-solana-geyser-plugin-interface = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-geyser-plugin-interface" }
-solana-geyser-plugin-manager = { path = "./solana/geyser-plugin-manager" }
-solana-loader-v4-program = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-loader-v4-program" }
-solana-logger = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-logger" }
-solana-measure = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-measure" }
-solana-metrics = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-metrics" }
-solana-perf = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-perf" }
-sleipnir-perf-service = { path = "./sleipnir-perf-service" }
-solana-program-runtime = { path = "./solana/program-runtime" }
-solana-rayon-threadlimit = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-rayon-threadlimit" }
-solana-streamer = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-streamer" }
-solana-sdk = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-sdk" }
-solana-svm = { path = "./solana/svm" }
-solana-storage-proto = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-storage-proto" }
-solana-system-program = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-system-program" }
-solana-transaction-status = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-transaction-status" }
+solana-accounts-db = { path = "./solana/accounts-db", version = "1.19.0" }
+solana-account-decoder = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-account-decoder", version = "1.19.0" }
+solana-bpf-loader-program = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-bpf-loader-program", version = "1.19.0" }
+solana-cost-model = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-cost-model", version = "1.19.0" }
+solana-frozen-abi = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-frozen-abi", version = "1.19.0" }
+solana-frozen-abi-macro = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-frozen-abi-macro", version = "1.19.0" }
+solana-geyser-plugin-interface = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-geyser-plugin-interface", version = "1.19.0" }
+solana-geyser-plugin-manager = { path = "./solana/geyser-plugin-manager", version = "1.19.0" }
+solana-loader-v4-program = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-loader-v4-program", version = "1.19.0" }
+solana-logger = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-logger", version = "1.19.0" }
+solana-measure = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-measure", version = "1.19.0" }
+solana-metrics = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-metrics", version = "1.19.0" }
+solana-perf = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-perf", version = "1.19.0" }
+solana-program-runtime = { path = "./solana/program-runtime", version = "1.19.0" }
+solana-rayon-threadlimit = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-rayon-threadlimit", version = "1.19.0" }
+solana-streamer = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-streamer", version = "1.19.0" }
+solana-sdk = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-sdk", version = "1.19.0" }
+solana-svm = { path = "./solana/svm", version = "1.19.0" }
+solana-storage-proto = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-storage-proto", version = "1.19.0" }
+solana-system-program = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-system-program", version = "1.19.0" }
+solana-transaction-status = { git = "https://github.com/solana-labs/solana", rev = "30adda4a71", package = "solana-transaction-status", version = "1.19.0" }
 solana_rbpf = "=0.8.0"
 spl-token = "=4.0.1"
 spl-token-2022 = "=2.0.1"
@@ -150,6 +153,7 @@ tokio-util = "0.7.10"
 tonic = "0.10.2"
 tonic-build = "0.10.2"
 tonic-health = "0.10.2"
+url = "2.5.0"
 vergen = "8.3.1"
 zstd = "0.11.2"
 

sleipnir-accounts/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "sleipnir-accounts"
+version.workspace = true
+authors.workspace = true
+repository.workspace = true
+homepage.workspace = true
+license.workspace = true
+edition.workspace = true
+
+[dependencies]
+async-trait = { workspace = true }
+conjunto-transwise = { workspace = true }
+log = { workspace = true }
+sleipnir-bank = { workspace = true }
+sleipnir-mutator = { workspace = true }
+sleipnir-processor = { workspace = true }
+sleipnir-transaction-status = { workspace = true }
+solana-sdk = { workspace = true }
+tokio = { workspace = true }
+thiserror = { workspace = true }
+url = { workspace = true }

sleipnir-accounts/src/bank_account_provider.rs

@@ -0,0 +1,19 @@
+use sleipnir_bank::bank::Bank;
+use solana_sdk::{account::AccountSharedData, pubkey::Pubkey};
+use std::sync::Arc;
+
+use crate::InternalAccountProvider;
+
+pub struct BankAccountProvider(Arc<Bank>);
+
+impl BankAccountProvider {
+    pub fn new(bank: Arc<Bank>) -> Self {
+        Self(bank)
+    }
+}
+
+impl InternalAccountProvider for BankAccountProvider {
+    fn get_account(&self, pubkey: &Pubkey) -> Option<AccountSharedData> {
+        self.0.get_account(pubkey)
+    }
+}

sleipnir-accounts/src/errors.rs

@@ -0,0 +1,30 @@
+use thiserror::Error;
+
+pub type AccountsResult<T> = std::result::Result<T, AccountsError>;
+
+#[derive(Error, Debug)]
+pub enum AccountsError {
+    #[error("TranswiseError")]
+    TranswiseError(#[from] conjunto_transwise::errors::TranswiseError),
+
+    #[error("MutatorError")]
+    MutatorError(#[from] sleipnir_mutator::errors::MutatorError),
+
+    #[error("UrlParseError")]
+    UrlParseError(#[from] url::ParseError),
+
+    #[error("SanitizeError")]
+    SanitizeError(#[from] solana_sdk::sanitize::SanitizeError),
+
+    #[error("TransactionError")]
+    TransactionError(#[from] solana_sdk::transaction::TransactionError),
+
+    #[error("InvalidRpcUrl '{0}'")]
+    InvalidRpcUrl(String),
+
+    #[error("FailedToUpdateUrlScheme")]
+    FailedToUpdateUrlScheme,
+
+    #[error("FailedToUpdateUrlPort")]
+    FailedToUpdateUrlPort,
+}

sleipnir-accounts/src/external_accounts.rs

@@ -0,0 +1,148 @@
+use std::{
+    collections::HashMap,
+    ops::Deref,
+    sync::{RwLock, RwLockReadGuard, RwLockWriteGuard},
+    time::Duration,
+};
+
+use solana_sdk::pubkey::Pubkey;
+
+use crate::utils::get_epoch;
+
+// -----------------
+// ExternalAccounts
+// -----------------
+pub trait ExternalAccount {
+    fn new(pubkey: Pubkey, now: Duration) -> Self;
+    fn cloned_at(&self) -> Duration;
+}
+
+#[derive(Debug)]
+pub struct ExternalAccounts<T: ExternalAccount> {
+    accounts: RwLock<HashMap<Pubkey, T>>,
+}
+
+impl<T: ExternalAccount> Default for ExternalAccounts<T> {
+    fn default() -> Self {
+        Self {
+            accounts: RwLock::new(HashMap::new()),
+        }
+    }
+}
+
+impl<T: ExternalAccount> ExternalAccounts<T> {
+    pub fn insert(&self, pubkey: Pubkey) {
+        let now = get_epoch();
+        self.write_accounts().insert(pubkey, T::new(pubkey, now));
+    }
+
+    pub fn has(&self, pubkey: &Pubkey) -> bool {
+        self.read_accounts().contains_key(pubkey)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.read_accounts().is_empty()
+    }
+
+    pub fn len(&self) -> usize {
+        self.read_accounts().len()
+    }
+
+    pub fn cloned_at(&self, pubkey: &Pubkey) -> Option<Duration> {
+        self.read_accounts()
+            .get(pubkey)
+            .map(|account| account.cloned_at())
+    }
+
+    fn read_accounts(&self) -> RwLockReadGuard<HashMap<Pubkey, T>> {
+        self.accounts
+            .read()
+            .expect("RwLock of external accounts is poisoned")
+    }
+
+    fn write_accounts(&self) -> RwLockWriteGuard<HashMap<Pubkey, T>> {
+        self.accounts
+            .write()
+            .expect("RwLock of external accounts is poisoned")
+    }
+}
+
+// -----------------
+// ExternalReadonlyAccounts
+// -----------------
+#[derive(Default, Debug)]
+pub struct ExternalReadonlyAccounts(ExternalAccounts<ExternalReadonlyAccount>);
+
+impl Deref for ExternalReadonlyAccounts {
+    type Target = ExternalAccounts<ExternalReadonlyAccount>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+#[derive(Debug)]
+pub struct ExternalReadonlyAccount {
+    pub pubkey: Pubkey,
+    pub cloned_at: Duration,
+    pub updated_at: Duration,
+}
+
+impl ExternalAccount for ExternalReadonlyAccount {
+    fn new(pubkey: Pubkey, now: Duration) -> Self {
+        Self {
+            pubkey,
+            cloned_at: now,
+            updated_at: now,
+        }
+    }
+
+    fn cloned_at(&self) -> Duration {
+        self.cloned_at
+    }
+}
+
+impl ExternalReadonlyAccounts {
+    pub fn get_updated_at(&self, pubkey: &Pubkey) -> Option<Duration> {
+        self.read_accounts()
+            .get(pubkey)
+            .map(|account| account.updated_at)
+    }
+}
+
+// -----------------
+// ExternalWritableAccounts
+// -----------------
+#[derive(Default, Debug)]
+pub struct ExternalWritableAccounts(ExternalAccounts<ExternalWritableAccount>);
+
+impl Deref for ExternalWritableAccounts {
+    type Target = ExternalAccounts<ExternalWritableAccount>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+#[derive(Debug)]
+pub struct ExternalWritableAccount {
+    pub pubkey: Pubkey,
+    pub cloned_at: Duration,
+    pub updated_at: Duration,
+    pub last_committed_at: Option<Duration>,
+}
+
+impl ExternalAccount for ExternalWritableAccount {
+    fn new(pubkey: Pubkey, now: Duration) -> Self {
+        Self {
+            pubkey,
+            cloned_at: now,
+            updated_at: now,
+            last_committed_at: None,
+        }
+    }
+
+    fn cloned_at(&self) -> Duration {
+        self.cloned_at
+    }
+}