Skip to content

Commit

Permalink
feat: revisit config files
Browse files Browse the repository at this point in the history
  • Loading branch information
Ludovic Galabru authored and Ludovic Galabru committed Jan 11, 2024
1 parent 4e7bdc6 commit 1c80b67
Show file tree
Hide file tree
Showing 13 changed files with 243 additions and 183 deletions.
49 changes: 35 additions & 14 deletions components/ordhook-cli/src/cli/mod.rs
View file
Expand Up @@ -575,15 +575,14 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
while let Some(block_height) = block_range.pop_front() {
let inscriptions =
find_all_inscriptions_in_block(&block_height, &inscriptions_db_conn, ctx);
let mut locations =
let locations =
find_all_transfers_in_block(&block_height, &inscriptions_db_conn, ctx);

let mut total_transfers_in_block = 0;

for (_, inscription) in inscriptions.iter() {
println!("Inscription {} revealed at block #{} (inscription_number {}, ordinal_number {})", inscription.get_inscription_id(), block_height, inscription.inscription_number.jubilee, inscription.ordinal_number);
if let Some(transfers) = locations.remove(&inscription.get_inscription_id())
{
if let Some(transfers) = locations.get(&inscription.ordinal_number) {
for t in transfers.iter().skip(1) {
total_transfers_in_block += 1;
println!(
Expand Down Expand Up @@ -678,6 +677,8 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
&transaction_identifier,
0,
&Arc::new(cache),
config.resources.ulimit,
config.resources.memory_available,
true,
ctx,
)?;
Expand Down Expand Up @@ -707,7 +708,13 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
let last_known_block =
find_latest_inscription_block_height(&inscriptions_db_conn, ctx)?;
if last_known_block.is_none() {
open_ordhook_db_conn_rocks_db_loop(true, &config.expected_cache_path(), ctx);
open_ordhook_db_conn_rocks_db_loop(
true,
&config.expected_cache_path(),
config.resources.ulimit,
config.resources.memory_available,
ctx,
);
}

let ordhook_config = config.get_ordhook_config();
Expand Down Expand Up @@ -766,7 +773,13 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
Command::Db(OrdhookDbCommand::New(cmd)) => {
let config = ConfigFile::default(false, false, false, &cmd.config_path)?;
initialize_ordhook_db(&config.expected_cache_path(), ctx);
open_ordhook_db_conn_rocks_db_loop(true, &config.expected_cache_path(), ctx);
open_ordhook_db_conn_rocks_db_loop(
true,
&config.expected_cache_path(),
config.resources.ulimit,
config.resources.memory_available,
ctx,
);
}
Command::Db(OrdhookDbCommand::Sync(cmd)) => {
let config = ConfigFile::default(false, false, false, &cmd.config_path)?;
Expand All @@ -779,10 +792,7 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
let config = ConfigFile::default(false, false, false, &cmd.config_path)?;
let mut ordhook_config = config.get_ordhook_config();
if let Some(network_threads) = cmd.network_threads {
ordhook_config.network_thread_max = network_threads;
}
if let Some(network_threads) = cmd.network_threads {
ordhook_config.network_thread_max = network_threads;
ordhook_config.resources.bitcoind_rpc_threads = network_threads;
}
let blocks = cmd.get_blocks();
let block_ingestion_processor =
Expand All @@ -800,6 +810,8 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
let blocks_db = open_ordhook_db_conn_rocks_db_loop(
false,
&config.get_ordhook_config().db_path,
config.resources.ulimit,
config.resources.memory_available,
ctx,
);
for i in cmd.get_blocks().into_iter() {
Expand All @@ -819,7 +831,7 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
let config = ConfigFile::default(false, false, false, &cmd.config_path)?;
let mut ordhook_config = config.get_ordhook_config();
if let Some(network_threads) = cmd.network_threads {
ordhook_config.network_thread_max = network_threads;
ordhook_config.resources.bitcoind_rpc_threads = network_threads;
}
let block_post_processor = match cmd.repair_observers {
Some(true) => {
Expand Down Expand Up @@ -870,8 +882,12 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
Command::Db(OrdhookDbCommand::Check(cmd)) => {
let config = ConfigFile::default(false, false, false, &cmd.config_path)?;
{
let blocks_db =
open_readonly_ordhook_db_conn_rocks_db(&config.expected_cache_path(), ctx)?;
let blocks_db = open_readonly_ordhook_db_conn_rocks_db(
&config.expected_cache_path(),
config.resources.ulimit,
config.resources.memory_available,
ctx,
)?;
let tip = find_last_block_inserted(&blocks_db);
println!("Tip: {}", tip);
let missing_blocks = find_missing_blocks(&blocks_db, 1, tip, ctx);
Expand All @@ -880,8 +896,13 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
}
Command::Db(OrdhookDbCommand::Drop(cmd)) => {
let config = ConfigFile::default(false, false, false, &cmd.config_path)?;
let blocks_db =
open_ordhook_db_conn_rocks_db_loop(true, &config.expected_cache_path(), ctx);
let blocks_db = open_ordhook_db_conn_rocks_db_loop(
true,
&config.expected_cache_path(),
config.resources.ulimit,
config.resources.memory_available,
ctx,
);
let inscriptions_db_conn_rw =
open_readwrite_ordhook_db_conn(&config.expected_cache_path(), ctx)?;

Expand Down
90 changes: 36 additions & 54 deletions components/ordhook-cli/src/config/file.rs
View file
@@ -1,29 +1,24 @@
use ordhook::chainhook_sdk::indexer::IndexerConfig;
use ordhook::chainhook_sdk::observer::DEFAULT_INGESTION_PORT;
use ordhook::chainhook_sdk::types::{
BitcoinBlockSignaling, BitcoinNetwork, StacksNetwork, StacksNodeConfig,
};
use ordhook::config::{
BootstrapConfig, Config, LimitsConfig, LogConfig, PredicatesApi, PredicatesApiConfig,
StorageConfig,
Config, LogConfig, PredicatesApi, PredicatesApiConfig, ResourcesConfig, SnapshotConfig,
StorageConfig, DEFAULT_BITCOIND_RPC_THREADS_AVAILABLE, DEFAULT_CONTROL_PORT,
DEFAULT_MEMORY_AVAILABLE, DEFAULT_ULIMIT,
};
use std::fs::File;
use std::io::{BufReader, Read};

pub const DEFAULT_INGESTION_PORT: u16 = 20455;
pub const DEFAULT_CONTROL_PORT: u16 = 20456;
pub const STACKS_SCAN_THREAD_POOL_SIZE: usize = 10;
pub const BITCOIN_SCAN_THREAD_POOL_SIZE: usize = 10;
pub const STACKS_MAX_PREDICATE_REGISTRATION: usize = 50;
pub const BITCOIN_MAX_PREDICATE_REGISTRATION: usize = 50;

#[derive(Deserialize, Debug, Clone)]
pub struct ConfigFile {
pub storage: StorageConfigFile,
pub http_api: Option<PredicatesApiConfigFile>,
pub limits: LimitsConfigFile,
pub resources: ResourcesConfigFile,
pub network: NetworkConfigFile,
pub logs: Option<LogConfigFile>,
pub bootstrap: Option<BootstrapConfigFile>,
pub snapthot: Option<SnapshotConfigFile>,
}

impl ConfigFile {
Expand Down Expand Up @@ -54,12 +49,12 @@ impl ConfigFile {
_ => return Err("network.mode not supported".to_string()),
};

let bootstrap = match config_file.bootstrap {
let snapshot = match config_file.snapthot {
Some(bootstrap) => match bootstrap.download_url {
Some(ref url) => BootstrapConfig::Download(url.to_string()),
None => BootstrapConfig::Build,
Some(ref url) => SnapshotConfig::Download(url.to_string()),
None => SnapshotConfig::Build,
},
None => BootstrapConfig::Build,
None => SnapshotConfig::Build,
};

let config = Config {
Expand All @@ -76,36 +71,25 @@ impl ConfigFile {
}),
},
},
bootstrap,
limits: LimitsConfig {
max_number_of_stacks_predicates: config_file
.limits
.max_number_of_stacks_predicates
.unwrap_or(STACKS_MAX_PREDICATE_REGISTRATION),
max_number_of_bitcoin_predicates: config_file
.limits
.max_number_of_bitcoin_predicates
.unwrap_or(BITCOIN_MAX_PREDICATE_REGISTRATION),
max_number_of_concurrent_stacks_scans: config_file
.limits
.max_number_of_concurrent_stacks_scans
.unwrap_or(STACKS_SCAN_THREAD_POOL_SIZE),
max_number_of_concurrent_bitcoin_scans: config_file
.limits
.max_number_of_concurrent_bitcoin_scans
.unwrap_or(BITCOIN_SCAN_THREAD_POOL_SIZE),
max_number_of_processing_threads: config_file
.limits
.max_number_of_processing_threads
.unwrap_or(1.max(num_cpus::get().saturating_sub(1))),
bitcoin_concurrent_http_requests_max: config_file
.limits
.bitcoin_concurrent_http_requests_max
.unwrap_or(1.max(num_cpus::get().saturating_sub(1))),
max_caching_memory_size_mb: config_file
.limits
.max_caching_memory_size_mb
.unwrap_or(2048),
snapshot,
resources: ResourcesConfig {
ulimit: config_file.resources.ulimit.unwrap_or(DEFAULT_ULIMIT),
cpu_core_available: config_file
.resources
.cpu_core_available
.unwrap_or(num_cpus::get()),
memory_available: config_file
.resources
.memory_available
.unwrap_or(DEFAULT_MEMORY_AVAILABLE),
bitcoind_rpc_threads: config_file
.resources
.bitcoind_rpc_threads
.unwrap_or(DEFAULT_BITCOIND_RPC_THREADS_AVAILABLE),
expected_observers_count: config_file
.resources
.expected_observers_count
.unwrap_or(1),
},
network: IndexerConfig {
bitcoind_rpc_url: config_file.network.bitcoind_rpc_url.to_string(),
Expand Down Expand Up @@ -176,19 +160,17 @@ pub struct PredicatesApiConfigFile {
}

#[derive(Deserialize, Debug, Clone)]
pub struct BootstrapConfigFile {
pub struct SnapshotConfigFile {
pub download_url: Option<String>,
}

#[derive(Deserialize, Debug, Clone)]
pub struct LimitsConfigFile {
pub max_number_of_bitcoin_predicates: Option<usize>,
pub max_number_of_concurrent_bitcoin_scans: Option<usize>,
pub max_number_of_stacks_predicates: Option<usize>,
pub max_number_of_concurrent_stacks_scans: Option<usize>,
pub max_number_of_processing_threads: Option<usize>,
pub max_caching_memory_size_mb: Option<usize>,
pub bitcoin_concurrent_http_requests_max: Option<usize>,
pub struct ResourcesConfigFile {
pub ulimit: Option<usize>,
pub cpu_core_available: Option<usize>,
pub memory_available: Option<usize>,
pub bitcoind_rpc_threads: Option<usize>,
pub expected_observers_count: Option<usize>,
}

#[derive(Deserialize, Debug, Clone)]
Expand Down
14 changes: 7 additions & 7 deletions components/ordhook-cli/src/config/generator.rs
View file
Expand Up @@ -26,16 +26,16 @@ bitcoind_zmq_url = "tcp://0.0.0.0:18543"
# but stacks can also be used:
# stacks_node_rpc_url = "http://0.0.0.0:20443"
[limits]
max_number_of_bitcoin_predicates = 100
max_number_of_concurrent_bitcoin_scans = 100
max_number_of_processing_threads = 16
bitcoin_concurrent_http_requests_max = 16
max_caching_memory_size_mb = 32000
[resources]
ulimit = 2048
cpu_core_available = 16
memory_available = 32
bitcoind_rpc_threads = 8
expected_observers_count = 1
# Disable the following section if the state
# must be built locally
[bootstrap]
[snapshot]
download_url = "https://archive.hiro.so/mainnet/ordhook/mainnet-ordhook-sqlite-latest"
[logs]
Expand Down

0 comments on commit 1c80b67

Please sign in to comment.