Skip to content

Commit

Permalink
feat: polish first impression
Browse files Browse the repository at this point in the history
  • Loading branch information
@lgalabru
lgalabru committed Jul 20, 2023
1 parent 74ae98c commit 3c2b00c
Show file tree
Hide file tree
Showing 6 changed files with 79 additions and 106 deletions.
10 changes: 10 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions components/hord-cli/Cargo.toml
View file
Expand Up @@ -39,6 +39,7 @@ rusqlite = { version = "0.27.0", features = ["bundled"] }
anyhow = { version = "1.0.56", features = ["backtrace"] }
schemars = { version = "0.8.10", git = "https://github.com/hirosystems/schemars.git", branch = "feat-chainhook-fixes" }
pprof = { version = "0.12", features = ["flamegraph"], optional = true }
progressing = '3'

[dependencies.rocksdb]
version = "0.20.1"
Expand Down
73 changes: 48 additions & 25 deletions components/hord-cli/src/archive/mod.rs
View file
@@ -1,12 +1,14 @@
use crate::config::Config;
use crate::utils::{read_file_content_at_path, write_file_content_at_path};
use crate::utils::read_file_content_at_path;
use chainhook_sdk::types::BitcoinNetwork;
use chainhook_sdk::utils::Context;
use flate2::read::GzDecoder;
use futures_util::StreamExt;
use std::fs;
use std::io::{self, Cursor};
use std::io::{Read, Write};
use progressing::Baring;
use progressing::mapping::Bar as MappingBar;

pub fn default_sqlite_file_path(_network: &BitcoinNetwork) -> String {
format!("hord.sqlite").to_lowercase()
Expand All @@ -16,28 +18,28 @@ pub fn default_sqlite_sha_file_path(_network: &BitcoinNetwork) -> String {
format!("hord.sqlite.sha256").to_lowercase()
}

pub async fn download_sqlite_file(config: &Config) -> Result<(), String> {
pub async fn download_sqlite_file(config: &Config, ctx: &Context) -> Result<(), String> {
let mut destination_path = config.expected_cache_path();
std::fs::create_dir_all(&destination_path).unwrap_or_else(|e| {
println!("{}", e.to_string());
});

let remote_sha_url = config.expected_remote_ordinals_sqlite_sha256();
let res = reqwest::get(&remote_sha_url)
.await
.or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?
.bytes()
.await
.or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?;

let mut local_sha_file_path = destination_path.clone();
local_sha_file_path.push(default_sqlite_sha_file_path(
&config.network.bitcoin_network,
));
// let remote_sha_url = config.expected_remote_ordinals_sqlite_sha256();
// let res = reqwest::get(&remote_sha_url)
// .await
// .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?
// .bytes()
// .await
// .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?;

write_file_content_at_path(&local_sha_file_path, &res.to_vec())?;
// let mut local_sha_file_path = destination_path.clone();
// local_sha_file_path.push(default_sqlite_sha_file_path(
// &config.network.bitcoin_network,
// ));
// write_file_content_at_path(&local_sha_file_path, &res.to_vec())?;

let file_url = config.expected_remote_ordinals_sqlite_url();
println!("=> {file_url}");
let res = reqwest::get(&file_url)
.await
.or(Err(format!("Failed to GET from '{}'", &file_url)))?;
Expand All @@ -59,9 +61,25 @@ pub async fn download_sqlite_file(config: &Config) -> Result<(), String> {
});

if res.status() == reqwest::StatusCode::OK {
let mut progress_bar = MappingBar::with_range(0i64, 4_800_001_704);
progress_bar.set_len(40);
info!(
ctx.expect_logger(),
"{}", progress_bar
);
let mut stream = res.bytes_stream();
let mut progress = 0;
while let Some(item) = stream.next().await {
let chunk = item.or(Err(format!("Error while downloading file")))?;
progress += chunk.len() as i64;
progress_bar.set(progress);
if progress_bar.has_progressed_significantly() {
progress_bar.remember_significant_progress();
info!(
ctx.expect_logger(),
"{}", progress_bar
);
}
tx.send_async(chunk.to_vec())
.await
.map_err(|e| format!("unable to download stacks event: {}", e.to_string()))?;
Expand Down Expand Up @@ -116,14 +134,14 @@ pub async fn download_ordinals_dataset_if_required(config: &Config, ctx: &Contex
let url = config.expected_remote_ordinals_sqlite_url();
let mut sqlite_file_path = config.expected_cache_path();
sqlite_file_path.push(default_sqlite_file_path(&config.network.bitcoin_network));
let mut tsv_sha_file_path = config.expected_cache_path();
tsv_sha_file_path.push(default_sqlite_sha_file_path(
let mut sqlite_sha_file_path = config.expected_cache_path();
sqlite_sha_file_path.push(default_sqlite_sha_file_path(
&config.network.bitcoin_network,
));

// Download archive if not already present in cache
// Load the local
let local_sha_file = read_file_content_at_path(&tsv_sha_file_path);
let local_sha_file = read_file_content_at_path(&sqlite_sha_file_path);
let sha_url = config.expected_remote_ordinals_sqlite_sha256();

let remote_sha_file = match reqwest::get(&sha_url).await {
Expand All @@ -136,22 +154,27 @@ pub async fn download_ordinals_dataset_if_required(config: &Config, ctx: &Contex
if cache_not_expired {
info!(
ctx.expect_logger(),
"More recent Stacks archive file detected"
"More recent hord.sqlite file detected"
);
}
cache_not_expired == false
}
(_, _) => {
info!(
ctx.expect_logger(),
"Unable to retrieve Stacks archive file locally"
);
true
match std::fs::metadata(&sqlite_file_path) {
Ok(_) => false,
_ => {
info!(
ctx.expect_logger(),
"Unable to retrieve hord.sqlite file locally"
);
true
}
}
}
};
if should_download {
info!(ctx.expect_logger(), "Downloading {}", url);
match download_sqlite_file(&config).await {
match download_sqlite_file(&config, &ctx).await {
Ok(_) => {}
Err(e) => {
error!(ctx.expect_logger(), "{}", e);
Expand Down
22 changes: 15 additions & 7 deletions components/hord-cli/src/cli/mod.rs
View file
@@ -1,3 +1,4 @@
use crate::archive::download_ordinals_dataset_if_required;
use crate::config::generator::generate_config;
use crate::config::Config;
use crate::scan::bitcoin::scan_bitcoin_chainstate_via_rpc_using_predicate;
Expand Down Expand Up @@ -455,9 +456,13 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
.get_sorted_entries();

if let Some(ref post_to) = cmd.post_to {
info!(ctx.expect_logger(), "A fully synchronized bitcoind node is required for retrieving inscriptions content.");
info!(ctx.expect_logger(), "Checking {}...", config.network.bitcoind_rpc_url);
let tip = check_bitcoind_connection(&config).await?;
if tip < cmd.end_block {
error!(ctx.expect_logger(), "unable to scan block range [{}, {}]: underlying bitcoind synchronized until block {} ", cmd.start_block, cmd.end_block, tip);
error!(ctx.expect_logger(), "Unable to scan block range [{}, {}]: underlying bitcoind synchronized until block {} ", cmd.start_block, cmd.end_block, tip);
} else {
info!(ctx.expect_logger(), "Starting scan");
}

let predicate_spec = build_predicate_from_cli(
Expand All @@ -470,6 +475,8 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
scan_bitcoin_chainstate_via_rpc_using_predicate(&predicate_spec, &config, &ctx)
.await?;
} else {
let _ = download_ordinals_dataset_if_required(&config, ctx).await;

let inscriptions_db_conn =
open_readonly_hord_db_conn(&config.expected_cache_path(), &ctx)?;
while let Some(block_height) = block_range.pop_front() {
Expand Down Expand Up @@ -501,12 +508,13 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
);
}
}
println!(
"Inscriptions revealed: {}, inscriptions transferred: {total_transfers}",
inscriptions.len()
);

println!("-----");
if total_transfers > 0 && inscriptions.len() > 0 {
println!(
"Inscriptions revealed: {}, inscriptions transferred: {total_transfers}",
inscriptions.len()
);
println!("-----");
}
}
}
}
Expand Down
78 changes: 5 additions & 73 deletions components/hord-cli/src/config/mod.rs
View file
Expand Up @@ -12,12 +12,8 @@ use std::fs::File;
use std::io::{BufReader, Read};
use std::path::PathBuf;

const DEFAULT_MAINNET_STACKS_TSV_ARCHIVE: &str =
"https://archive.hiro.so/mainnet/stacks-blockchain-api/mainnet-stacks-blockchain-api-latest";
const DEFAULT_TESTNET_STACKS_TSV_ARCHIVE: &str =
"https://archive.hiro.so/testnet/stacks-blockchain-api/testnet-stacks-blockchain-api-latest";
const DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE: &str =
"https://archive.hiro.so/mainnet/chainhooks/hord-latest.sqlite";
"https://archive.hiro.so/mainnet/chainhooks/hord.sqlite";
const DEFAULT_REDIS_URI: &str = "redis://localhost:6379/";

pub const DEFAULT_INGESTION_PORT: u16 = 20455;
Expand Down Expand Up @@ -56,8 +52,6 @@ pub struct PredicatesApiConfig {

#[derive(Clone, Debug)]
pub enum EventSourceConfig {
StacksTsvPath(PathConfig),
StacksTsvUrl(UrlConfig),
OrdinalsSqlitePath(PathConfig),
OrdinalsSqliteUrl(UrlConfig),
}
Expand Down Expand Up @@ -154,11 +148,11 @@ impl Config {
if let Some(dst) = source.tsv_file_path.take() {
let mut file_path = PathBuf::new();
file_path.push(dst);
event_sources.push(EventSourceConfig::StacksTsvPath(PathConfig { file_path }));
event_sources.push(EventSourceConfig::OrdinalsSqlitePath(PathConfig { file_path }));
continue;
}
if let Some(file_url) = source.tsv_file_url.take() {
event_sources.push(EventSourceConfig::StacksTsvUrl(UrlConfig { file_url }));
event_sources.push(EventSourceConfig::OrdinalsSqliteUrl(UrlConfig { file_url }));
continue;
}
}
Expand Down Expand Up @@ -234,22 +228,14 @@ impl Config {
pub fn is_initial_ingestion_required(&self) -> bool {
for source in self.event_sources.iter() {
match source {
EventSourceConfig::StacksTsvUrl(_) | EventSourceConfig::StacksTsvPath(_) => {
EventSourceConfig::OrdinalsSqlitePath(_) | EventSourceConfig::OrdinalsSqliteUrl(_) => {
return true
}
_ => {}
}
}
return false;
}

pub fn add_local_stacks_tsv_source(&mut self, file_path: &PathBuf) {
self.event_sources
.push(EventSourceConfig::StacksTsvPath(PathConfig {
file_path: file_path.clone(),
}));
}

pub fn add_ordinals_sqlite_remote_source_url(&mut self, file_url: &str) {
self.event_sources
.push(EventSourceConfig::OrdinalsSqliteUrl(UrlConfig {
Expand All @@ -275,15 +261,6 @@ impl Config {
}
}

pub fn expected_local_stacks_tsv_file(&self) -> &PathBuf {
for source in self.event_sources.iter() {
if let EventSourceConfig::StacksTsvPath(config) = source {
return &config.file_path;
}
}
panic!("expected local-tsv source")
}

pub fn expected_cache_path(&self) -> PathBuf {
let mut destination_path = PathBuf::new();
destination_path.push(&self.storage.working_dir);
Expand All @@ -299,23 +276,6 @@ impl Config {
panic!("expected remote-tsv source")
}

fn expected_remote_stacks_tsv_base_url(&self) -> &String {
for source in self.event_sources.iter() {
if let EventSourceConfig::StacksTsvUrl(config) = source {
return &config.file_url;
}
}
panic!("expected remote-tsv source")
}

pub fn expected_remote_stacks_tsv_sha256(&self) -> String {
format!("{}.sha256", self.expected_remote_stacks_tsv_base_url())
}

pub fn expected_remote_stacks_tsv_url(&self) -> String {
format!("{}.gz", self.expected_remote_stacks_tsv_base_url())
}

pub fn expected_remote_ordinals_sqlite_sha256(&self) -> String {
format!("{}.sha256", self.expected_remote_ordinals_sqlite_base_url())
}
Expand All @@ -324,15 +284,6 @@ impl Config {
format!("{}.gz", self.expected_remote_ordinals_sqlite_base_url())
}

pub fn rely_on_remote_stacks_tsv(&self) -> bool {
for source in self.event_sources.iter() {
if let EventSourceConfig::StacksTsvUrl(_config) = source {
return true;
}
}
false
}

pub fn rely_on_remote_ordinals_sqlite(&self) -> bool {
for source in self.event_sources.iter() {
if let EventSourceConfig::OrdinalsSqliteUrl(_config) = source {
Expand All @@ -342,20 +293,6 @@ impl Config {
false
}

pub fn should_download_remote_stacks_tsv(&self) -> bool {
let mut rely_on_remote_tsv = false;
let mut remote_tsv_present_locally = false;
for source in self.event_sources.iter() {
if let EventSourceConfig::StacksTsvUrl(_config) = source {
rely_on_remote_tsv = true;
}
if let EventSourceConfig::StacksTsvPath(_config) = source {
remote_tsv_present_locally = true;
}
}
rely_on_remote_tsv == true && remote_tsv_present_locally == false
}

pub fn should_download_remote_ordinals_sqlite(&self) -> bool {
let mut rely_on_remote_tsv = false;
let mut remote_tsv_present_locally = false;
Expand Down Expand Up @@ -421,9 +358,7 @@ impl Config {
working_dir: default_cache_path(),
},
http_api: PredicatesApi::Off,
event_sources: vec![EventSourceConfig::StacksTsvUrl(UrlConfig {
file_url: DEFAULT_TESTNET_STACKS_TSV_ARCHIVE.into(),
})],
event_sources: vec![],
limits: LimitsConfig {
max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION,
max_number_of_concurrent_bitcoin_scans: BITCOIN_SCAN_THREAD_POOL_SIZE,
Expand Down Expand Up @@ -453,9 +388,6 @@ impl Config {
},
http_api: PredicatesApi::Off,
event_sources: vec![
EventSourceConfig::StacksTsvUrl(UrlConfig {
file_url: DEFAULT_MAINNET_STACKS_TSV_ARCHIVE.into(),
}),
EventSourceConfig::OrdinalsSqliteUrl(UrlConfig {
file_url: DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE.into(),
}),
Expand Down
1 change: 0 additions & 1 deletion components/hord-cli/src/db/mod.rs
View file
@@ -1,6 +1,5 @@
use std::{
collections::{BTreeMap, HashMap},
fs::File,
hash::BuildHasherDefault,
path::PathBuf,
sync::{mpsc::Sender, Arc},
Expand Down

0 comments on commit 3c2b00c

Please sign in to comment.