diff --git a/Cargo.lock b/Cargo.lock index cf6c68670d..2fd303f1ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1754,6 +1754,31 @@ dependencies = [ "memmap2", ] +[[package]] +name = "db-sync-explorer" +version = "0.1.0" +dependencies = [ + "bigdecimal", + "chrono", + "clap 3.2.23", + "color-eyre", + "diesel 2.0.2", + "diesel-derive-enum", + "futures-util", + "http-zipkin", + "jortestkit", + "microtype", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", + "vit-servicing-station-lib", + "voting_tools_rs", + "warp", +] + [[package]] name = "deflate" version = "0.8.6" @@ -1821,6 +1846,7 @@ dependencies = [ "bigdecimal", "bitflags", "byteorder", + "chrono", "diesel_derives 2.0.1", "itoa 1.0.4", "num-bigint", diff --git a/Cargo.toml b/Cargo.toml index c4e4c20cce..0d3ddc8a47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ members = [ "src/vit-testing/integration-tests", "src/vit-testing/mainnet-tools", "src/vit-testing/mainnet-lib", + "src/vit-testing/db-sync-explorer", "src/vit-testing/snapshot-trigger-service", "src/vit-testing/signals-handler", "src/vit-testing/scheduler-service-lib", diff --git a/src/vit-testing/db-sync-explorer/Cargo.toml b/src/vit-testing/db-sync-explorer/Cargo.toml new file mode 100644 index 0000000000..5b7196374f --- /dev/null +++ b/src/vit-testing/db-sync-explorer/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "db-sync-explorer" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +color-eyre = "0.6" # error handling +voting_tools_rs = { path = "../../voting-tools-rs"} # db sync schema +microtype = { version = "0.7.5", features = ["serde"] } # defining secrecy and db internals +diesel = { version = "2", features = ["postgres", "64-column-tables", "numeric", "serde_json", "r2d2"]} # connector lib +diesel-derive-enum = "2.0.0-rc.0" #extension to diesel +chrono = "0.4.23" +tracing = "0.1" +bigdecimal = "0.3.0" +warp = { version = "0.3.2", features = ["tls"] } +vit-servicing-station-lib = { path = "../../vit-servicing-station/vit-servicing-station-lib" } +futures-util = "0.3.25" +thiserror = "1.0.37" +http-zipkin = "0.3.0" +jortestkit = { path = "../../jortestkit" } +tokio = { version = "1.18.0", features = ["macros", "signal", "rt", "fs", "sync"] } +tracing-subscriber = "0.3" +clap = { version = "3.2", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + diff --git a/src/vit-testing/db-sync-explorer/config-snapshot.json b/src/vit-testing/db-sync-explorer/config-snapshot.json new file mode 100644 index 0000000000..159ed87b89 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/config-snapshot.json @@ -0,0 +1,16 @@ +{ + "port": 8080, + "result-dir": "./data", + "voting-tools": { + "bin": "voting-tools-mock", + "network": { + "testnet": 1234231321 + }, + "db": "cexplorer", + "db-user": "cexplorer", + "db-pass": "pass", + "db-host": "/alloc", + "scale": 1000000 + }, + "token":"RBj0OfHw5jT87A" +} diff --git a/src/vit-testing/db-sync-explorer/src/bin/db-sync-explorer.rs b/src/vit-testing/db-sync-explorer/src/bin/db-sync-explorer.rs new file mode 100644 index 0000000000..6fac2d2ee2 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/bin/db-sync-explorer.rs @@ -0,0 +1,34 @@ +use std::path::Path; + +use clap::Parser; +use color_eyre::Result; +use db_sync_explorer::{connect, rest, Args, Config}; + +#[tokio::main] +async fn main() -> Result<()> { + color_eyre::install()?; + tracing_subscriber::fmt().init(); + + let Args { config, .. } = Args::parse(); + + let configuration: Config = read_config(&config)?; + let db_pool = connect(configuration.db)?; + + let address = configuration.address; + let context = rest::v0::context::new_shared_context(db_pool, configuration.token); + + let app = rest::v0::filter(context).await; + + tracing::debug!("listening on {}", address); + let (_, server) = warp::serve(app).bind_with_graceful_shutdown( + address, + vit_servicing_station_lib::server::signals::watch_signal_for_shutdown(), + ); + server.await; + Ok(()) +} + +pub fn read_config>(config: P) -> Result { + let contents = std::fs::read_to_string(&config)?; + serde_json::from_str(&contents).map_err(Into::into) +} diff --git a/src/vit-testing/db-sync-explorer/src/cli.rs b/src/vit-testing/db-sync-explorer/src/cli.rs new file mode 100644 index 0000000000..d2b1ed26eb --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/cli.rs @@ -0,0 +1,12 @@ +use clap::Parser; +use std::path::PathBuf; + +#[derive(Debug, Parser)] +#[non_exhaustive] +#[clap(about = "Create a voting power snapshot")] +/// CLI arguments for db-sync-explorer +pub struct Args { + /// configuration file + #[clap(long)] + pub config: PathBuf, +} diff --git a/src/vit-testing/db-sync-explorer/src/config.rs b/src/vit-testing/db-sync-explorer/src/config.rs new file mode 100644 index 0000000000..55d04f24f7 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/config.rs @@ -0,0 +1,10 @@ +use crate::DbConfig; +use serde::Deserialize; +use std::net::SocketAddr; + +#[derive(Debug, Deserialize)] +pub struct Config { + pub address: SocketAddr, + pub db: DbConfig, + pub token: Option, +} diff --git a/src/vit-testing/db-sync-explorer/src/db/config.rs b/src/vit-testing/db-sync-explorer/src/db/config.rs new file mode 100644 index 0000000000..d3d2feb934 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/db/config.rs @@ -0,0 +1,35 @@ +use microtype::microtype; +use serde::Deserialize; + +microtype! { + #[derive(Debug, PartialEq,Eq, Clone)] + #[string] + pub String { + /// Database name + DbName, + /// Database user + DbUser, + /// Database host + DbHost, + } + + #[secret] + #[string] + pub String { + /// Database password + DbPass, + } +} + +/// Information required to connect to a database +#[derive(Debug, Clone, Deserialize)] +pub struct DbConfig { + /// The name of the database + pub name: DbName, + /// The user to connect with + pub user: DbUser, + /// The hostname to connect to + pub host: DbHost, + /// The corresponding password for this user + pub password: Option, +} diff --git a/src/vit-testing/db-sync-explorer/src/db/connector.rs b/src/vit-testing/db-sync-explorer/src/db/connector.rs new file mode 100644 index 0000000000..d5f50e6510 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/db/connector.rs @@ -0,0 +1,30 @@ +use crate::db::config::DbConfig; +use color_eyre::Result; +use diesel::r2d2::{ConnectionManager, Pool}; +use diesel::PgConnection; +use microtype::secrecy::ExposeSecret; +use microtype::secrecy::Zeroize; + +pub type DbPool = Pool>; +/// Connect to the database with the provided credentials +/// +/// # Errors +/// +/// Returns an error if connecting to the database fails +pub fn connect( + DbConfig { + name, + user, + host, + password, + }: DbConfig, +) -> Result { + let mut password = password + .map(|p| format!(":{}", p.expose_secret())) + .unwrap_or_default(); + + let url = format!("postgres://{user}{password}@{host}/{name}"); + let pool = Pool::builder().build(ConnectionManager::new(url))?; + password.zeroize(); + Ok(pool) +} diff --git a/src/vit-testing/db-sync-explorer/src/db/mod.rs b/src/vit-testing/db-sync-explorer/src/db/mod.rs new file mode 100644 index 0000000000..c35249621d --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/db/mod.rs @@ -0,0 +1,13 @@ +mod config; +mod connector; +mod model; +pub(crate) mod query; +pub mod schema; +pub mod types; + +pub use config::DbConfig; +pub use connector::{connect, DbPool}; +pub use model::{ + BehindDuration, Meta, Progress, TransactionConfirmation, TransactionConfirmationRow, +}; +pub use query::behind; diff --git a/src/vit-testing/db-sync-explorer/src/db/model.rs b/src/vit-testing/db-sync-explorer/src/db/model.rs new file mode 100644 index 0000000000..17e3444327 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/db/model.rs @@ -0,0 +1,46 @@ +use bigdecimal::BigDecimal; +use diesel::sql_types::{Numeric, Timestamp}; +use diesel::{Queryable, QueryableByName}; +use serde::{Deserialize, Serialize}; +use std::time::SystemTime; + +#[derive(Queryable, PartialEq, Eq, Debug, Serialize, Clone)] +pub struct Meta { + pub id: i64, + pub start_time: SystemTime, + pub version: String, + pub network_name: String, +} + +#[derive(QueryableByName, Debug, Serialize, Deserialize, Clone)] +pub struct BehindDuration { + #[diesel(sql_type = Timestamp)] + behind_by: SystemTime, +} + +#[derive(QueryableByName, Debug, Serialize, Deserialize, Clone)] +pub struct Progress { + #[diesel(sql_type = Numeric)] + sync_percentage: BigDecimal, +} + +#[derive(Debug, Serialize)] +pub struct TransactionConfirmation { + epoch_no: Option, + slot_no: Option, + absolute_slot: Option, + block_no: Option, +} + +impl From for TransactionConfirmation { + fn from(row: TransactionConfirmationRow) -> Self { + TransactionConfirmation { + epoch_no: row.0, + slot_no: row.1, + absolute_slot: row.2, + block_no: row.3, + } + } +} + +pub type TransactionConfirmationRow = (Option, Option, Option, Option); diff --git a/src/vit-testing/db-sync-explorer/src/db/query.rs b/src/vit-testing/db-sync-explorer/src/db/query.rs new file mode 100644 index 0000000000..4c5758979e --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/db/query.rs @@ -0,0 +1,63 @@ +use crate::db::model::Progress; +use crate::db::{DbPool, TransactionConfirmationRow}; +use crate::rest::v0::errors::HandleError; +use crate::BehindDuration; +use diesel::QueryDsl; +use diesel::{sql_function, sql_query, sql_types::Text, RunQueryDsl}; +sql_function! (fn decode(string: Text, format: Text) -> Bytea); + +pub async fn hash( + input_hash: String, + pool: &DbPool, +) -> Result, HandleError> { + let mut db_conn = pool.get().map_err(HandleError::Connection)?; + + tokio::task::spawn_blocking(move || { + use crate::db::schema::{ + block::{self, block_no, epoch_no, epoch_slot_no, id, slot_no}, + tx::{self, block_id}, + }; + use diesel::ExpressionMethods; + use diesel::JoinOnDsl; + + let inner_join = block::table.on(id.eq(block_id)); + + let table = tx::table.inner_join(inner_join); + + table + .select((epoch_no, slot_no, epoch_slot_no, block_no)) + .filter(tx::hash.eq(decode(input_hash, "hex"))) + .load(&mut db_conn) + .map_err(HandleError::Database) + }) + .await + .map_err(HandleError::Join)? +} + +/// Running sql from dbsync examples: +/// +pub async fn sync_progress(pool: &DbPool) -> Result, HandleError> { + let mut db_conn = pool.get().map_err(HandleError::Connection)?; + + let query = sql_query("select + 100 * (extract (epoch from (max (time) at time zone 'UTC')) - extract (epoch from (min (time) at time zone 'UTC'))) + / (extract (epoch from (now () at time zone 'UTC')) - extract (epoch from (min (time) at time zone 'UTC'))) + as sync_percentage + from block"); + + tokio::task::spawn_blocking(move || query.load(&mut db_conn).map_err(HandleError::Database)) + .await + .map_err(HandleError::Join)? +} + +/// Running sql from dbsync examples: +/// +pub async fn behind(pool: &DbPool) -> Result, HandleError> { + let mut db_conn = pool.get().map_err(HandleError::Connection)?; + + let query = sql_query("select max (time) as behind_by from block"); + + tokio::task::spawn_blocking(move || query.load(&mut db_conn).map_err(HandleError::Database)) + .await + .map_err(HandleError::Join)? +} diff --git a/src/vit-testing/db-sync-explorer/src/db/schema.rs b/src/vit-testing/db-sync-explorer/src/db/schema.rs new file mode 100644 index 0000000000..deec7be7ec --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/db/schema.rs @@ -0,0 +1,802 @@ +use diesel::{allow_tables_to_appear_in_same_query, joinable, table}; + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + ada_pots (id) { + id -> Int8, + slot_no -> Int8, + epoch_no -> Int4, + treasury -> Numeric, + reserves -> Numeric, + rewards -> Numeric, + utxo -> Numeric, + deposits -> Numeric, + fees -> Numeric, + block_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + block (id) { + id -> Int8, + hash -> Bytea, + epoch_no -> Nullable, + slot_no -> Nullable, + epoch_slot_no -> Nullable, + block_no -> Nullable, + previous_id -> Nullable, + slot_leader_id -> Int8, + size -> Int4, + time -> Timestamp, + tx_count -> Int8, + proto_major -> Int4, + proto_minor -> Int4, + vrf_key -> Nullable, + op_cert -> Nullable, + op_cert_counter -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + collateral_tx_in (id) { + id -> Int8, + tx_in_id -> Int8, + tx_out_id -> Int8, + tx_out_index -> Int2, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + collateral_tx_out (id) { + id -> Int8, + tx_id -> Int8, + index -> Int2, + address -> Varchar, + address_raw -> Bytea, + address_has_script -> Bool, + payment_cred -> Nullable, + stake_address_id -> Nullable, + value -> Numeric, + data_hash -> Nullable, + multi_assets_descr -> Varchar, + inline_datum_id -> Nullable, + reference_script_id -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + cost_model (id) { + id -> Int8, + costs -> Jsonb, + block_id -> Int8, + hash -> Bytea, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + datum (id) { + id -> Int8, + hash -> Bytea, + tx_id -> Int8, + value -> Nullable, + bytes -> Bytea, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + delegation (id) { + id -> Int8, + addr_id -> Int8, + cert_index -> Int4, + pool_hash_id -> Int8, + active_epoch_no -> Int8, + tx_id -> Int8, + slot_no -> Int8, + redeemer_id -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + delisted_pool (id) { + id -> Int8, + hash_raw -> Bytea, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + epoch (id) { + id -> Int8, + out_sum -> Numeric, + fees -> Numeric, + tx_count -> Int4, + blk_count -> Int4, + no -> Int4, + start_time -> Timestamp, + end_time -> Timestamp, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + epoch_param (id) { + id -> Int8, + epoch_no -> Int4, + min_fee_a -> Int4, + min_fee_b -> Int4, + max_block_size -> Int4, + max_tx_size -> Int4, + max_bh_size -> Int4, + key_deposit -> Numeric, + pool_deposit -> Numeric, + max_epoch -> Int4, + optimal_pool_count -> Int4, + influence -> Float8, + monetary_expand_rate -> Float8, + treasury_growth_rate -> Float8, + decentralisation -> Float8, + protocol_major -> Int4, + protocol_minor -> Int4, + min_utxo_value -> Numeric, + min_pool_cost -> Numeric, + nonce -> Nullable, + cost_model_id -> Nullable, + price_mem -> Nullable, + price_step -> Nullable, + max_tx_ex_mem -> Nullable, + max_tx_ex_steps -> Nullable, + max_block_ex_mem -> Nullable, + max_block_ex_steps -> Nullable, + max_val_size -> Nullable, + collateral_percent -> Nullable, + max_collateral_inputs -> Nullable, + block_id -> Int8, + extra_entropy -> Nullable, + coins_per_utxo_size -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + epoch_stake (id) { + id -> Int8, + addr_id -> Int8, + pool_id -> Int8, + amount -> Numeric, + epoch_no -> Int4, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + epoch_sync_time (id) { + id -> Int8, + no -> Int8, + seconds -> Int8, + state -> Syncstatetype, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + extra_key_witness (id) { + id -> Int8, + hash -> Bytea, + tx_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + ma_tx_mint (id) { + id -> Int8, + quantity -> Numeric, + tx_id -> Int8, + ident -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + ma_tx_out (id) { + id -> Int8, + quantity -> Numeric, + tx_out_id -> Int8, + ident -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + meta (id) { + id -> Int8, + start_time -> Timestamp, + network_name -> Varchar, + version -> Varchar, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + multi_asset (id) { + id -> Int8, + policy -> Bytea, + name -> Bytea, + fingerprint -> Varchar, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + param_proposal (id) { + id -> Int8, + epoch_no -> Int4, + key -> Bytea, + min_fee_a -> Nullable, + min_fee_b -> Nullable, + max_block_size -> Nullable, + max_tx_size -> Nullable, + max_bh_size -> Nullable, + key_deposit -> Nullable, + pool_deposit -> Nullable, + max_epoch -> Nullable, + optimal_pool_count -> Nullable, + influence -> Nullable, + monetary_expand_rate -> Nullable, + treasury_growth_rate -> Nullable, + decentralisation -> Nullable, + entropy -> Nullable, + protocol_major -> Nullable, + protocol_minor -> Nullable, + min_utxo_value -> Nullable, + min_pool_cost -> Nullable, + cost_model_id -> Nullable, + price_mem -> Nullable, + price_step -> Nullable, + max_tx_ex_mem -> Nullable, + max_tx_ex_steps -> Nullable, + max_block_ex_mem -> Nullable, + max_block_ex_steps -> Nullable, + max_val_size -> Nullable, + collateral_percent -> Nullable, + max_collateral_inputs -> Nullable, + registered_tx_id -> Int8, + coins_per_utxo_size -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + pool_hash (id) { + id -> Int8, + hash_raw -> Bytea, + view -> Varchar, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + pool_metadata_ref (id) { + id -> Int8, + pool_id -> Int8, + url -> Varchar, + hash -> Bytea, + registered_tx_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + pool_offline_data (id) { + id -> Int8, + pool_id -> Int8, + ticker_name -> Varchar, + hash -> Bytea, + json -> Jsonb, + bytes -> Bytea, + pmr_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + pool_offline_fetch_error (id) { + id -> Int8, + pool_id -> Int8, + fetch_time -> Timestamp, + pmr_id -> Int8, + fetch_error -> Varchar, + retry_count -> Int4, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + pool_owner (id) { + id -> Int8, + addr_id -> Int8, + pool_update_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + pool_relay (id) { + id -> Int8, + update_id -> Int8, + ipv4 -> Nullable, + ipv6 -> Nullable, + dns_name -> Nullable, + dns_srv_name -> Nullable, + port -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + pool_retire (id) { + id -> Int8, + hash_id -> Int8, + cert_index -> Int4, + announced_tx_id -> Int8, + retiring_epoch -> Int4, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + pool_update (id) { + id -> Int8, + hash_id -> Int8, + cert_index -> Int4, + vrf_key_hash -> Bytea, + pledge -> Numeric, + active_epoch_no -> Int8, + meta_id -> Nullable, + margin -> Float8, + fixed_cost -> Numeric, + registered_tx_id -> Int8, + reward_addr_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + pot_transfer (id) { + id -> Int8, + cert_index -> Int4, + treasury -> Numeric, + reserves -> Numeric, + tx_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + redeemer (id) { + id -> Int8, + tx_id -> Int8, + unit_mem -> Int8, + unit_steps -> Int8, + fee -> Nullable, + purpose -> Scriptpurposetype, + index -> Int4, + script_hash -> Nullable, + redeemer_data_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + redeemer_data (id) { + id -> Int8, + hash -> Bytea, + tx_id -> Int8, + value -> Nullable, + bytes -> Bytea, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + reference_tx_in (id) { + id -> Int8, + tx_in_id -> Int8, + tx_out_id -> Int8, + tx_out_index -> Int2, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + reserve (id) { + id -> Int8, + addr_id -> Int8, + cert_index -> Int4, + amount -> Numeric, + tx_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + reserved_pool_ticker (id) { + id -> Int8, + name -> Varchar, + pool_hash -> Bytea, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + reward (id) { + id -> Int8, + addr_id -> Int8, + #[sql_name = "type"] + type_ -> Rewardtype, + amount -> Numeric, + earned_epoch -> Int8, + spendable_epoch -> Int8, + pool_id -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + schema_version (id) { + id -> Int8, + stage_one -> Int8, + stage_two -> Int8, + stage_three -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + script (id) { + id -> Int8, + tx_id -> Int8, + hash -> Bytea, + #[sql_name = "type"] + type_ -> Scripttype, + json -> Nullable, + bytes -> Nullable, + serialised_size -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + slot_leader (id) { + id -> Int8, + hash -> Bytea, + pool_hash_id -> Nullable, + description -> Varchar, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + stake_address (id) { + id -> Int8, + hash_raw -> Bytea, + view -> Varchar, + script_hash -> Nullable, + tx_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + stake_deregistration (id) { + id -> Int8, + addr_id -> Int8, + cert_index -> Int4, + epoch_no -> Int4, + tx_id -> Int8, + redeemer_id -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + stake_registration (id) { + id -> Int8, + addr_id -> Int8, + cert_index -> Int4, + epoch_no -> Int4, + tx_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + treasury (id) { + id -> Int8, + addr_id -> Int8, + cert_index -> Int4, + amount -> Numeric, + tx_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + tx (id) { + id -> Int8, + hash -> Bytea, + block_id -> Int8, + block_index -> Int4, + out_sum -> Numeric, + fee -> Numeric, + deposit -> Int8, + size -> Int4, + invalid_before -> Nullable, + invalid_hereafter -> Nullable, + valid_contract -> Bool, + script_size -> Int4, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + tx_in (id) { + id -> Int8, + tx_in_id -> Int8, + tx_out_id -> Int8, + tx_out_index -> Int2, + redeemer_id -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + tx_metadata (id) { + id -> Int8, + key -> Numeric, + json -> Nullable, + bytes -> Bytea, + tx_id -> Int8, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + tx_out (id) { + id -> Int8, + tx_id -> Int8, + index -> Int2, + address -> Varchar, + address_raw -> Bytea, + address_has_script -> Bool, + payment_cred -> Nullable, + stake_address_id -> Nullable, + value -> Numeric, + data_hash -> Nullable, + inline_datum_id -> Nullable, + reference_script_id -> Nullable, + } +} + +table! { + use diesel::sql_types::*; + use crate::db::types::*; + + withdrawal (id) { + id -> Int8, + addr_id -> Int8, + amount -> Numeric, + redeemer_id -> Nullable, + tx_id -> Int8, + } +} + +joinable!(ada_pots -> block (block_id)); +joinable!(block -> slot_leader (slot_leader_id)); +joinable!(collateral_tx_out -> datum (inline_datum_id)); +joinable!(collateral_tx_out -> script (reference_script_id)); +joinable!(collateral_tx_out -> stake_address (stake_address_id)); +joinable!(collateral_tx_out -> tx (tx_id)); +joinable!(cost_model -> block (block_id)); +joinable!(datum -> tx (tx_id)); +joinable!(delegation -> pool_hash (pool_hash_id)); +joinable!(delegation -> redeemer (redeemer_id)); +joinable!(delegation -> stake_address (addr_id)); +joinable!(delegation -> tx (tx_id)); +joinable!(epoch_param -> block (block_id)); +joinable!(epoch_param -> cost_model (cost_model_id)); +joinable!(epoch_stake -> pool_hash (pool_id)); +joinable!(epoch_stake -> stake_address (addr_id)); +joinable!(extra_key_witness -> tx (tx_id)); +joinable!(ma_tx_mint -> multi_asset (ident)); +joinable!(ma_tx_mint -> tx (tx_id)); +joinable!(ma_tx_out -> multi_asset (ident)); +joinable!(ma_tx_out -> tx_out (tx_out_id)); +joinable!(param_proposal -> cost_model (cost_model_id)); +joinable!(param_proposal -> tx (registered_tx_id)); +joinable!(pool_metadata_ref -> pool_hash (pool_id)); +joinable!(pool_metadata_ref -> tx (registered_tx_id)); +joinable!(pool_offline_data -> pool_hash (pool_id)); +joinable!(pool_offline_data -> pool_metadata_ref (pmr_id)); +joinable!(pool_offline_fetch_error -> pool_hash (pool_id)); +joinable!(pool_offline_fetch_error -> pool_metadata_ref (pmr_id)); +joinable!(pool_owner -> pool_update (pool_update_id)); +joinable!(pool_owner -> stake_address (addr_id)); +joinable!(pool_relay -> pool_update (update_id)); +joinable!(pool_retire -> pool_hash (hash_id)); +joinable!(pool_retire -> tx (announced_tx_id)); +joinable!(pool_update -> pool_hash (hash_id)); +joinable!(pool_update -> pool_metadata_ref (meta_id)); +joinable!(pool_update -> stake_address (reward_addr_id)); +joinable!(pool_update -> tx (registered_tx_id)); +joinable!(pot_transfer -> tx (tx_id)); +joinable!(redeemer -> redeemer_data (redeemer_data_id)); +joinable!(redeemer -> tx (tx_id)); +joinable!(redeemer_data -> tx (tx_id)); +joinable!(reserve -> stake_address (addr_id)); +joinable!(reserve -> tx (tx_id)); +joinable!(reward -> pool_hash (pool_id)); +joinable!(reward -> stake_address (addr_id)); +joinable!(script -> tx (tx_id)); +joinable!(slot_leader -> pool_hash (pool_hash_id)); +joinable!(stake_address -> tx (tx_id)); +joinable!(stake_deregistration -> redeemer (redeemer_id)); +joinable!(stake_deregistration -> stake_address (addr_id)); +joinable!(stake_deregistration -> tx (tx_id)); +joinable!(stake_registration -> stake_address (addr_id)); +joinable!(stake_registration -> tx (tx_id)); +joinable!(treasury -> stake_address (addr_id)); +joinable!(treasury -> tx (tx_id)); +joinable!(tx -> block (block_id)); +joinable!(tx_in -> redeemer (redeemer_id)); +joinable!(tx_metadata -> tx (tx_id)); +joinable!(tx_out -> datum (inline_datum_id)); +joinable!(tx_out -> script (reference_script_id)); +joinable!(tx_out -> stake_address (stake_address_id)); +joinable!(tx_out -> tx (tx_id)); +joinable!(withdrawal -> redeemer (redeemer_id)); +joinable!(withdrawal -> stake_address (addr_id)); +joinable!(withdrawal -> tx (tx_id)); + +allow_tables_to_appear_in_same_query!( + ada_pots, + block, + collateral_tx_in, + collateral_tx_out, + cost_model, + datum, + delegation, + delisted_pool, + epoch, + epoch_param, + epoch_stake, + epoch_sync_time, + extra_key_witness, + ma_tx_mint, + ma_tx_out, + meta, + multi_asset, + param_proposal, + pool_hash, + pool_metadata_ref, + pool_offline_data, + pool_offline_fetch_error, + pool_owner, + pool_relay, + pool_retire, + pool_update, + pot_transfer, + redeemer, + redeemer_data, + reference_tx_in, + reserve, + reserved_pool_ticker, + reward, + schema_version, + script, + slot_leader, + stake_address, + stake_deregistration, + stake_registration, + treasury, + tx, + tx_in, + tx_metadata, + tx_out, + withdrawal, +); diff --git a/src/vit-testing/db-sync-explorer/src/db/types.rs b/src/vit-testing/db-sync-explorer/src/db/types.rs new file mode 100644 index 0000000000..a9ddd091e6 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/db/types.rs @@ -0,0 +1,41 @@ +//! Extra types defined in postgres +//! +//! db-sync defines a few types in postgres. When diesel generates a schema, it uses these types, +//! but since they don't exist on the Rust side, type checking fails. So we provide them in this +//! module +//! +//! Many types here have slightly wonky capitalization, but that's just what diesel expects + +use diesel::SqlType; +use diesel_derive_enum::DbEnum; + +#[derive(Debug, Clone, Copy, DbEnum, SqlType)] +pub enum Rewardtype { + Leader, + Member, + Reserves, + Treasury, + Refund, +} + +#[derive(Debug, Clone, Copy, DbEnum, SqlType)] +pub enum Scriptpurposetype { + Spend, + Mint, + Cert, + Reward, +} + +#[derive(Debug, Clone, Copy, DbEnum, SqlType)] +pub enum Scripttype { + Multisig, + Timelock, + PlutusV1, + PlutusV2, +} + +#[derive(Debug, Clone, Copy, DbEnum, SqlType)] +pub enum Syncstatetype { + Lagging, + Following, +} diff --git a/src/vit-testing/db-sync-explorer/src/lib.rs b/src/vit-testing/db-sync-explorer/src/lib.rs new file mode 100644 index 0000000000..bc87406195 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/lib.rs @@ -0,0 +1,8 @@ +mod cli; +mod config; +mod db; +pub mod rest; + +pub use cli::Args; +pub use config::Config; +pub use db::{connect, BehindDuration, DbConfig, TransactionConfirmation}; diff --git a/src/vit-testing/db-sync-explorer/src/rest/mod.rs b/src/vit-testing/db-sync-explorer/src/rest/mod.rs new file mode 100644 index 0000000000..2d24cd45f5 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/mod.rs @@ -0,0 +1 @@ +pub mod v0; diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/context.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/context.rs new file mode 100644 index 0000000000..161808e7ff --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/context.rs @@ -0,0 +1,29 @@ +use std::sync::Arc; +use tokio::sync::RwLock; + +use crate::db::DbPool; + +pub type SharedContext = Arc>; + +pub struct Context { + pub db_connection_pool: DbPool, + pub token: Option, +} + +impl Context { + pub fn new(db_connection_pool: DbPool, token: Option) -> Self { + Self { + db_connection_pool, + token, + } + } + + pub fn token(&self) -> &Option { + &self.token + } +} + +pub fn new_shared_context(db_connection_pool: DbPool, token: Option) -> SharedContext { + let context = Context::new(db_connection_pool, token); + Arc::new(RwLock::new(context)) +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/api_token.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/api_token.rs new file mode 100644 index 0000000000..2e24efe9a2 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/api_token.rs @@ -0,0 +1,39 @@ +use crate::rest::v0::context::SharedContext; +use jortestkit::web::api_token::{APIToken, APITokenManager, TokenError, API_TOKEN_HEADER}; +use warp::{Filter, Rejection}; + +pub async fn api_token_filter( + context: SharedContext, +) -> impl Filter + Clone { + let is_token_enabled = context.read().await.token().is_some(); + + if is_token_enabled { + warp::header::header(API_TOKEN_HEADER) + .and(warp::any().map(move || context.clone())) + .and_then(authorize_token) + .and(warp::any()) + .untuple_one() + .boxed() + } else { + warp::any().boxed() + } +} + +pub async fn authorize_token(token: String, context: SharedContext) -> Result<(), Rejection> { + let api_token = APIToken::from_string(token).map_err(warp::reject::custom)?; + + let context = context.read().await; + let maybe_token = context.token().as_ref(); + + if maybe_token.is_none() { + return Ok(()); + } + + let manager = + APITokenManager::new(maybe_token.unwrap().clone()).map_err(warp::reject::custom)?; + + if !manager.is_token_valid(api_token) { + return Err(warp::reject::custom(TokenError::UnauthorizedToken)); + } + Ok(()) +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/health/handlers.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/health/handlers.rs new file mode 100644 index 0000000000..4d8ce9d55a --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/health/handlers.rs @@ -0,0 +1,6 @@ +use crate::rest::v0::context::SharedContext; +use warp::{Rejection, Reply}; + +pub async fn check_health(_context: SharedContext) -> Result { + Ok(warp::reply()) +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/health/mod.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/health/mod.rs new file mode 100644 index 0000000000..2a6c75df57 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/health/mod.rs @@ -0,0 +1,4 @@ +mod handlers; +mod routes; + +pub use routes::filter; diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/health/routes.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/health/routes.rs new file mode 100644 index 0000000000..62dd133b27 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/health/routes.rs @@ -0,0 +1,19 @@ +use super::handlers::*; +use crate::rest::v0::context::SharedContext; +use warp::filters::BoxedFilter; +use warp::{Filter, Rejection, Reply}; + +pub async fn filter( + root: BoxedFilter<()>, + context: SharedContext, +) -> impl Filter + Clone { + let with_context = warp::any().map(move || context.clone()); + + let health_filter = warp::path::end() + .and(warp::get()) + .and(with_context) + .and_then(check_health) + .boxed(); + + root.and(health_filter).boxed() +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/handlers.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/handlers.rs new file mode 100644 index 0000000000..882b5c28da --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/handlers.rs @@ -0,0 +1,8 @@ +use super::logic; +use crate::rest::v0::context::SharedContext; +use crate::rest::v0::result::HandlerResult; +use warp::{Rejection, Reply}; + +pub async fn get_meta_info(context: SharedContext) -> Result { + Ok(HandlerResult(logic::get_meta_info(context).await)) +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/logic.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/logic.rs new file mode 100644 index 0000000000..a40524cc49 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/logic.rs @@ -0,0 +1,18 @@ +use crate::db::Meta; +use crate::rest::v0::context::SharedContext; +use crate::rest::v0::errors::HandleError; +use diesel::RunQueryDsl; + +pub async fn get_meta_info(context: SharedContext) -> Result, HandleError> { + let pool = &context.read().await.db_connection_pool; + let mut db_conn = pool.get().map_err(HandleError::Connection)?; + + tokio::task::spawn_blocking(move || { + use crate::db::schema::meta; + meta::table + .load(&mut db_conn) + .map_err(HandleError::Database) + }) + .await + .map_err(HandleError::Join)? +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/mod.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/mod.rs new file mode 100644 index 0000000000..a2344c5167 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/mod.rs @@ -0,0 +1,5 @@ +mod handlers; +mod logic; +mod routes; + +pub use routes::filter; diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/routes.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/routes.rs new file mode 100644 index 0000000000..8cb5fac62c --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/meta/routes.rs @@ -0,0 +1,19 @@ +use super::handlers::*; +use crate::rest::v0::context::SharedContext; +use warp::filters::BoxedFilter; +use warp::{Filter, Rejection, Reply}; + +pub async fn filter( + root: BoxedFilter<()>, + context: SharedContext, +) -> impl Filter + Clone { + let with_context = warp::any().map(move || context.clone()); + + let meta_info = warp::path::end() + .and(warp::get()) + .and(with_context) + .and_then(get_meta_info) + .boxed(); + + root.and(meta_info).boxed() +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/mod.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/mod.rs new file mode 100644 index 0000000000..d4553c60ae --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/mod.rs @@ -0,0 +1,35 @@ +use crate::rest::v0::context::SharedContext; +use warp::filters::BoxedFilter; +use warp::{Filter, Rejection, Reply}; + +mod api_token; +mod health; +mod meta; +mod sync; +mod tx; + +pub async fn filter( + root: BoxedFilter<()>, + context: SharedContext, +) -> impl Filter + Clone { + let sync_root = warp::path!("sync" / ..); + let sync_filter = sync::filter(sync_root.boxed(), context.clone()).await; + + let health_root = warp::path!("health" / ..); + let health_filter = health::filter(health_root.boxed(), context.clone()).await; + + let tx_root = warp::path!("tx" / ..); + let tx_filter = tx::filter(tx_root.boxed(), context.clone()).await; + + let meta_root = warp::path!("meta" / ..); + let meta_filter = meta::filter(meta_root.boxed(), context.clone()).await; + + let api_token_filter = if context.read().await.token().is_some() { + api_token::api_token_filter(context).await.boxed() + } else { + warp::any().boxed() + }; + + root.and(api_token_filter.and(sync_filter.or(health_filter).or(tx_filter).or(meta_filter))) + .boxed() +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/handlers.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/handlers.rs new file mode 100644 index 0000000000..2be008fd8b --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/handlers.rs @@ -0,0 +1,14 @@ +use super::logic; +use crate::rest::v0::context::SharedContext; +use crate::rest::v0::result::HandlerResult; +use warp::{Rejection, Reply}; + +pub async fn get_interval_behind_now_filter( + context: SharedContext, +) -> Result { + Ok(HandlerResult(logic::get_interval_behind_now(context).await)) +} + +pub async fn progress_filter(context: SharedContext) -> Result { + Ok(HandlerResult(logic::get_sync_progress(context).await)) +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/logic.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/logic.rs new file mode 100644 index 0000000000..4adb60caaf --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/logic.rs @@ -0,0 +1,31 @@ +use crate::db::Progress; +use crate::rest::v0::context::SharedContext; +use crate::rest::v0::errors::HandleError; +use crate::{db, BehindDuration}; + +pub async fn get_interval_behind_now( + context: SharedContext, +) -> Result { + let pool = &context.read().await.db_connection_pool; + let result = db::query::behind(pool).await?; + if result.is_empty() || result.len() > 1 { + Err(HandleError::DatabaseInconsistency( + "expected only 1 record for maximum block time".to_string(), + )) + } else { + Ok(result[0].clone()) + } +} + +pub async fn get_sync_progress(context: SharedContext) -> Result { + let pool = &context.read().await.db_connection_pool; + let result = db::query::sync_progress(pool).await?; + + if result.is_empty() || result.len() > 1 { + Err(HandleError::DatabaseInconsistency( + "expected only 1 record for sync progress".to_string(), + )) + } else { + Ok(result[0].clone()) + } +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/mod.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/mod.rs new file mode 100644 index 0000000000..a2344c5167 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/mod.rs @@ -0,0 +1,5 @@ +mod handlers; +mod logic; +mod routes; + +pub use routes::filter; diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/routes.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/routes.rs new file mode 100644 index 0000000000..05d766cfc6 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/sync/routes.rs @@ -0,0 +1,24 @@ +use super::handlers::*; +use crate::rest::v0::context::SharedContext; +use warp::filters::BoxedFilter; +use warp::{Filter, Rejection, Reply}; + +pub async fn filter( + root: BoxedFilter<()>, + context: SharedContext, +) -> impl Filter + Clone { + let with_context = warp::any().map(move || context.clone()); + + let behind = warp::path!("behind") + .and(warp::get()) + .and(with_context.clone()) + .and_then(get_interval_behind_now_filter) + .boxed(); + + let progress = warp::path!("progress") + .and(warp::get()) + .and(with_context) + .and_then(progress_filter); + + root.and(behind.or(progress)).boxed() +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/handlers.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/handlers.rs new file mode 100644 index 0000000000..26f9d238ca --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/handlers.rs @@ -0,0 +1,8 @@ +use super::logic; +use crate::rest::v0::context::SharedContext; +use crate::rest::v0::result::HandlerResult; +use warp::{Rejection, Reply}; + +pub async fn get_tx_by_hash(hash: String, context: SharedContext) -> Result { + Ok(HandlerResult(logic::get_tx_by_hash(hash, context).await)) +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/logic.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/logic.rs new file mode 100644 index 0000000000..3462691ddf --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/logic.rs @@ -0,0 +1,13 @@ +use crate::rest::v0::context::SharedContext; +use crate::rest::v0::errors::HandleError; +use crate::{db, TransactionConfirmation}; + +pub async fn get_tx_by_hash( + hash: String, + context: SharedContext, +) -> Result, HandleError> { + let pool = &context.read().await.db_connection_pool; + db::query::hash(hash, pool) + .await + .map(|x| x.into_iter().map(Into::into).collect()) +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/mod.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/mod.rs new file mode 100644 index 0000000000..a2344c5167 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/mod.rs @@ -0,0 +1,5 @@ +mod handlers; +mod logic; +mod routes; + +pub use routes::filter; diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/routes.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/routes.rs new file mode 100644 index 0000000000..42dd83d32f --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/endpoints/tx/routes.rs @@ -0,0 +1,19 @@ +use super::handlers::*; +use crate::rest::v0::context::SharedContext; +use warp::filters::BoxedFilter; +use warp::{Filter, Rejection, Reply}; + +pub async fn filter( + root: BoxedFilter<()>, + context: SharedContext, +) -> impl Filter + Clone { + let with_context = warp::any().map(move || context.clone()); + + let tx_by_hash = warp::path!("hash" / String) + .and(warp::get()) + .and(with_context) + .and_then(get_tx_by_hash) + .boxed(); + + root.and(tx_by_hash).boxed() +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/errors.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/errors.rs new file mode 100644 index 0000000000..c0820303b3 --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/errors.rs @@ -0,0 +1,77 @@ +use thiserror::Error; +use warp::{reply::Response, Rejection, Reply}; + +#[derive(Error, Debug)] +pub enum HandleError { + #[error("The data requested data for `{0}` is not available")] + NotFound(String), + + #[error("Internal error")] + Connection(#[from] diesel::r2d2::PoolError), + + #[error("Unexpected state of database: {0}")] + DatabaseInconsistency(String), + + #[error("Unauthorized token")] + UnauthorizedToken, + + #[error("Connection")] + Database(#[from] diesel::result::Error), + + #[error("Internal error, cause: {0}")] + InternalError(String), + + #[error("Invalid header {0}, cause: {1}")] + InvalidHeader(&'static str, &'static str), + + #[error("Bad Request: {0}")] + BadRequest(String), + + #[error("Request interrupted")] + Join(#[from] tokio::task::JoinError), +} + +impl HandleError { + fn to_status_code(&self) -> warp::http::StatusCode { + match self { + HandleError::NotFound(_) => warp::http::StatusCode::NOT_FOUND, + HandleError::Connection(_) => warp::http::StatusCode::SERVICE_UNAVAILABLE, + HandleError::InternalError(_) => warp::http::StatusCode::INTERNAL_SERVER_ERROR, + HandleError::UnauthorizedToken => warp::http::StatusCode::UNAUTHORIZED, + HandleError::InvalidHeader(_, _) => warp::http::StatusCode::BAD_REQUEST, + HandleError::BadRequest(_) => warp::http::StatusCode::BAD_REQUEST, + HandleError::DatabaseInconsistency(_) => warp::http::StatusCode::SERVICE_UNAVAILABLE, + HandleError::Database(_) => warp::http::StatusCode::SERVICE_UNAVAILABLE, + HandleError::Join(_) => warp::http::StatusCode::SERVICE_UNAVAILABLE, + } + } + + fn to_message(&self) -> String { + format!("{}", self) + } + + fn to_response(&self) -> Response { + let status_code = self.to_status_code(); + warp::reply::with_status(warp::reply::json(&self.to_json()), status_code).into_response() + } + + fn to_json(&self) -> serde_json::Value { + serde_json::json!({"code": self.to_status_code().as_u16(), "message" : self.to_message()}) + } +} + +impl warp::Reply for HandleError { + fn into_response(self) -> Response { + self.to_response() + } +} + +impl warp::reject::Reject for HandleError {} + +pub async fn handle_rejection(err: Rejection) -> Result { + if let Some(handle_error) = err.find::() { + return Ok(handle_error.to_response()); + } + + Err(err) +} diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/handlers.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/handlers.rs new file mode 100644 index 0000000000..8db8c8fc1d --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/handlers.rs @@ -0,0 +1,57 @@ +use bb8::{Pool}; +use crate::db::{Meta}; +use axum::{ + + extract::{ State}, + http::{ StatusCode}, + + +}; +use bb8_postgres::PostgresConnectionManager; +use tokio_postgres::NoTls; +type ConnectionPool = Pool>; +use diesel::RunQueryDsl; + + +pub async fn get_meta_information( + State(pool): State, +) -> Result { + let mut conn = pool.get().await.map_err(internal_error)?; + + // use crate::db::schema::meta; + + // let meta = meta::table.load::(&mut *conn).map_err(internal_error)?; + + let row = conn + .query_one("select 1 + 1", &[]) + .await + .map_err(internal_error)?; + + let two: i32 = row.try_get(0).map_err(internal_error)?; + + Ok(two.to_string()) + /* if meta.is_empty() || meta.len() > 2 { + Err((StatusCode::INTERNAL_SERVER_ERROR,"data inconsistency.. expected only one meta record".to_string())) + } else { + Ok(meta[0].clone()) + }*/ +} + +fn internal_error(err: E) -> (StatusCode, String) + where + E: std::error::Error, +{ + (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + +} +/* +let results: Vec = db.exec(move |conn| behind().load(conn)).unwrap(); +let time : DateTime = results[0].behind_by.into(); +let diff: chrono::Duration = chrono::offset::Utc::now() - time; +println!("{}",diff); + +let results: Vec = db.exec(move |conn| hash("7b27ea78e32fdb4522ad63495a0f89289663435f3904ff5d12c529a47c8a37b8").load::(conn)).unwrap(); +let results: Vec = results.into_iter().map(TransactionConfirmation::from).collect(); +println!("{results:?}"); + +Ok(())*/ \ No newline at end of file diff --git a/src/vit-testing/db-sync-explorer/src/rest/v0/mod.rs b/src/vit-testing/db-sync-explorer/src/rest/v0/mod.rs new file mode 100644 index 0000000000..c215b0f0ed --- /dev/null +++ b/src/vit-testing/db-sync-explorer/src/rest/v0/mod.rs @@ -0,0 +1,48 @@ +pub mod context; +pub mod endpoints; +pub mod errors; +pub mod result; + +pub use context::{new_shared_context, Context, SharedContext}; +use warp::{Filter, Rejection, Reply}; + +const V0_REQUEST_TRACE_NAME: &str = "v0_request"; + +pub async fn filter( + ctx: SharedContext, +) -> impl Filter + Clone { + let api_root = warp::path!("api" / "v0" / ..); + // log request statistics + let log = warp::filters::trace::trace(|info| { + use http_zipkin::get_trace_context; + use tracing::field::Empty; + let span = tracing::span!( + tracing::Level::DEBUG, + "rest_api_request", + method = %info.method(), + path = info.path(), + version = ?info.version(), + remote_addr = Empty, + trace_id = Empty, + span_id = Empty, + parent_span_id = Empty, + ); + if let Some(remote_addr) = info.remote_addr() { + span.record("remote_addr", remote_addr.to_string().as_str()); + } + if let Some(trace_context) = get_trace_context(info.request_headers()) { + span.record("trace_id", trace_context.trace_id().to_string().as_str()); + span.record("span_id", trace_context.span_id().to_string().as_str()); + if let Some(parent_span_id) = trace_context.parent_id() { + span.record("parent_span_id", parent_span_id.to_string().as_str()); + } + } + span + }); + + let v0 = endpoints::filter(api_root.boxed(), ctx.clone()).await; + + v0.with(warp::trace::named(V0_REQUEST_TRACE_NAME)) + .recover(errors::handle_rejection) + .with(log) +} diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/address.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/address.rs index a74bf851c5..c0798358d7 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/address.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/address.rs @@ -1,8 +1,8 @@ +use crate::cardano_cli::mock::command::write_to_file_or_println; +use crate::cardano_cli::mock::fake; use std::io; use std::path::PathBuf; use structopt::StructOpt; -use crate::cardano_cli::mock::command::write_to_file_or_println; -use crate::cardano_cli::mock::fake; #[derive(StructOpt, Debug)] pub enum Address { diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/query.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/query.rs index 3d94688944..b3d0ac7d19 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/query.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/query.rs @@ -1,8 +1,8 @@ +use crate::cardano_cli::mock::command::write_to_file_or_println; +use crate::cardano_cli::mock::fake; use std::io::Error; use std::path::PathBuf; use structopt::StructOpt; -use crate::cardano_cli::mock::command::write_to_file_or_println; -use crate::cardano_cli::mock::fake; #[derive(StructOpt, Debug)] pub enum Query { diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/stake_address.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/stake_address.rs index dd64a01f30..2e3ee96ae1 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/stake_address.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/stake_address.rs @@ -1,9 +1,9 @@ +use crate::cardano_cli::mock::command::write_to_file_or_println; +use crate::cardano_cli::mock::fake; use std::fs::File; use std::io::Write; use std::path::PathBuf; use structopt::StructOpt; -use crate::cardano_cli::mock::command::write_to_file_or_println; -use crate::cardano_cli::mock::fake; #[derive(StructOpt, Debug)] pub enum StakeAddress { diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/transaction.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/transaction.rs index 3d6a3f6071..d7b0e4aff6 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/transaction.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/mock/command/transaction.rs @@ -1,10 +1,10 @@ +use crate::cardano_cli::mock::command::write_to_file_or_println; +use crate::cardano_cli::mock::fake; use std::fs::File; use std::io; use std::io::Write; use std::path::PathBuf; use structopt::StructOpt; -use crate::cardano_cli::mock::command::write_to_file_or_println; -use crate::cardano_cli::mock::fake; #[derive(StructOpt, Debug)] pub enum Transaction { diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/mod.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/mod.rs index f8ff667a78..9620981dc9 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/mod.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/mod.rs @@ -1,4 +1,4 @@ -mod wrapper; mod mock; +mod wrapper; pub use mock::Command as MockCommand; diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/address.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/address.rs index 566c292979..fa373d3111 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/address.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/address.rs @@ -1,9 +1,9 @@ +use crate::cardano_cli::wrapper::cli::command; +use crate::cardano_cli::wrapper::Error; +use snapshot_trigger_service::config::NetworkType; use std::io::Write; use std::path::Path; use std::process::ExitStatus; -use snapshot_trigger_service::config::NetworkType; -use crate::cardano_cli::wrapper::cli::command; -use crate::cardano_cli::wrapper::Error; pub struct Address { address_command: command::Address, diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/query.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/query.rs index a2ac338c84..596e2a39b8 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/query.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/query.rs @@ -1,12 +1,12 @@ +use crate::cardano_cli::mock::fake::Tip; +use crate::cardano_cli::wrapper::cli::command; +use crate::cardano_cli::wrapper::Error; use jortestkit::prelude::ProcessOutput; use serde::{Deserialize, Serialize}; +use snapshot_trigger_service::config::NetworkType; use std::collections::HashMap; use std::path::Path; use std::process::ExitStatus; -use snapshot_trigger_service::config::NetworkType; -use crate::cardano_cli::mock::fake::Tip; -use crate::cardano_cli::wrapper::cli::command; -use crate::cardano_cli::wrapper::Error; pub struct Query { query_command: command::Query, diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/stake_address.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/stake_address.rs index ba6b2d7191..12ef3b9cb4 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/stake_address.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/stake_address.rs @@ -1,9 +1,9 @@ +use crate::cardano_cli::wrapper::cli::command; +use crate::cardano_cli::wrapper::Error; +use snapshot_trigger_service::config::NetworkType; use std::io::Write; use std::path::Path; use std::process::ExitStatus; -use snapshot_trigger_service::config::NetworkType; -use crate::cardano_cli::wrapper::cli::command; -use crate::cardano_cli::wrapper::Error; pub struct StakeAddress { stake_address_command: command::StakeAddress, diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/transaction.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/transaction.rs index 2f4238a465..fd5944468e 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/transaction.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/api/transaction.rs @@ -1,13 +1,13 @@ +use crate::cardano_cli::wrapper::cli::command; +use crate::cardano_cli::wrapper::Error; +use jortestkit::prelude::ProcessOutput; +use snapshot_trigger_service::config::NetworkType; use std::fs::File; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::ExitStatus; use tracing::debug; -use jortestkit::prelude::ProcessOutput; -use snapshot_trigger_service::config::NetworkType; use uuid::Uuid; -use crate::cardano_cli::wrapper::cli::command; -use crate::cardano_cli::wrapper::Error; pub struct Transaction { transaction_command: command::Transaction, diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/address/build.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/address/build.rs index 10ecc6f29c..6d61b07528 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/address/build.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/address/build.rs @@ -1,8 +1,8 @@ +use crate::cardano_cli::wrapper::utils::CommandExt; +use snapshot_trigger_service::config::NetworkType; use std::path::Path; use std::process::Command; use tracing::debug; -use snapshot_trigger_service::config::NetworkType; -use crate::cardano_cli::wrapper::utils::CommandExt; pub struct AddressBuilder { command: Command, diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/mod.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/mod.rs index c730b8ba17..329bea5079 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/mod.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/mod.rs @@ -6,8 +6,8 @@ mod transaction; pub use address::Address; pub use query::Query; pub use stake_address::StakeAddress; -pub use transaction::Transaction; use std::process::Command; +pub use transaction::Transaction; pub struct Root { command: Command, diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/query.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/query.rs index 1c47fef501..8b57bd9a9f 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/query.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/query.rs @@ -1,7 +1,7 @@ +use crate::cardano_cli::wrapper::utils::CommandExt; +use snapshot_trigger_service::config::NetworkType; use std::path::Path; use std::process::Command; -use snapshot_trigger_service::config::NetworkType; -use crate::cardano_cli::wrapper::utils::CommandExt; pub struct Query { command: Command, diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/stake_address/build.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/stake_address/build.rs index efb84ceb75..3c0f260b14 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/stake_address/build.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/stake_address/build.rs @@ -1,8 +1,8 @@ +use crate::cardano_cli::wrapper::utils::CommandExt; +use snapshot_trigger_service::config::NetworkType; use std::path::Path; use std::process::Command; use tracing::debug; -use snapshot_trigger_service::config::NetworkType; -use crate::cardano_cli::wrapper::utils::CommandExt; pub struct StakeAddressBuilder { command: Command, diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/transaction/build.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/transaction/build.rs index e042229d06..653ce3f7e0 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/transaction/build.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/transaction/build.rs @@ -1,8 +1,8 @@ +use crate::cardano_cli::wrapper::utils::CommandExt; +use snapshot_trigger_service::config::NetworkType; use std::path::Path; use std::process::Command; use tracing::debug; -use snapshot_trigger_service::config::NetworkType; -use crate::cardano_cli::wrapper::utils::CommandExt; pub struct Builder { command: Command, diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/transaction/sign.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/transaction/sign.rs index 440ed5f452..f4e73dbdad 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/transaction/sign.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/command/transaction/sign.rs @@ -1,8 +1,8 @@ +use crate::cardano_cli::wrapper::utils::CommandExt; +use snapshot_trigger_service::config::NetworkType; use std::path::Path; use std::process::Command; use tracing::debug; -use snapshot_trigger_service::config::NetworkType; -use crate::cardano_cli::wrapper::utils::CommandExt; pub struct Builder { command: Command, diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/mod.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/mod.rs index fcad503c7c..b3e0e59c1f 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/mod.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/cli/mod.rs @@ -1,7 +1,7 @@ -use std::path::PathBuf; -use std::process::Command; use crate::cardano_cli::wrapper::cli::api::{Address, Query, StakeAddress, Transaction}; use crate::cardano_cli::wrapper::cli::command::Root; +use std::path::PathBuf; +use std::process::Command; pub mod api; pub mod command; diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/data.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/data.rs index 482df4b1a7..f9566ccd00 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/data.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/data.rs @@ -1,9 +1,9 @@ #![allow(dead_code)] use super::error::Error; +use crate::cardano_cli::wrapper::utils::write_content; use serde::{Deserialize, Serialize}; use std::path::Path; -use crate::cardano_cli::wrapper::utils::write_content; #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] pub struct CardanoKeyTemplate { diff --git a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/mod.rs b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/mod.rs index fe28b93789..b734bc2161 100644 --- a/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/mod.rs +++ b/src/vit-testing/mainnet-tools/src/cardano_cli/wrapper/mod.rs @@ -5,6 +5,6 @@ mod data; mod error; pub mod utils; -pub use cli::{Api,command::*}; +pub use cli::{command::*, Api}; pub use data::CardanoKeyTemplate; pub use error::Error; diff --git a/src/voting-tools-rs/Cargo.toml b/src/voting-tools-rs/Cargo.toml index fb2dd197ab..ff57260ac9 100644 --- a/src/voting-tools-rs/Cargo.toml +++ b/src/voting-tools-rs/Cargo.toml @@ -16,7 +16,7 @@ serde_json = "1" microtype = { version = "0.7.5", features = ["serde"] } once_cell = "1" -diesel = { version = "2", features = ["postgres", "64-column-tables", "numeric", "serde_json", "r2d2"]} +diesel = { version = "2", features = ["postgres", "64-column-tables", "numeric", "serde_json", "r2d2", "chrono"]} diesel-derive-enum = "2.0.0-rc.0" bigdecimal = { version = "0.3", features = ["serde"] } diff --git a/src/voting-tools-rs/src/model/mod.rs b/src/voting-tools-rs/src/model/mod.rs index 8fabecff35..7f6392ff11 100644 --- a/src/voting-tools-rs/src/model/mod.rs +++ b/src/voting-tools-rs/src/model/mod.rs @@ -64,8 +64,11 @@ microtype! { #[derive(Debug, PartialEq, Clone)] #[string] pub String { + /// Database name DbName, + /// Database user DbUser, + /// Database host DbHost, RewardsAddr, StakeAddr, @@ -78,6 +81,7 @@ microtype! { #[secret] #[string] pub String { + /// Database password DbPass, }