From 6001d57225e7549000ef9f8298cbf0524caefefc Mon Sep 17 00:00:00 2001 From: charmful0x Date: Tue, 13 Aug 2024 16:16:23 +0200 Subject: [PATCH 1/6] feat: add /stats endpoint && fmt --- src/main.rs | 12 +++--- src/utils/env_var.rs | 2 +- src/utils/mod.rs | 4 +- src/utils/planetscale.rs | 72 ++++++++++++++++++++++++------------ src/utils/schema.rs | 72 +++++++++++++++++++++++++----------- src/utils/server_handlers.rs | 21 ++++++++--- src/utils/transaction.rs | 10 ++++- 7 files changed, 132 insertions(+), 61 deletions(-) diff --git a/src/main.rs b/src/main.rs index 870386a..2d1b556 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,10 @@ use crate::utils::archive_block::archive; -use crate::utils::schema::Network; use crate::utils::planetscale::{ps_archive_block, ps_get_latest_block_id}; -use crate::utils::server_handlers::{handle_block, weave_gm}; +use crate::utils::schema::Network; +use crate::utils::server_handlers::{handle_block, handle_stats, handle_weave_gm}; +use axum::{routing::get, Router}; use std::thread; use std::time::Duration; -use axum::{routing::get, Router}; use tokio::task; mod utils; @@ -19,8 +19,9 @@ async fn main() -> shuttle_axum::ShuttleAxum { println!("\n{:#?}\n\n", network); // server routes let router = Router::new() - .route("/", get(weave_gm)) - .route("/block/:id", get(handle_block)); + .route("/", get(handle_weave_gm)) + .route("/stats", get(handle_stats)) + .route("/block/:id", get(handle_block)); // poll blocks & archive in parallel task::spawn(async move { @@ -40,4 +41,3 @@ async fn main() -> shuttle_axum::ShuttleAxum { Ok(router.into()) } - diff --git a/src/utils/env_var.rs b/src/utils/env_var.rs index 086650f..e7c7de6 100644 --- a/src/utils/env_var.rs +++ b/src/utils/env_var.rs @@ -7,4 +7,4 @@ pub fn get_env_var(key: &str) -> Result { Ok(val) => Ok(val), Err(e) => Err(e), } -} \ No newline at end of file +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 6ccd214..21d9d22 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,7 +1,7 @@ pub mod archive_block; pub mod env_var; pub mod get_block; -pub mod schema; -pub mod transaction; pub mod planetscale; +pub mod schema; pub mod server_handlers; +pub mod transaction; diff --git a/src/utils/planetscale.rs b/src/utils/planetscale.rs index ba563f2..5089683 100644 --- a/src/utils/planetscale.rs +++ b/src/utils/planetscale.rs @@ -1,34 +1,35 @@ +use std::fmt::format; + use crate::utils::env_var::get_env_var; -use crate::utils::schema::{Network, PsGetBlockTxid}; -use planetscale_driver::{PSConnection, query}; +use crate::utils::schema::{Network, PsGetBlockTxid, PsGetExtremeBlock}; use anyhow::Error; +use planetscale_driver::{query, PSConnection}; use serde_json::Value; async fn ps_init() -> PSConnection { - let host = get_env_var("DATABASE_HOST").unwrap(); let username = get_env_var("DATABASE_USERNAME").unwrap(); let password = get_env_var("DATABASE_PASSWORD").unwrap(); - let conn: PSConnection = PSConnection::new( - &host, - &username, - &password, - ); + let conn: PSConnection = PSConnection::new(&host, &username, &password); - conn + conn } -pub async fn ps_archive_block(network_block_id: &u64, wvm_calldata_txid: &str) -> Result<(), Error> { +pub async fn ps_archive_block( + network_block_id: &u64, + wvm_calldata_txid: &str, +) -> Result<(), Error> { // format to the table VAR(66) limitation - let wvm_calldata_txid = wvm_calldata_txid.trim_matches('"'); + let wvm_calldata_txid = wvm_calldata_txid.trim_matches('"'); let conn = ps_init().await; - let res = query("INSERT INTO WeaveVMArchiver(NetworkBlockId, WeaveVMArchiveTxid) VALUES($0, \"$1\")") - .bind(network_block_id) - .bind(wvm_calldata_txid) - .execute(&conn) - .await; + let res = + query("INSERT INTO WeaveVMArchiver(NetworkBlockId, WeaveVMArchiveTxid) VALUES($0, \"$1\")") + .bind(network_block_id) + .bind(wvm_calldata_txid) + .execute(&conn) + .await; match res { Ok(result) => { @@ -46,7 +47,11 @@ pub async fn ps_get_latest_block_id() -> u64 { let network = Network::config(); let conn = ps_init().await; - let latest_archived: u64 = query("SELECT MAX(NetworkBlockId) AS LatestNetworkBlockId FROM WeaveVMArchiver;").fetch_scalar(&conn).await.unwrap_or(network.start_block); + let latest_archived: u64 = + query("SELECT MAX(NetworkBlockId) AS LatestNetworkBlockId FROM WeaveVMArchiver;") + .fetch_scalar(&conn) + .await + .unwrap_or(network.start_block); // return latest archived block in planetscale + 1 // so the process can start archiving from latest_archived + 1 latest_archived + 1 @@ -55,13 +60,32 @@ pub async fn ps_get_latest_block_id() -> u64 { pub async fn ps_get_archived_block_txid(id: u64) -> Value { let conn = ps_init().await; - let query_formatted = format!("SELECT WeaveVMArchiveTxid FROM WeaveVMArchiver WHERE NetworkBlockId = {}", id); - let txid: PsGetBlockTxid = - query(&query_formatted) - .fetch_one(&conn) - .await - .unwrap(); + let query_formatted = format!( + "SELECT WeaveVMArchiveTxid FROM WeaveVMArchiver WHERE NetworkBlockId = {}", + id + ); + let txid: PsGetBlockTxid = query(&query_formatted).fetch_one(&conn).await.unwrap(); let res = serde_json::json!(txid); res -} \ No newline at end of file +} + +pub async fn ps_get_blocks_extremes(extreme: &str) -> Value { + let conn = ps_init().await; + + let query_type = match extreme { + "first" => "ASC", + "last" => "DESC", + _ => panic!("invalid extreme value. Use 'first' or 'last'."), + }; + + let query_formatted = format!( + "SELECT NetworkBlockId FROM WeaveVMArchiver ORDER BY NetworkBlockId {} LIMIT 1;", + query_type + ); + + let query: PsGetExtremeBlock = query(&query_formatted).fetch_one(&conn).await.unwrap(); + + let res = serde_json::json!(query); + res +} diff --git a/src/utils/schema.rs b/src/utils/schema.rs index 7ae7d92..5a88506 100644 --- a/src/utils/schema.rs +++ b/src/utils/schema.rs @@ -1,13 +1,15 @@ use crate::utils::env_var::get_env_var; +use crate::utils::transaction::get_archiver_balance; use borsh::to_vec; use borsh_derive::{BorshDeserialize, BorshSerialize}; +use ethers::types::U256; use ethers_providers::{Http, Provider}; +use planetscale_driver::Database; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::convert::TryFrom; use std::fs::File; use std::io::{Read, Write}; -use planetscale_driver::Database; #[derive(Serialize, Deserialize, Debug)] pub struct Network { @@ -57,26 +59,26 @@ impl Network { pub struct Block { pub base_fee_per_gas: Option, // "baseFeePerGas" pub blob_gas_used: Option, // "blobGasUsed" - pub difficulty: Option, // "difficulty" + pub difficulty: Option, // "difficulty" pub excess_blob_gas: Option, // "excessBlobGas" - pub extra_data: Option, // "extraData" - pub gas_limit: Option, // "gasLimit" - pub gas_used: Option, // "gasUsed" - pub hash: Option, // "hash" - pub logs_bloom: Option, // "logsBloom" - pub miner: Option, // "miner" - pub mix_hash: Option, // "mixHash" - pub nonce: Option, // "nonce" - pub number: Option, // "number" + pub extra_data: Option, // "extraData" + pub gas_limit: Option, // "gasLimit" + pub gas_used: Option, // "gasUsed" + pub hash: Option, // "hash" + pub logs_bloom: Option, // "logsBloom" + pub miner: Option, // "miner" + pub mix_hash: Option, // "mixHash" + pub nonce: Option, // "nonce" + pub number: Option, // "number" pub parent_beacon_block_root: Option, // "parentBeaconBlockRoot" - pub parent_hash: Option, // "parentHash" - pub receipts_root: Option, // "receiptsRoot" + pub parent_hash: Option, // "parentHash" + pub receipts_root: Option, // "receiptsRoot" pub seal_fields: Vec, // "sealFields" as an array of strings - pub sha3_uncles: Option, // "sha3Uncles" - pub size: Option, // "size" - pub state_root: Option, // "stateRoot" - pub timestamp: Option, // "timestamp" - pub total_difficulty: Option, // "totalDifficulty" + pub sha3_uncles: Option, // "sha3Uncles" + pub size: Option, // "size" + pub state_root: Option, // "stateRoot" + pub timestamp: Option, // "timestamp" + pub total_difficulty: Option, // "totalDifficulty" pub transactions: Vec, // "transactions" as an array of strings } @@ -94,8 +96,36 @@ impl Block { } } - #[derive(Database, Debug, Serialize)] pub struct PsGetBlockTxid { - pub wvm_archive_txid : String -} \ No newline at end of file + pub wvm_archive_txid: String, +} + +#[derive(Database, Debug, Serialize)] +pub struct PsGetExtremeBlock { + pub block_id: u64, +} + +#[derive(Debug, Serialize)] +pub struct StatsServerResponse { + first_block: Option, + last_block: Option, + total_archived_blocks: u64, + archiver_balance: U256, +} + +impl StatsServerResponse { + pub async fn new(first_block: Option, last_block: Option) -> StatsServerResponse { + let total_archived_blocks = last_block.unwrap_or(0) - first_block.unwrap_or(0); + let archiver_balance = get_archiver_balance().await; + let archiver_balance = Some(archiver_balance).unwrap(); + + let instance: StatsServerResponse = StatsServerResponse { + first_block, + last_block, + total_archived_blocks, + archiver_balance, + }; + instance + } +} diff --git a/src/utils/server_handlers.rs b/src/utils/server_handlers.rs index 924e1e4..1c9f65a 100644 --- a/src/utils/server_handlers.rs +++ b/src/utils/server_handlers.rs @@ -1,11 +1,9 @@ -use crate::utils::planetscale::ps_get_archived_block_txid; +use crate::utils::planetscale::{ps_get_archived_block_txid, ps_get_blocks_extremes}; +use crate::utils::schema::StatsServerResponse; +use axum::{extract::Path, response::Json}; use serde_json::Value; -use axum::{ - extract::Path, - response::Json -}; -pub async fn weave_gm() -> &'static str { +pub async fn handle_weave_gm() -> &'static str { "WeaveGM!" } @@ -14,3 +12,14 @@ pub async fn handle_block(Path(id): Path) -> Json { Json(txid) } +pub async fn handle_stats() -> Json { + let first = ps_get_blocks_extremes("first").await; + let last = ps_get_blocks_extremes("last").await; + + let first_block = first.get("block_id").unwrap().as_u64(); + let last_block = last.get("block_id").unwrap().as_u64(); + let stats_res = StatsServerResponse::new(first_block, last_block).await; + + let res = serde_json::to_value(&stats_res).unwrap(); + Json(res) +} diff --git a/src/utils/transaction.rs b/src/utils/transaction.rs index 1412018..ea72dd7 100644 --- a/src/utils/transaction.rs +++ b/src/utils/transaction.rs @@ -29,6 +29,14 @@ async fn assert_non_zero_balance(provider: &Provider, address: &Address) { assert!(balance > 0.into()); } +pub async fn get_archiver_balance() -> U256 { + let network = Network::config(); + let provider = Network::provider(&network, true).await; + let address = network.archiver_address.parse::
().unwrap(); + let balance = provider.get_balance(address, None).await.unwrap(); + balance +} + async fn send_transaction( client: &Client, address_from: &Address, @@ -51,4 +59,4 @@ async fn send_transaction( println!("\nWeaveVM Archiving TXID: {}", txid); Ok(txid) -} \ No newline at end of file +} From cedbd02bf7c7ffaedf452f78aacdb8f0f056d1a4 Mon Sep 17 00:00:00 2001 From: charmful0x Date: Tue, 13 Aug 2024 16:31:03 +0200 Subject: [PATCH 2/6] feat: add more fields & rename /stats to /info --- src/main.rs | 4 ++-- src/utils/schema.rs | 19 ++++++++++++++----- src/utils/server_handlers.rs | 6 +++--- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2d1b556..8b3ff93 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use crate::utils::archive_block::archive; use crate::utils::planetscale::{ps_archive_block, ps_get_latest_block_id}; use crate::utils::schema::Network; -use crate::utils::server_handlers::{handle_block, handle_stats, handle_weave_gm}; +use crate::utils::server_handlers::{handle_block, handle_info, handle_weave_gm}; use axum::{routing::get, Router}; use std::thread; use std::time::Duration; @@ -20,7 +20,7 @@ async fn main() -> shuttle_axum::ShuttleAxum { // server routes let router = Router::new() .route("/", get(handle_weave_gm)) - .route("/stats", get(handle_stats)) + .route("/stats", get(handle_info)) .route("/block/:id", get(handle_block)); // poll blocks & archive in parallel diff --git a/src/utils/schema.rs b/src/utils/schema.rs index 5a88506..5babbd0 100644 --- a/src/utils/schema.rs +++ b/src/utils/schema.rs @@ -107,24 +107,33 @@ pub struct PsGetExtremeBlock { } #[derive(Debug, Serialize)] -pub struct StatsServerResponse { +pub struct InfoServerResponse { first_block: Option, last_block: Option, total_archived_blocks: u64, archiver_balance: U256, + archiver_address: String, + network_name: String, + network_chain_id: u32, + network_rpc: String } -impl StatsServerResponse { - pub async fn new(first_block: Option, last_block: Option) -> StatsServerResponse { +impl InfoServerResponse { + pub async fn new(first_block: Option, last_block: Option) -> InfoServerResponse { + let network = Network::config(); let total_archived_blocks = last_block.unwrap_or(0) - first_block.unwrap_or(0); let archiver_balance = get_archiver_balance().await; let archiver_balance = Some(archiver_balance).unwrap(); - let instance: StatsServerResponse = StatsServerResponse { + let instance: InfoServerResponse = InfoServerResponse { + archiver_balance, first_block, last_block, total_archived_blocks, - archiver_balance, + archiver_address: network.archiver_address, + network_name: network.name, + network_chain_id: network.network_chain_id, + network_rpc: network.network_rpc }; instance } diff --git a/src/utils/server_handlers.rs b/src/utils/server_handlers.rs index 1c9f65a..22c4fbd 100644 --- a/src/utils/server_handlers.rs +++ b/src/utils/server_handlers.rs @@ -1,5 +1,5 @@ use crate::utils::planetscale::{ps_get_archived_block_txid, ps_get_blocks_extremes}; -use crate::utils::schema::StatsServerResponse; +use crate::utils::schema::InfoServerResponse; use axum::{extract::Path, response::Json}; use serde_json::Value; @@ -12,13 +12,13 @@ pub async fn handle_block(Path(id): Path) -> Json { Json(txid) } -pub async fn handle_stats() -> Json { +pub async fn handle_info() -> Json { let first = ps_get_blocks_extremes("first").await; let last = ps_get_blocks_extremes("last").await; let first_block = first.get("block_id").unwrap().as_u64(); let last_block = last.get("block_id").unwrap().as_u64(); - let stats_res = StatsServerResponse::new(first_block, last_block).await; + let stats_res = InfoServerResponse::new(first_block, last_block).await; let res = serde_json::to_value(&stats_res).unwrap(); Json(res) From 9426fdff7069face21b948f16017cd84097b8fbe Mon Sep 17 00:00:00 2001 From: charmful0x Date: Tue, 13 Aug 2024 16:31:47 +0200 Subject: [PATCH 3/6] chore: fmt --- src/utils/schema.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/schema.rs b/src/utils/schema.rs index 5babbd0..863c986 100644 --- a/src/utils/schema.rs +++ b/src/utils/schema.rs @@ -115,7 +115,7 @@ pub struct InfoServerResponse { archiver_address: String, network_name: String, network_chain_id: u32, - network_rpc: String + network_rpc: String, } impl InfoServerResponse { @@ -133,7 +133,7 @@ impl InfoServerResponse { archiver_address: network.archiver_address, network_name: network.name, network_chain_id: network.network_chain_id, - network_rpc: network.network_rpc + network_rpc: network.network_rpc, }; instance } From 8ac5cc0ab573ee92d7016cab8a82cd5813fa5c10 Mon Sep 17 00:00:00 2001 From: charmful0x Date: Tue, 13 Aug 2024 16:34:31 +0200 Subject: [PATCH 4/6] docs: add /info endpoint --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 7aa1416..5e391d3 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,26 @@ The WeaveVM Archiver node operates as follows: As mentioned, PlanetScale is used for cloud indexing, which allows a WeaveVM Archiver node to expose its WeaveVM data as a RESTful API. +### WeaveVM Archiver node instance info + +```bash +curl -X GET https://your_app.shuttleapp.rs/info +``` +**returns:** + +```rs +pub struct InfoServerResponse { + first_block: Option, + last_block: Option, + total_archived_blocks: u64, + archiver_balance: U256, + archiver_address: String, + network_name: String, + network_chain_id: u32, + network_rpc: String, +} +``` + ### Retrieve the WVM archive TXID for a given EVM block ID ```bash From 2e1c4917113b5949876857406471cdaf063b4da5 Mon Sep 17 00:00:00 2001 From: charmful0x Date: Tue, 13 Aug 2024 16:35:02 +0200 Subject: [PATCH 5/6] fix: rename route na,e --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 8b3ff93..859ae8b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,7 +20,7 @@ async fn main() -> shuttle_axum::ShuttleAxum { // server routes let router = Router::new() .route("/", get(handle_weave_gm)) - .route("/stats", get(handle_info)) + .route("/info", get(handle_info)) .route("/block/:id", get(handle_block)); // poll blocks & archive in parallel From 91deccb627b0585fc033d2b74601b2f6f285598b Mon Sep 17 00:00:00 2001 From: charmful0x Date: Tue, 13 Aug 2024 16:36:19 +0200 Subject: [PATCH 6/6] feat: v0.1.2 --- Cargo.toml | 2 +- src/utils/planetscale.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c8ba99f..a90218b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wvm-archiver" -version = "0.1.1" +version = "0.1.2" edition = "2021" description = "EL data pipeline for WVM testnet v0" authors = ["charmful0x "] diff --git a/src/utils/planetscale.rs b/src/utils/planetscale.rs index 5089683..9db3b6f 100644 --- a/src/utils/planetscale.rs +++ b/src/utils/planetscale.rs @@ -1,5 +1,3 @@ -use std::fmt::format; - use crate::utils::env_var::get_env_var; use crate::utils::schema::{Network, PsGetBlockTxid, PsGetExtremeBlock}; use anyhow::Error;