From 96adbe9e30c5e59962b418890e512fef3ab5f363 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Thu, 17 Nov 2022 12:48:23 +0100 Subject: [PATCH 01/34] metrics server and collection --- Cargo.lock | 84 ++++++++++++++++++++++++++++ Cargo.toml | 3 + src/metrics_server.rs | 124 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+) create mode 100644 src/metrics_server.rs diff --git a/Cargo.lock b/Cargo.lock index c590ff0..858c9b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -951,6 +951,18 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "5.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f" +dependencies = [ + "cfg-if 1.0.0", + "hashbrown 0.12.3", + "lock_api", + "parking_lot_core 0.9.3", +] + [[package]] name = "derive_builder" version = "0.11.2" @@ -1529,6 +1541,7 @@ dependencies = [ "dotenv", "futures", "hex", + "hyper", "itertools 0.9.0", "lazy_static", "near-jsonrpc-client", @@ -1536,7 +1549,9 @@ dependencies = [ "near-lake-framework", "near-primitives", "num-traits", + "opentelemetry", "primitive-types", + "prometheus", "quote", "regex", "serde", @@ -2146,6 +2161,54 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry_api" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +dependencies = [ + "fnv", + "futures-channel", + "futures-util", + "indexmap", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +dependencies = [ + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "percent-encoding", + "rand 0.8.5", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "os_str_bytes" version = "6.2.0" @@ -2364,6 +2427,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "memchr", + "parking_lot 0.12.1", + "protobuf", + "thiserror", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "quote" version = "1.0.20" diff --git a/Cargo.toml b/Cargo.toml index f2edc63..0f57722 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,3 +39,6 @@ near-jsonrpc-primitives = "0.14.0" near-jsonrpc-client = "0.4.0-beta.0" near-lake-framework = "0.5.0" near-primitives = "0.14.0" +hyper = { version = "0.14", features = ["full"] } +prometheus = "0.13.1" +opentelemetry = { version = "0.18.0", features = ["rt-tokio", "metrics"] } diff --git a/src/metrics_server.rs b/src/metrics_server.rs new file mode 100644 index 0000000..f65d01c --- /dev/null +++ b/src/metrics_server.rs @@ -0,0 +1,124 @@ +use hyper::{ + header::CONTENT_TYPE, + service::{make_service_fn, service_fn}, + Body, Method, Request, Response, Server, +}; +use opentelemetry::{ + global, + metrics::{Counter, Histogram}, + sdk::{ + export::metrics::aggregation, + metrics::{controllers, processors, selectors}, + }, + Context, KeyValue, +}; +use opentelemetry_prometheus::PrometheusExporter; +use prometheus::{Encoder, TextEncoder}; +use std::convert::Infallible; +use std::sync::Arc; +use std::time::SystemTime; + +// lazy_static! { +// static ref HANDLER_ALL: [KeyValue; 1] = [KeyValue::new("handler", "all")]; +// } + +async fn serve_req( + cx: Context, + req: Request, + state: Arc, +) -> Result, hyper::Error> { + println!("Receiving request at path {}", req.uri()); + let request_start = SystemTime::now(); + + state.curr_block_no_counter.add(&cx, 1, &[]); + + let response = match (req.method(), req.uri().path()) { + (&Method::GET, "/metrics") => { + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + let metric_families = state.exporter.registry().gather(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + // state.http_body_gauge.record(&cx, buffer.len() as u64, &[]); + + Response::builder() + .status(200) + .header(CONTENT_TYPE, encoder.format_type()) + .body(Body::from(buffer)) + .unwrap() + } + (&Method::GET, "/") => Response::builder() + .status(200) + .body(Body::from("Hello World")) + .unwrap(), + _ => Response::builder() + .status(404) + .body(Body::from("Missing Page")) + .unwrap(), + }; + + state.block_processing_duration_histogram.record( + &cx, + request_start.elapsed().map_or(0.0, |d| d.as_secs_f64()), + &[], + ); + + Ok(response) +} + +struct AppState { + exporter: PrometheusExporter, + curr_block_no_counter: Counter, + block_processing_duration_histogram: Histogram, +} + +#[tokio::main] +pub async fn init_metrics_server() -> Result<(), Box> { + let controller = controllers::basic( + processors::factory( + selectors::simple::histogram([1.0, 2.0, 5.0, 10.0, 20.0, 50.0]), + aggregation::cumulative_temporality_selector(), + ) + .with_memory(true), + ) + .build(); + + let exporter = opentelemetry_prometheus::exporter(controller).init(); + let cx = Context::new(); + + let meter = global::meter("ex.com/hyper"); + let state = Arc::new(AppState { + exporter, + curr_block_no_counter: meter + .u64_counter("indexer.blocks_total") + .with_description("Total number of blocks processed.") + .init(), + block_processing_duration_histogram: meter + .f64_histogram("example.block_processing_duration_seconds") + .with_description("The time it takes each block to process.") + .init(), + }); + + // For every connection, we must make a `Service` to handle all + // incoming HTTP requests on said connection. + let make_svc = make_service_fn(move |_conn| { + let state = state.clone(); + let cx = cx.clone(); + + async move { + Ok::<_, Infallible>(service_fn(move |req| { + serve_req(cx.clone(), req, state.clone()) + })) + } + }); + + let addr = ([127, 0, 0, 1], 3000).into(); + + let server = Server::bind(&addr).serve(make_svc); + + println!("Starting metrics server at http://{}", addr); + + server.await?; + + Ok(()) +} + From 9d58f2274d885491e7ba6d91d75eb20bdbab1abd Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Thu, 17 Nov 2022 12:52:10 +0100 Subject: [PATCH 02/34] /metrics enabled --- Cargo.lock | 12 ++++++++++++ Cargo.toml | 1 + src/main.rs | 12 ++++++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 858c9b5..e33e498 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1550,6 +1550,7 @@ dependencies = [ "near-primitives", "num-traits", "opentelemetry", + "opentelemetry-prometheus", "primitive-types", "prometheus", "quote", @@ -2171,6 +2172,17 @@ dependencies = [ "opentelemetry_sdk", ] +[[package]] +name = "opentelemetry-prometheus" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c3d833835a53cf91331d2cfb27e9121f5a95261f31f08a1f79ab31688b8da8" +dependencies = [ + "opentelemetry", + "prometheus", + "protobuf", +] + [[package]] name = "opentelemetry_api" version = "0.18.0" diff --git a/Cargo.toml b/Cargo.toml index 0f57722..dc19025 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,3 +42,4 @@ near-primitives = "0.14.0" hyper = { version = "0.14", features = ["full"] } prometheus = "0.13.1" opentelemetry = { version = "0.18.0", features = ["rt-tokio", "metrics"] } +opentelemetry-prometheus = "0.11.0" \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index e7e8aad..c1c8308 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,9 +4,10 @@ use clap::Parser; use dotenv::dotenv; use futures::StreamExt; use std::env; -use tokio::sync::Mutex; +use tokio::{sync::Mutex, task}; use tracing_subscriber::EnvFilter; use tracing_utils::DefaultSubcriberGuard; +use metrics_server::init_metrics_server; use crate::configs::Opts; use near_lake_framework::near_indexer_primitives; @@ -15,6 +16,7 @@ mod db_adapters; mod models; mod rpc_helpers; mod tracing_utils; +mod metrics_server; pub(crate) const LOGGING_PREFIX: &str = "indexer_events"; @@ -45,7 +47,13 @@ async fn main() -> anyhow::Result<()> { .build()?; let _writer_guard = init_tracing(); - + + task::spawn_blocking(move || { + if let Err(val) = init_metrics_server() { + tracing::error!(target: LOGGING_PREFIX, "Error setting up the metrics server. -> {}", val) + } + }); + let (lake_handle, stream) = near_lake_framework::streamer(config); let json_rpc_client = near_jsonrpc_client::JsonRpcClient::connect(&opts.near_archival_rpc_url); From e7d9c78200619808975272b1c202b5cc2500e928 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Thu, 17 Nov 2022 18:25:33 +0100 Subject: [PATCH 03/34] port changed from .env --- src/metrics_server.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/metrics_server.rs b/src/metrics_server.rs index f65d01c..f52f7a4 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -14,7 +14,7 @@ use opentelemetry::{ }; use opentelemetry_prometheus::PrometheusExporter; use prometheus::{Encoder, TextEncoder}; -use std::convert::Infallible; +use std::{convert::Infallible, env}; use std::sync::Arc; use std::time::SystemTime; @@ -111,8 +111,13 @@ pub async fn init_metrics_server() -> Result<(), Box val.parse::().unwrap(), + _ => 3000, + }; + let addr = ([127, 0, 0, 1], port).into(); + let server = Server::bind(&addr).serve(make_svc); println!("Starting metrics server at http://{}", addr); From 83f88ca9718899bed01a337120b4cbf2c1fa06c9 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Thu, 17 Nov 2022 22:19:18 +0100 Subject: [PATCH 04/34] block processing metrics --- src/main.rs | 68 +++++++-------------- src/metrics_server.rs | 133 ++++++++++++++++-------------------------- src/tracing_utils.rs | 124 ++++++++++++++++++++++++++++++++++----- 3 files changed, 180 insertions(+), 145 deletions(-) diff --git a/src/main.rs b/src/main.rs index c1c8308..b9c6e4f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,22 +1,22 @@ // TODO cleanup imports in all the files in the end +use crate::configs::Opts; use cached::SizedCache; use clap::Parser; use dotenv::dotenv; use futures::StreamExt; -use std::env; +use metrics_server::{init_metrics_server, BLOCK_PROCESSED_TOTAL, BLOCK_PROCESSING_TIME}; +use near_lake_framework::near_indexer_primitives; +use std::{env, time::SystemTime}; use tokio::{sync::Mutex, task}; use tracing_subscriber::EnvFilter; -use tracing_utils::DefaultSubcriberGuard; -use metrics_server::init_metrics_server; - -use crate::configs::Opts; -use near_lake_framework::near_indexer_primitives; mod configs; mod db_adapters; +mod metrics_server; mod models; mod rpc_helpers; mod tracing_utils; -mod metrics_server; +#[macro_use] +extern crate lazy_static; pub(crate) const LOGGING_PREFIX: &str = "indexer_events"; @@ -46,14 +46,18 @@ async fn main() -> anyhow::Result<()> { .blocks_preload_pool_size(100) .build()?; - let _writer_guard = init_tracing(); - + let env_filter = EnvFilter::new("near_lake_framework=info,indexer_events=info"); + let _subscriber = tracing_utils::init_tracing(env_filter).await; + task::spawn_blocking(move || { - if let Err(val) = init_metrics_server() { - tracing::error!(target: LOGGING_PREFIX, "Error setting up the metrics server. -> {}", val) + if let Err(_val) = init_metrics_server() { + tracing::error!( + target: LOGGING_PREFIX, + "Error setting up the metrics server." + ); } }); - + let (lake_handle, stream) = near_lake_framework::streamer(config); let json_rpc_client = near_jsonrpc_client::JsonRpcClient::connect(&opts.near_archival_rpc_url); @@ -110,6 +114,8 @@ async fn handle_streamer_message( ft_balance_cache: &FtBalanceCache, contracts: &db_adapters::contracts::ContractsHelper, ) -> anyhow::Result { + let request_start = SystemTime::now(); + if streamer_message.block.header.height % 100 == 0 { tracing::info!( target: crate::LOGGING_PREFIX, @@ -127,40 +133,8 @@ async fn handle_streamer_message( contracts, ) .await?; - Ok(streamer_message.block.header.height) -} - -fn init_tracing() -> DefaultSubcriberGuard { - let mut env_filter = EnvFilter::new("near_lake_framework=info,indexer_events=info"); - - if let Ok(rust_log) = env::var("RUST_LOG") { - if !rust_log.is_empty() { - for directive in rust_log.split(',').filter_map(|s| match s.parse() { - Ok(directive) => Some(directive), - Err(err) => { - tracing::warn!( - target: crate::LOGGING_PREFIX, - "Ignoring directive `{}`: {}", - s, - err - ); - None - } - }) { - env_filter = env_filter.add_directive(directive); - } - } - } - - let (non_blocking_writer, _guard) = tracing_appender::non_blocking(std::io::stderr()); - - let subscriber = tracing_subscriber::FmtSubscriber::builder() - .with_env_filter(env_filter) - .with_writer(non_blocking_writer) - .finish(); - DefaultSubcriberGuard { - subscriber_guard: tracing::subscriber::set_default(subscriber), - writer_guard: _guard, - } + BLOCK_PROCESSED_TOTAL.inc(); + BLOCK_PROCESSING_TIME.observe(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); + Ok(streamer_message.block.header.height) } diff --git a/src/metrics_server.rs b/src/metrics_server.rs index f52f7a4..2d27487 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -3,52 +3,63 @@ use hyper::{ service::{make_service_fn, service_fn}, Body, Method, Request, Response, Server, }; -use opentelemetry::{ - global, - metrics::{Counter, Histogram}, - sdk::{ - export::metrics::aggregation, - metrics::{controllers, processors, selectors}, - }, - Context, KeyValue, -}; -use opentelemetry_prometheus::PrometheusExporter; -use prometheus::{Encoder, TextEncoder}; +use prometheus::{Encoder, Histogram, HistogramOpts, IntCounter, Opts}; use std::{convert::Infallible, env}; -use std::sync::Arc; -use std::time::SystemTime; -// lazy_static! { -// static ref HANDLER_ALL: [KeyValue; 1] = [KeyValue::new("handler", "all")]; -// } +pub type Result = std::result::Result; + +pub fn try_create_histogram(name: &str, help: &str) -> Result { + let opts = HistogramOpts::new(name, help); + let histogram = Histogram::with_opts(opts)?; + prometheus::register(Box::new(histogram.clone()))?; + Ok(histogram) +} -async fn serve_req( - cx: Context, - req: Request, - state: Arc, -) -> Result, hyper::Error> { - println!("Receiving request at path {}", req.uri()); - let request_start = SystemTime::now(); +/// Attempts to crate an `IntCounter`, returning `Err` if the registry does not accept the counter +/// (potentially due to naming conflict). +pub fn try_create_int_counter(name: &str, help: &str) -> Result { + let opts = Opts::new(name, help); + let counter = IntCounter::with_opts(opts)?; + prometheus::register(Box::new(counter.clone()))?; + Ok(counter) +} - state.curr_block_no_counter.add(&cx, 1, &[]); +lazy_static! { + pub static ref BLOCK_PROCESSED_TOTAL: IntCounter = + try_create_int_counter("total_blocks_processed", "Total number of blocks processed") + .unwrap(); + pub static ref BLOCK_PROCESSING_TIME: Histogram = + try_create_histogram("block_processing_time", "Time spent indexing a block").unwrap(); +} +async fn serve_req(req: Request) -> Result, hyper::Error> { let response = match (req.method(), req.uri().path()) { (&Method::GET, "/metrics") => { - let mut buffer = vec![]; - let encoder = TextEncoder::new(); - let metric_families = state.exporter.registry().gather(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - // state.http_body_gauge.record(&cx, buffer.len() as u64, &[]); + let encoder = prometheus::TextEncoder::new(); + + let mut buffer = Vec::new(); + if let Err(e) = encoder.encode(&prometheus::gather(), &mut buffer) { + eprintln!("could not encode custom metrics: {}", e); + }; + + let res = match String::from_utf8(buffer.clone()) { + Ok(v) => v, + Err(e) => { + eprintln!("custom metrics could not be from_utf8'd: {}", e); + String::default() + } + }; + buffer.clear(); Response::builder() .status(200) .header(CONTENT_TYPE, encoder.format_type()) - .body(Body::from(buffer)) + .body(Body::from(res)) .unwrap() } (&Method::GET, "/") => Response::builder() .status(200) - .body(Body::from("Hello World")) + .body(Body::from("")) .unwrap(), _ => Response::builder() .status(404) @@ -56,74 +67,30 @@ async fn serve_req( .unwrap(), }; - state.block_processing_duration_histogram.record( - &cx, - request_start.elapsed().map_or(0.0, |d| d.as_secs_f64()), - &[], - ); - Ok(response) } -struct AppState { - exporter: PrometheusExporter, - curr_block_no_counter: Counter, - block_processing_duration_histogram: Histogram, -} - #[tokio::main] -pub async fn init_metrics_server() -> Result<(), Box> { - let controller = controllers::basic( - processors::factory( - selectors::simple::histogram([1.0, 2.0, 5.0, 10.0, 20.0, 50.0]), - aggregation::cumulative_temporality_selector(), - ) - .with_memory(true), - ) - .build(); - - let exporter = opentelemetry_prometheus::exporter(controller).init(); - let cx = Context::new(); - - let meter = global::meter("ex.com/hyper"); - let state = Arc::new(AppState { - exporter, - curr_block_no_counter: meter - .u64_counter("indexer.blocks_total") - .with_description("Total number of blocks processed.") - .init(), - block_processing_duration_histogram: meter - .f64_histogram("example.block_processing_duration_seconds") - .with_description("The time it takes each block to process.") - .init(), - }); - +pub async fn init_metrics_server() -> std::result::Result<(), ()> { // For every connection, we must make a `Service` to handle all // incoming HTTP requests on said connection. - let make_svc = make_service_fn(move |_conn| { - let state = state.clone(); - let cx = cx.clone(); - - async move { - Ok::<_, Infallible>(service_fn(move |req| { - serve_req(cx.clone(), req, state.clone()) - })) - } + let make_svc = make_service_fn(move |_conn| async move { + Ok::<_, Infallible>(service_fn(move |req| serve_req(req))) }); let port: u16 = match env::var("PORT") { - Ok(val) => val.parse::().unwrap(), + Ok(val) => val.parse::().unwrap(), _ => 3000, }; let addr = ([127, 0, 0, 1], port).into(); - + let server = Server::bind(&addr).serve(make_svc); println!("Starting metrics server at http://{}", addr); - server.await?; - + if let Err(e) = server.await { + eprintln!("server error: {}", e); + } Ok(()) } - diff --git a/src/tracing_utils.rs b/src/tracing_utils.rs index 423b667..13122ba 100644 --- a/src/tracing_utils.rs +++ b/src/tracing_utils.rs @@ -1,16 +1,110 @@ -/// The resource representing a registered subscriber for non_blocking tracing logger -/// -/// since the non_blocking_writer writes to a new thread "at a later point in time", -/// to ensure that all logs get flushed out in case of panics, we return the writer_guard -/// which when dropped, immediately flushes whatever non_blocking_writer has in its -/// cache to its intended place, that could be a file or a std_out. -/// -/// https://docs.rs/tracing-appender/0.1.1/tracing_appender/non_blocking/struct.WorkerGuard.html -/// -/// Once dropped, the subscriber is unregistered, and the output is flushed. Any messages output -/// after this value is dropped will be delivered to a previously active subscriber, if any. -#[allow(dead_code)] -pub struct DefaultSubcriberGuard { - pub subscriber_guard: tracing::subscriber::DefaultGuard, - pub writer_guard: tracing_appender::non_blocking::WorkerGuard, +pub use {tracing, tracing_appender, tracing_subscriber}; + +use std::env; +use tracing_appender::non_blocking::NonBlocking; +use tracing_subscriber::filter::Filtered; +use tracing_subscriber::layer::{Layered, SubscriberExt}; +use tracing_subscriber::registry::LookupSpan; +use tracing_subscriber::{fmt, EnvFilter, Layer}; + +type LogLayer = Layered< + Filtered< + fmt::Layer, + EnvFilter, + Inner, + >, + Inner, +>; + +pub struct DefaultSubscriberGuard { + // We must first drop the `local_subscriber_guard` so that no new messages are delivered to + // this subscriber while we take care of flushing the messages already in queue. If dropped the + // other way around, the events/spans generated while the subscriber drop guard runs would be + // lost. + subscriber: Option, + #[allow(dead_code)] + writer_guard: Option, +} + +impl DefaultSubscriberGuard { + /// Register this default subscriber globally , for all threads. + /// + /// Must not be called more than once. Mutually exclusive with `Self::local`. + pub fn global(mut self) -> Self { + if let Some(subscriber) = self.subscriber.take() { + tracing::subscriber::set_global_default(subscriber) + .expect("could not set a global subscriber"); + } else { + panic!("trying to set a default subscriber that has been already taken") + } + self + } +} + +fn is_terminal() -> bool { + // Crate `atty` provides a platform-independent way of checking whether the output is a tty. + atty::is(atty::Stream::Stderr) +} + +fn add_non_blocking_log_layer( + filter: EnvFilter, + writer: NonBlocking, + ansi: bool, + subscriber: S, +) -> LogLayer +where + S: tracing::Subscriber + for<'span> LookupSpan<'span> + Send + Sync, +{ + let layer = fmt::layer() + .with_ansi(ansi) + .with_writer(writer) + .with_filter(filter); + + subscriber.with(layer) +} + +pub fn default_subscriber_with_non_blocking_layer( + env_filter: EnvFilter, +) -> DefaultSubscriberGuard { + let color_output = std::env::var_os("NO_COLOR").is_none() && is_terminal(); + + let stderr = std::io::stderr(); + let lined_stderr = std::io::LineWriter::new(stderr); + let (writer, writer_guard) = tracing_appender::non_blocking(lined_stderr); + + let subscriber = tracing_subscriber::registry(); + + let subscriber = add_non_blocking_log_layer(env_filter, writer, color_output, subscriber); + + DefaultSubscriberGuard { + subscriber: Some(subscriber), + writer_guard: Some(writer_guard), + } +} + +pub async fn init_tracing( + mut env_filter: EnvFilter, +) -> DefaultSubscriberGuard { + if let Ok(rust_log) = env::var("RUST_LOG") { + if !rust_log.is_empty() { + for directive in rust_log.split(',').filter_map(|s| match s.parse() { + Ok(directive) => Some(directive), + Err(err) => { + tracing::warn!( + target: crate::LOGGING_PREFIX, + "Ignoring directive `{}`: {}", + s, + err + ); + None + } + }) { + env_filter = env_filter.add_directive(directive); + } + } + } + + let subscriber = default_subscriber_with_non_blocking_layer(env_filter); + + subscriber.global() } From cdc57a93c74d6cb7cbcfd611c317874cbb249fd6 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Fri, 18 Nov 2022 12:51:07 +0100 Subject: [PATCH 05/34] changed port for cloud run compatability --- src/metrics_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics_server.rs b/src/metrics_server.rs index 2d27487..8a8c560 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -83,7 +83,7 @@ pub async fn init_metrics_server() -> std::result::Result<(), ()> { _ => 3000, }; - let addr = ([127, 0, 0, 1], port).into(); + let addr = ([0, 0, 0, 0], port).into(); let server = Server::bind(&addr).serve(make_svc); From d24b7ed0a01a353a01d10c72901ada4edcad482f Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Fri, 18 Nov 2022 13:06:02 +0100 Subject: [PATCH 06/34] fix dependencies --- Cargo.toml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dc19025..2d15053 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,16 +30,14 @@ syn = "1.0.90" tempfile = "3.3.0" tokio = { version = "1", features = ["full"] } tokio-stream = { version = "0.1" } -tracing = "0.1.35" -tracing-subscriber = { version = "0.3.11", features = ["fmt", "std", "env-filter"] } +tracing = { version = "0.1.36", features = ["std"] } tracing-appender = "0.2.2" +prometheus = "0.13.1" +atty = "0.2" quote = "1.0.17" - +tracing-subscriber = { version = "0.3", features = ["env-filter"] } near-jsonrpc-primitives = "0.14.0" near-jsonrpc-client = "0.4.0-beta.0" near-lake-framework = "0.5.0" near-primitives = "0.14.0" -hyper = { version = "0.14", features = ["full"] } -prometheus = "0.13.1" -opentelemetry = { version = "0.18.0", features = ["rt-tokio", "metrics"] } -opentelemetry-prometheus = "0.11.0" \ No newline at end of file +hyper = { version = "0.14", features = ["full"] } \ No newline at end of file From f73f96ae0da006b897684022fff46850b66b9493 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Fri, 18 Nov 2022 16:28:22 +0100 Subject: [PATCH 07/34] start options sourced from .env bucket --- src/configs.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/configs.rs b/src/configs.rs index 6c2879c..41e7569 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -16,14 +16,14 @@ pub(crate) struct Opts { #[clap(long)] pub debug: bool, /// AWS S3 bucket name to get the stream from - #[clap(long)] + #[clap(long, env="s3_bucket_name")] pub s3_bucket_name: String, /// AWS S3 bucket region - #[clap(long)] + #[clap(long, env="s3_region_name")] pub s3_region_name: String, /// Block height to start the stream from - #[clap(long, short)] + #[clap(long, short, env="start_block_height")] pub start_block_height: u64, - #[clap(long, short)] + #[clap(long, short, env="near_archival_rpc_url")] pub near_archival_rpc_url: String, } From 364549c78c6770556c07d0a6f043cde49cc45827 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Fri, 18 Nov 2022 21:26:48 +0100 Subject: [PATCH 08/34] made credentials as arguments to program --- Cargo.lock | 84 +++++--------------------------------------------- Cargo.toml | 4 ++- src/configs.rs | 28 +++++++++++++++++ src/main.rs | 17 +++++----- 4 files changed, 47 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e33e498..02e6588 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -951,18 +951,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dashmap" -version = "5.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f" -dependencies = [ - "cfg-if 1.0.0", - "hashbrown 0.12.3", - "lock_api", - "parking_lot_core 0.9.3", -] - [[package]] name = "derive_builder" version = "0.11.2" @@ -1532,7 +1520,10 @@ name = "indexer-events" version = "0.1.0" dependencies = [ "anyhow", + "atty", "avro-rs", + "aws-sdk-s3", + "aws-types", "base64 0.11.0", "bigdecimal", "borsh", @@ -1549,8 +1540,6 @@ dependencies = [ "near-lake-framework", "near-primitives", "num-traits", - "opentelemetry", - "opentelemetry-prometheus", "primitive-types", "prometheus", "quote", @@ -2162,65 +2151,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "opentelemetry" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" -dependencies = [ - "opentelemetry_api", - "opentelemetry_sdk", -] - -[[package]] -name = "opentelemetry-prometheus" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c3d833835a53cf91331d2cfb27e9121f5a95261f31f08a1f79ab31688b8da8" -dependencies = [ - "opentelemetry", - "prometheus", - "protobuf", -] - -[[package]] -name = "opentelemetry_api" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" -dependencies = [ - "fnv", - "futures-channel", - "futures-util", - "indexmap", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" -dependencies = [ - "async-trait", - "crossbeam-channel", - "dashmap", - "fnv", - "futures-channel", - "futures-executor", - "futures-util", - "once_cell", - "opentelemetry_api", - "percent-encoding", - "rand 0.8.5", - "thiserror", - "tokio", - "tokio-stream", -] - [[package]] name = "os_str_bytes" version = "6.2.0" @@ -3342,9 +3272,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.35" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "log", @@ -3366,9 +3296,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 2d15053..9324500 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,4 +40,6 @@ near-jsonrpc-primitives = "0.14.0" near-jsonrpc-client = "0.4.0-beta.0" near-lake-framework = "0.5.0" near-primitives = "0.14.0" -hyper = { version = "0.14", features = ["full"] } \ No newline at end of file +hyper = { version = "0.14", features = ["full"] } +aws-types= "0.13.0" +aws-sdk-s3="0.13.0" \ No newline at end of file diff --git a/src/configs.rs b/src/configs.rs index 41e7569..b83ac8c 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -18,6 +18,12 @@ pub(crate) struct Opts { /// AWS S3 bucket name to get the stream from #[clap(long, env="s3_bucket_name")] pub s3_bucket_name: String, + /// AWS Access Key with the rights to read from AWS S3 + #[clap(long, env="NEAR_LAKE_AWS_KEY")] + pub lake_aws_access_key: String, + #[clap(long, env="NEAR_LAKE_AWS_SECRET")] + /// AWS Secret Access Key with the rights to read from AWS S3 + pub lake_aws_secret_access_key: String, /// AWS S3 bucket region #[clap(long, env="s3_region_name")] pub s3_region_name: String, @@ -27,3 +33,25 @@ pub(crate) struct Opts { #[clap(long, short, env="near_archival_rpc_url")] pub near_archival_rpc_url: String, } + +impl Opts { + // Creates AWS Credentials for NEAR Lake + fn lake_credentials(&self) -> aws_types::credentials::SharedCredentialsProvider { + let provider = aws_types::Credentials::new( + self.lake_aws_access_key.clone(), + self.lake_aws_secret_access_key.clone(), + None, + None, + "events_indexer", + ); + aws_types::credentials::SharedCredentialsProvider::new(provider) + } + + /// Creates AWS Shared Config for NEAR Lake + pub fn lake_aws_sdk_config(&self) -> aws_types::sdk_config::SdkConfig { + aws_types::sdk_config::SdkConfig::builder() + .credentials_provider(self.lake_credentials()) + .region(aws_types::region::Region::new(self.s3_region_name.clone())) + .build() + } +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index b9c6e4f..77d3e4f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ mod metrics_server; mod models; mod rpc_helpers; mod tracing_utils; +use aws_sdk_s3; #[macro_use] extern crate lazy_static; @@ -37,15 +38,15 @@ async fn main() -> anyhow::Result<()> { dotenv().ok(); let opts: Opts = Opts::parse(); + let s3_config = aws_sdk_s3::config::Builder::from(&opts.lake_aws_sdk_config()).build(); + let config_builder = near_lake_framework::LakeConfigBuilder::default().s3_config(s3_config); + + let config = + config_builder + .mainnet() + .start_block_height(opts.start_block_height).build()?; let pool = sqlx::PgPool::connect(&env::var("DATABASE_URL")?).await?; - - let config = near_lake_framework::LakeConfigBuilder::default() - .s3_bucket_name(opts.s3_bucket_name) - .s3_region_name(opts.s3_region_name) - .start_block_height(opts.start_block_height) - .blocks_preload_pool_size(100) - .build()?; - + let env_filter = EnvFilter::new("near_lake_framework=info,indexer_events=info"); let _subscriber = tracing_utils::init_tracing(env_filter).await; From 9526fbcc9efe8df39425dfa0a68044cdace8f73f Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Sat, 19 Nov 2022 13:08:52 +0100 Subject: [PATCH 09/34] added /probe to see if indexer is active --- Cargo.lock | 117 ++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 3 +- src/configs.rs | 14 ++--- src/main.rs | 32 +++++++----- src/metrics_server.rs | 100 +++++++++++++++++++++++++++++++----- 5 files changed, 230 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 02e6588..ba06f60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,6 +73,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.58" @@ -688,15 +697,17 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ - "libc", + "iana-time-zone", + "js-sys", "num-integer", "num-traits", "serde", "time 0.1.44", + "wasm-bindgen", "winapi", ] @@ -748,6 +759,16 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -881,6 +902,50 @@ dependencies = [ "zeroize", ] +[[package]] +name = "cxx" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "darling" version = "0.13.4" @@ -1469,6 +1534,30 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1528,6 +1617,7 @@ dependencies = [ "bigdecimal", "borsh", "cached", + "chrono", "clap", "dotenv", "futures", @@ -1646,6 +1736,15 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "link-cplusplus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +dependencies = [ + "cc", +] + [[package]] name = "lock_api" version = "0.4.7" @@ -2676,6 +2775,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" + [[package]] name = "sct" version = "0.6.1" @@ -3406,6 +3511,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "unicode-xid" version = "0.2.3" diff --git a/Cargo.toml b/Cargo.toml index 9324500..2e1a2e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,4 +42,5 @@ near-lake-framework = "0.5.0" near-primitives = "0.14.0" hyper = { version = "0.14", features = ["full"] } aws-types= "0.13.0" -aws-sdk-s3="0.13.0" \ No newline at end of file +aws-sdk-s3="0.13.0" +chrono="0.4.23" \ No newline at end of file diff --git a/src/configs.rs b/src/configs.rs index b83ac8c..422473b 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -16,21 +16,21 @@ pub(crate) struct Opts { #[clap(long)] pub debug: bool, /// AWS S3 bucket name to get the stream from - #[clap(long, env="s3_bucket_name")] + #[clap(long, env = "s3_bucket_name")] pub s3_bucket_name: String, /// AWS Access Key with the rights to read from AWS S3 - #[clap(long, env="NEAR_LAKE_AWS_KEY")] + #[clap(long, env = "NEAR_LAKE_AWS_KEY")] pub lake_aws_access_key: String, - #[clap(long, env="NEAR_LAKE_AWS_SECRET")] + #[clap(long, env = "NEAR_LAKE_AWS_SECRET")] /// AWS Secret Access Key with the rights to read from AWS S3 pub lake_aws_secret_access_key: String, /// AWS S3 bucket region - #[clap(long, env="s3_region_name")] + #[clap(long, env = "s3_region_name")] pub s3_region_name: String, /// Block height to start the stream from - #[clap(long, short, env="start_block_height")] + #[clap(long, short, env = "start_block_height")] pub start_block_height: u64, - #[clap(long, short, env="near_archival_rpc_url")] + #[clap(long, short, env = "near_archival_rpc_url")] pub near_archival_rpc_url: String, } @@ -54,4 +54,4 @@ impl Opts { .region(aws_types::region::Region::new(self.s3_region_name.clone())) .build() } -} \ No newline at end of file +} diff --git a/src/main.rs b/src/main.rs index 77d3e4f..f4060b5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,10 +4,16 @@ use cached::SizedCache; use clap::Parser; use dotenv::dotenv; use futures::StreamExt; -use metrics_server::{init_metrics_server, BLOCK_PROCESSED_TOTAL, BLOCK_PROCESSING_TIME}; +use metrics_server::{ + init_metrics_server, BLOCK_PROCESSED_TOTAL, LATEST_BLOCK_HEIGHT, LATEST_BLOCK_TIMESTAMP, +}; use near_lake_framework::near_indexer_primitives; -use std::{env, time::SystemTime}; +use std::{ + env, + time::{SystemTime, UNIX_EPOCH}, +}; use tokio::{sync::Mutex, task}; +use aws_sdk_s3; use tracing_subscriber::EnvFilter; mod configs; mod db_adapters; @@ -15,7 +21,6 @@ mod metrics_server; mod models; mod rpc_helpers; mod tracing_utils; -use aws_sdk_s3; #[macro_use] extern crate lazy_static; @@ -40,13 +45,13 @@ async fn main() -> anyhow::Result<()> { let s3_config = aws_sdk_s3::config::Builder::from(&opts.lake_aws_sdk_config()).build(); let config_builder = near_lake_framework::LakeConfigBuilder::default().s3_config(s3_config); - - let config = - config_builder - .mainnet() - .start_block_height(opts.start_block_height).build()?; + + let config = config_builder + .mainnet() + .start_block_height(opts.start_block_height) + .build()?; let pool = sqlx::PgPool::connect(&env::var("DATABASE_URL")?).await?; - + let env_filter = EnvFilter::new("near_lake_framework=info,indexer_events=info"); let _subscriber = tracing_utils::init_tracing(env_filter).await; @@ -115,8 +120,6 @@ async fn handle_streamer_message( ft_balance_cache: &FtBalanceCache, contracts: &db_adapters::contracts::ContractsHelper, ) -> anyhow::Result { - let request_start = SystemTime::now(); - if streamer_message.block.header.height % 100 == 0 { tracing::info!( target: crate::LOGGING_PREFIX, @@ -135,7 +138,12 @@ async fn handle_streamer_message( ) .await?; + let time_since_the_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Something went wrong."); + + LATEST_BLOCK_HEIGHT.set(streamer_message.block.header.height.try_into().unwrap()); + LATEST_BLOCK_TIMESTAMP.set(time_since_the_epoch.as_secs_f64()); BLOCK_PROCESSED_TOTAL.inc(); - BLOCK_PROCESSING_TIME.observe(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); Ok(streamer_message.block.header.height) } diff --git a/src/metrics_server.rs b/src/metrics_server.rs index 8a8c560..6cd9bc4 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -3,20 +3,14 @@ use hyper::{ service::{make_service_fn, service_fn}, Body, Method, Request, Response, Server, }; -use prometheus::{Encoder, Histogram, HistogramOpts, IntCounter, Opts}; +use prometheus::{Encoder, Gauge, IntCounter, IntGauge, Opts}; use std::{convert::Infallible, env}; pub type Result = std::result::Result; +use chrono::{DateTime, NaiveDateTime, Utc}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; -pub fn try_create_histogram(name: &str, help: &str) -> Result { - let opts = HistogramOpts::new(name, help); - let histogram = Histogram::with_opts(opts)?; - prometheus::register(Box::new(histogram.clone()))?; - Ok(histogram) -} -/// Attempts to crate an `IntCounter`, returning `Err` if the registry does not accept the counter -/// (potentially due to naming conflict). pub fn try_create_int_counter(name: &str, help: &str) -> Result { let opts = Opts::new(name, help); let counter = IntCounter::with_opts(opts)?; @@ -24,12 +18,36 @@ pub fn try_create_int_counter(name: &str, help: &str) -> Result Result { + let opts = Opts::new(name, help); + let gauge = IntGauge::with_opts(opts)?; + prometheus::register(Box::new(gauge.clone()))?; + Ok(gauge) +} + +pub fn try_create_gauge(name: &str, help: &str) -> Result { + let opts = Opts::new(name, help); + let gauge = Gauge::with_opts(opts)?; + prometheus::register(Box::new(gauge.clone()))?; + Ok(gauge) +} + lazy_static! { pub static ref BLOCK_PROCESSED_TOTAL: IntCounter = try_create_int_counter("total_blocks_processed", "Total number of blocks processed") .unwrap(); - pub static ref BLOCK_PROCESSING_TIME: Histogram = - try_create_histogram("block_processing_time", "Time spent indexing a block").unwrap(); + pub static ref LATEST_BLOCK_HEIGHT: IntGauge = + try_create_int_gauge("latest_block_height", "Latest Block Height Of Indexer Thus Far") + .unwrap(); + pub static ref LATEST_BLOCK_TIMESTAMP: Gauge = + try_create_gauge("latest_block_timestamp", "Timestamp of when last block was indexed.") + .unwrap(); +} + +enum ProbeResult { + IndexerStarted, + IndexerNotStarted, + SomethingIsWrong, } async fn serve_req(req: Request) -> Result, hyper::Error> { @@ -57,9 +75,65 @@ async fn serve_req(req: Request) -> Result, hyper::Error> { .body(Body::from(res)) .unwrap() } + (&Method::GET, "/probe") => { + // Indexer should process at least 1 block within 2.5 minutes to report that it is working fine. + let encoder = prometheus::TextEncoder::new(); + let mut probe_result = ProbeResult::IndexerNotStarted; + let latest_block_timestamp = LATEST_BLOCK_TIMESTAMP.get(); + let last_known_blockheight = LATEST_BLOCK_HEIGHT.get() as u64; + + let time_since_the_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"); + let time_elapsed = time_since_the_epoch + .checked_sub(Duration::from_secs(latest_block_timestamp as u64)) + .unwrap(); + + if latest_block_timestamp != 0.0 { + match time_elapsed.as_secs().ge(&Duration::new(150, 0).as_secs()) { + true => probe_result = ProbeResult::SomethingIsWrong, + false => probe_result = ProbeResult::IndexerStarted, + } + } + let mut res = "".to_owned(); + + match probe_result { + ProbeResult::IndexerNotStarted => res.push_str("Indexer has not started yet."), + ProbeResult::IndexerStarted => { + res.push_str("Indexer is operating normally. Blocks reported at least every 2.5 minutes."); + } + ProbeResult::SomethingIsWrong => { + res.push_str("Something is wrong. Indexer has not reported any new blocks within 2.5 minutes."); + } + }; + + + if latest_block_timestamp != 0.0 { + res.push_str("\n Last block height was: "); + res.push_str(last_known_blockheight.to_string().as_str()); + res.push_str("\n last block was indexed at timestamp: "); + let last_block_datetime = + NaiveDateTime::from_timestamp_opt(latest_block_timestamp as i64, 0); + + match last_block_datetime { + Some(datetime) => { + let dt = DateTime::::from_utc(datetime, Utc); + res.push_str(&dt.format("%a %b %e %T %Y").to_string()); + } + None => res.push_str("Error Converting time."), + } + + } + + Response::builder() + .status(200) + .header(CONTENT_TYPE, encoder.format_type()) + .body(Body::from(res)) + .unwrap() + } (&Method::GET, "/") => Response::builder() .status(200) - .body(Body::from("")) + .body(Body::from("Service is running.")) .unwrap(), _ => Response::builder() .status(404) @@ -87,7 +161,7 @@ pub async fn init_metrics_server() -> std::result::Result<(), ()> { let server = Server::bind(&addr).serve(make_svc); - println!("Starting metrics server at http://{}", addr); + println!("Starting metrics server at http://{}/metrics", addr); if let Err(e) = server.await { eprintln!("server error: {}", e); From dcd691dfd33337f6f0c0e55dbce6125279901784 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Mon, 21 Nov 2022 21:55:40 +0100 Subject: [PATCH 10/34] enabled mainnet/testnet + changed config env vars --- src/configs.rs | 14 ++++++++------ src/main.rs | 20 +++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/configs.rs b/src/configs.rs index 422473b..3867286 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -16,22 +16,24 @@ pub(crate) struct Opts { #[clap(long)] pub debug: bool, /// AWS S3 bucket name to get the stream from - #[clap(long, env = "s3_bucket_name")] + #[clap(long, env)] pub s3_bucket_name: String, /// AWS Access Key with the rights to read from AWS S3 - #[clap(long, env = "NEAR_LAKE_AWS_KEY")] + #[clap(long, env)] pub lake_aws_access_key: String, - #[clap(long, env = "NEAR_LAKE_AWS_SECRET")] + #[clap(long, env)] /// AWS Secret Access Key with the rights to read from AWS S3 pub lake_aws_secret_access_key: String, /// AWS S3 bucket region - #[clap(long, env = "s3_region_name")] + #[clap(long, env)] pub s3_region_name: String, /// Block height to start the stream from - #[clap(long, short, env = "start_block_height")] + #[clap(long, short, env)] pub start_block_height: u64, - #[clap(long, short, env = "near_archival_rpc_url")] + #[clap(long, short, env)] pub near_archival_rpc_url: String, + #[clap(long,env)] + pub chain_id: String, } impl Opts { diff --git a/src/main.rs b/src/main.rs index f4060b5..ab5c48e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ // TODO cleanup imports in all the files in the end use crate::configs::Opts; +use aws_sdk_s3; use cached::SizedCache; use clap::Parser; use dotenv::dotenv; @@ -13,7 +14,6 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; use tokio::{sync::Mutex, task}; -use aws_sdk_s3; use tracing_subscriber::EnvFilter; mod configs; mod db_adapters; @@ -46,15 +46,21 @@ async fn main() -> anyhow::Result<()> { let s3_config = aws_sdk_s3::config::Builder::from(&opts.lake_aws_sdk_config()).build(); let config_builder = near_lake_framework::LakeConfigBuilder::default().s3_config(s3_config); - let config = config_builder - .mainnet() - .start_block_height(opts.start_block_height) - .build()?; - let pool = sqlx::PgPool::connect(&env::var("DATABASE_URL")?).await?; + let config = match opts.chain_id.as_str() { + "mainnet"=> config_builder.mainnet(), + "testnet" => config_builder.testnet(), + _ => panic!() + } + .start_block_height(opts.start_block_height) + .build()?; + let pool = sqlx::PgPool::connect(&env::var("DATABASE_URL")?).await?; + let env_filter = EnvFilter::new("near_lake_framework=info,indexer_events=info"); let _subscriber = tracing_utils::init_tracing(env_filter).await; - + + tracing::info!(target: LOGGING_PREFIX,"Chain_id: {}", opts.chain_id ); + task::spawn_blocking(move || { if let Err(_val) = init_metrics_server() { tracing::error!( From 4b9b55ae5b1edd8f9615eea2afa75283fb53a4e7 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Wed, 23 Nov 2022 21:58:49 +0100 Subject: [PATCH 11/34] json logging --- Cargo.lock | 28 ++++++++++++++++++++++++++++ Cargo.toml | 3 ++- src/tracing_utils.rs | 14 +++++++++++--- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba06f60..33aa16d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1324,6 +1324,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1643,6 +1653,7 @@ dependencies = [ "tokio-stream", "tracing", "tracing-appender", + "tracing-bunyan-formatter", "tracing-subscriber", ] @@ -3410,6 +3421,23 @@ dependencies = [ "syn", ] +[[package]] +name = "tracing-bunyan-formatter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2445962f94a813b2aaea29ceeccb6dce9fd3aa5b1cb45595cde755b00d021ad" +dependencies = [ + "gethostname", + "log", + "serde", + "serde_json", + "time 0.3.11", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-core" version = "0.1.30" diff --git a/Cargo.toml b/Cargo.toml index 2e1a2e0..9255b07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,4 +43,5 @@ near-primitives = "0.14.0" hyper = { version = "0.14", features = ["full"] } aws-types= "0.13.0" aws-sdk-s3="0.13.0" -chrono="0.4.23" \ No newline at end of file +chrono="0.4.23" +tracing-bunyan-formatter = "0.3.4" \ No newline at end of file diff --git a/src/tracing_utils.rs b/src/tracing_utils.rs index 13122ba..c1eb724 100644 --- a/src/tracing_utils.rs +++ b/src/tracing_utils.rs @@ -6,6 +6,7 @@ use tracing_subscriber::filter::Filtered; use tracing_subscriber::layer::{Layered, SubscriberExt}; use tracing_subscriber::registry::LookupSpan; use tracing_subscriber::{fmt, EnvFilter, Layer}; +use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer}; type LogLayer = Layered< Filtered< @@ -61,7 +62,7 @@ where .with_filter(filter); subscriber.with(layer) -} +} pub fn default_subscriber_with_non_blocking_layer( env_filter: EnvFilter, @@ -71,11 +72,18 @@ pub fn default_subscriber_with_non_blocking_layer( let stderr = std::io::stderr(); let lined_stderr = std::io::LineWriter::new(stderr); let (writer, writer_guard) = tracing_appender::non_blocking(lined_stderr); - + + let formatting_layer = BunyanFormattingLayer::new("indexer-events".into(), std::io::stdout); + + let subscriber = tracing_subscriber::registry(); let subscriber = add_non_blocking_log_layer(env_filter, writer, color_output, subscriber); - + + let subscriber = subscriber + .with(JsonStorageLayer) + .with(formatting_layer); + DefaultSubscriberGuard { subscriber: Some(subscriber), writer_guard: Some(writer_guard), From 506572fcf659706893ce3314e646cbd5930f2438 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Wed, 23 Nov 2022 22:18:57 +0100 Subject: [PATCH 12/34] fmt & clippy --- src/configs.rs | 2 +- src/main.rs | 13 ++++++------- src/metrics_server.rs | 23 ++++++++++++----------- src/tracing_utils.rs | 17 +++++++---------- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/configs.rs b/src/configs.rs index 3867286..5ba003d 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -32,7 +32,7 @@ pub(crate) struct Opts { pub start_block_height: u64, #[clap(long, short, env)] pub near_archival_rpc_url: String, - #[clap(long,env)] + #[clap(long, env)] pub chain_id: String, } diff --git a/src/main.rs b/src/main.rs index ab5c48e..3fc0810 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,5 @@ // TODO cleanup imports in all the files in the end use crate::configs::Opts; -use aws_sdk_s3; use cached::SizedCache; use clap::Parser; use dotenv::dotenv; @@ -47,20 +46,20 @@ async fn main() -> anyhow::Result<()> { let config_builder = near_lake_framework::LakeConfigBuilder::default().s3_config(s3_config); let config = match opts.chain_id.as_str() { - "mainnet"=> config_builder.mainnet(), + "mainnet" => config_builder.mainnet(), "testnet" => config_builder.testnet(), - _ => panic!() + _ => panic!(), } .start_block_height(opts.start_block_height) .build()?; let pool = sqlx::PgPool::connect(&env::var("DATABASE_URL")?).await?; - + let env_filter = EnvFilter::new("near_lake_framework=info,indexer_events=info"); let _subscriber = tracing_utils::init_tracing(env_filter).await; - - tracing::info!(target: LOGGING_PREFIX,"Chain_id: {}", opts.chain_id ); - + + tracing::info!(target: LOGGING_PREFIX, "Chain_id: {}", opts.chain_id); + task::spawn_blocking(move || { if let Err(_val) = init_metrics_server() { tracing::error!( diff --git a/src/metrics_server.rs b/src/metrics_server.rs index 6cd9bc4..d5d80b3 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -10,7 +10,6 @@ pub type Result = std::result::Result; use chrono::{DateTime, NaiveDateTime, Utc}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; - pub fn try_create_int_counter(name: &str, help: &str) -> Result { let opts = Opts::new(name, help); let counter = IntCounter::with_opts(opts)?; @@ -36,12 +35,16 @@ lazy_static! { pub static ref BLOCK_PROCESSED_TOTAL: IntCounter = try_create_int_counter("total_blocks_processed", "Total number of blocks processed") .unwrap(); - pub static ref LATEST_BLOCK_HEIGHT: IntGauge = - try_create_int_gauge("latest_block_height", "Latest Block Height Of Indexer Thus Far") - .unwrap(); - pub static ref LATEST_BLOCK_TIMESTAMP: Gauge = - try_create_gauge("latest_block_timestamp", "Timestamp of when last block was indexed.") - .unwrap(); + pub static ref LATEST_BLOCK_HEIGHT: IntGauge = try_create_int_gauge( + "latest_block_height", + "Latest Block Height Of Indexer Thus Far" + ) + .unwrap(); + pub static ref LATEST_BLOCK_TIMESTAMP: Gauge = try_create_gauge( + "latest_block_timestamp", + "Timestamp of when last block was indexed." + ) + .unwrap(); } enum ProbeResult { @@ -107,14 +110,13 @@ async fn serve_req(req: Request) -> Result, hyper::Error> { } }; - if latest_block_timestamp != 0.0 { res.push_str("\n Last block height was: "); res.push_str(last_known_blockheight.to_string().as_str()); res.push_str("\n last block was indexed at timestamp: "); let last_block_datetime = NaiveDateTime::from_timestamp_opt(latest_block_timestamp as i64, 0); - + match last_block_datetime { Some(datetime) => { let dt = DateTime::::from_utc(datetime, Utc); @@ -122,7 +124,6 @@ async fn serve_req(req: Request) -> Result, hyper::Error> { } None => res.push_str("Error Converting time."), } - } Response::builder() @@ -149,7 +150,7 @@ pub async fn init_metrics_server() -> std::result::Result<(), ()> { // For every connection, we must make a `Service` to handle all // incoming HTTP requests on said connection. let make_svc = make_service_fn(move |_conn| async move { - Ok::<_, Infallible>(service_fn(move |req| serve_req(req))) + Ok::<_, Infallible>(service_fn(serve_req)) }); let port: u16 = match env::var("PORT") { diff --git a/src/tracing_utils.rs b/src/tracing_utils.rs index c1eb724..bf0c673 100644 --- a/src/tracing_utils.rs +++ b/src/tracing_utils.rs @@ -2,11 +2,11 @@ pub use {tracing, tracing_appender, tracing_subscriber}; use std::env; use tracing_appender::non_blocking::NonBlocking; +use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer}; use tracing_subscriber::filter::Filtered; use tracing_subscriber::layer::{Layered, SubscriberExt}; use tracing_subscriber::registry::LookupSpan; use tracing_subscriber::{fmt, EnvFilter, Layer}; -use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer}; type LogLayer = Layered< Filtered< @@ -62,7 +62,7 @@ where .with_filter(filter); subscriber.with(layer) -} +} pub fn default_subscriber_with_non_blocking_layer( env_filter: EnvFilter, @@ -72,18 +72,15 @@ pub fn default_subscriber_with_non_blocking_layer( let stderr = std::io::stderr(); let lined_stderr = std::io::LineWriter::new(stderr); let (writer, writer_guard) = tracing_appender::non_blocking(lined_stderr); - + let formatting_layer = BunyanFormattingLayer::new("indexer-events".into(), std::io::stdout); - - + let subscriber = tracing_subscriber::registry(); let subscriber = add_non_blocking_log_layer(env_filter, writer, color_output, subscriber); - - let subscriber = subscriber - .with(JsonStorageLayer) - .with(formatting_layer); - + + let subscriber = subscriber.with(JsonStorageLayer).with(formatting_layer); + DefaultSubscriberGuard { subscriber: Some(subscriber), writer_guard: Some(writer_guard), From 429d6eb618af66b7b8bf1d3637fa11c73e97ff66 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Tue, 29 Nov 2022 07:21:35 +0100 Subject: [PATCH 13/34] updated metrics + suggestions --- src/main.rs | 19 +++++------ src/metrics_server.rs | 79 ++++++++++--------------------------------- src/tracing_utils.rs | 4 +++ 3 files changed, 31 insertions(+), 71 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3fc0810..521a5fc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,17 +1,16 @@ // TODO cleanup imports in all the files in the end use crate::configs::Opts; use cached::SizedCache; +use chrono::Utc; use clap::Parser; use dotenv::dotenv; use futures::StreamExt; use metrics_server::{ - init_metrics_server, BLOCK_PROCESSED_TOTAL, LATEST_BLOCK_HEIGHT, LATEST_BLOCK_TIMESTAMP, + init_metrics_server, BLOCK_PROCESSED_TOTAL, LAST_SEEN_BLOCK_HEIGHT, LATEST_BLOCK_TIMESTAMP_DIFF, }; use near_lake_framework::near_indexer_primitives; -use std::{ - env, - time::{SystemTime, UNIX_EPOCH}, -}; +use near_primitives::utils::from_timestamp; +use std::env; use tokio::{sync::Mutex, task}; use tracing_subscriber::EnvFilter; mod configs; @@ -143,12 +142,12 @@ async fn handle_streamer_message( ) .await?; - let time_since_the_epoch = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Something went wrong."); + let now = Utc::now(); + let block_timestamp = from_timestamp(streamer_message.block.header.timestamp_nanosec); - LATEST_BLOCK_HEIGHT.set(streamer_message.block.header.height.try_into().unwrap()); - LATEST_BLOCK_TIMESTAMP.set(time_since_the_epoch.as_secs_f64()); + LATEST_BLOCK_TIMESTAMP_DIFF.set((now - block_timestamp).num_seconds() as f64); + LAST_SEEN_BLOCK_HEIGHT.set(streamer_message.block.header.height.try_into().unwrap()); BLOCK_PROCESSED_TOTAL.inc(); + Ok(streamer_message.block.header.height) } diff --git a/src/metrics_server.rs b/src/metrics_server.rs index d5d80b3..479eda5 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -6,9 +6,9 @@ use hyper::{ use prometheus::{Encoder, Gauge, IntCounter, IntGauge, Opts}; use std::{convert::Infallible, env}; +use crate::LOGGING_PREFIX; + pub type Result = std::result::Result; -use chrono::{DateTime, NaiveDateTime, Utc}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; pub fn try_create_int_counter(name: &str, help: &str) -> Result { let opts = Opts::new(name, help); @@ -35,24 +35,18 @@ lazy_static! { pub static ref BLOCK_PROCESSED_TOTAL: IntCounter = try_create_int_counter("total_blocks_processed", "Total number of blocks processed") .unwrap(); - pub static ref LATEST_BLOCK_HEIGHT: IntGauge = try_create_int_gauge( - "latest_block_height", - "Latest Block Height Of Indexer Thus Far" + pub static ref LAST_SEEN_BLOCK_HEIGHT: IntGauge = try_create_int_gauge( + "last_seen_block_height", + "latest block height seen by indexer." ) .unwrap(); - pub static ref LATEST_BLOCK_TIMESTAMP: Gauge = try_create_gauge( + pub static ref LATEST_BLOCK_TIMESTAMP_DIFF: Gauge = try_create_gauge( "latest_block_timestamp", - "Timestamp of when last block was indexed." + "Difference between latest block timestamp and current time." ) .unwrap(); } -enum ProbeResult { - IndexerStarted, - IndexerNotStarted, - SomethingIsWrong, -} - async fn serve_req(req: Request) -> Result, hyper::Error> { let response = match (req.method(), req.uri().path()) { (&Method::GET, "/metrics") => { @@ -79,52 +73,16 @@ async fn serve_req(req: Request) -> Result, hyper::Error> { .unwrap() } (&Method::GET, "/probe") => { - // Indexer should process at least 1 block within 2.5 minutes to report that it is working fine. + // shows the last seen block height and difference between last block_timestamp and now let encoder = prometheus::TextEncoder::new(); - let mut probe_result = ProbeResult::IndexerNotStarted; - let latest_block_timestamp = LATEST_BLOCK_TIMESTAMP.get(); - let last_known_blockheight = LATEST_BLOCK_HEIGHT.get() as u64; - - let time_since_the_epoch = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards"); - let time_elapsed = time_since_the_epoch - .checked_sub(Duration::from_secs(latest_block_timestamp as u64)) - .unwrap(); - - if latest_block_timestamp != 0.0 { - match time_elapsed.as_secs().ge(&Duration::new(150, 0).as_secs()) { - true => probe_result = ProbeResult::SomethingIsWrong, - false => probe_result = ProbeResult::IndexerStarted, - } - } - let mut res = "".to_owned(); + let latest_block_timestamp_diff = LATEST_BLOCK_TIMESTAMP_DIFF.get(); + let last_seen_block_height = LAST_SEEN_BLOCK_HEIGHT.get(); - match probe_result { - ProbeResult::IndexerNotStarted => res.push_str("Indexer has not started yet."), - ProbeResult::IndexerStarted => { - res.push_str("Indexer is operating normally. Blocks reported at least every 2.5 minutes."); - } - ProbeResult::SomethingIsWrong => { - res.push_str("Something is wrong. Indexer has not reported any new blocks within 2.5 minutes."); - } - }; - - if latest_block_timestamp != 0.0 { - res.push_str("\n Last block height was: "); - res.push_str(last_known_blockheight.to_string().as_str()); - res.push_str("\n last block was indexed at timestamp: "); - let last_block_datetime = - NaiveDateTime::from_timestamp_opt(latest_block_timestamp as i64, 0); - - match last_block_datetime { - Some(datetime) => { - let dt = DateTime::::from_utc(datetime, Utc); - res.push_str(&dt.format("%a %b %e %T %Y").to_string()); - } - None => res.push_str("Error Converting time."), - } - } + let mut res = "".to_owned(); + res.push_str("\n Last seen block height: "); + res.push_str(last_seen_block_height.to_string().as_str()); + res.push_str("\n Last seen block timestamp and current time difference (in seconds): "); + res.push_str(latest_block_timestamp_diff.to_string().as_str()); Response::builder() .status(200) @@ -149,9 +107,8 @@ async fn serve_req(req: Request) -> Result, hyper::Error> { pub async fn init_metrics_server() -> std::result::Result<(), ()> { // For every connection, we must make a `Service` to handle all // incoming HTTP requests on said connection. - let make_svc = make_service_fn(move |_conn| async move { - Ok::<_, Infallible>(service_fn(serve_req)) - }); + let make_svc = + make_service_fn(move |_conn| async move { Ok::<_, Infallible>(service_fn(serve_req)) }); let port: u16 = match env::var("PORT") { Ok(val) => val.parse::().unwrap(), @@ -162,7 +119,7 @@ pub async fn init_metrics_server() -> std::result::Result<(), ()> { let server = Server::bind(&addr).serve(make_svc); - println!("Starting metrics server at http://{}/metrics", addr); + tracing::info!(target: LOGGING_PREFIX, "Starting metrics server at http://{}/metrics", addr); if let Err(e) = server.await { eprintln!("server error: {}", e); diff --git a/src/tracing_utils.rs b/src/tracing_utils.rs index bf0c673..a0b4ad5 100644 --- a/src/tracing_utils.rs +++ b/src/tracing_utils.rs @@ -17,6 +17,10 @@ type LogLayer = Layered< Inner, >; +/// https://docs.rs/tracing-appender/0.1.1/tracing_appender/non_blocking/struct.WorkerGuard.html +/// +/// Once dropped, the subscriber is unregistered, and the output is flushed. Any messages output +/// after this value is dropped will be delivered to a previously active subscriber, if any. pub struct DefaultSubscriberGuard { // We must first drop the `local_subscriber_guard` so that no new messages are delivered to // this subscriber while we take care of flushing the messages already in queue. If dropped the From 43d617b3625331db64f5f48c5770b17a35d23526 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Tue, 29 Nov 2022 07:23:11 +0100 Subject: [PATCH 14/34] updated crates order in .toml --- Cargo.toml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9255b07..7450715 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,12 +36,13 @@ prometheus = "0.13.1" atty = "0.2" quote = "1.0.17" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +aws-types= "0.13.0" +aws-sdk-s3="0.13.0" +chrono="0.4.23" +tracing-bunyan-formatter = "0.3.4" +hyper = { version = "0.14", features = ["full"] } + near-jsonrpc-primitives = "0.14.0" near-jsonrpc-client = "0.4.0-beta.0" near-lake-framework = "0.5.0" near-primitives = "0.14.0" -hyper = { version = "0.14", features = ["full"] } -aws-types= "0.13.0" -aws-sdk-s3="0.13.0" -chrono="0.4.23" -tracing-bunyan-formatter = "0.3.4" \ No newline at end of file From 676d348b1b9b1672a842607b8080898b885fafd8 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Tue, 29 Nov 2022 08:42:04 +0100 Subject: [PATCH 15/34] replace hyper with actix, clean up code --- Cargo.lock | 334 +++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 4 +- src/main.rs | 84 +++++------ src/metrics_server.rs | 119 ++++++--------- 4 files changed, 416 insertions(+), 125 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33aa16d..2102e16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,83 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "actix-codec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-sink", + "log", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-http" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c83abf9903e1f0ad9973cc4f7b9767fd5a03a583f51a5b7a339e07987cd2724" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "ahash", + "base64 0.13.0", + "bitflags", + "brotli", + "bytes", + "bytestring", + "derive_more", + "encoding_rs", + "flate2", + "futures-core", + "h2", + "http", + "httparse", + "httpdate", + "itoa", + "language-tags", + "local-channel", + "mime", + "percent-encoding", + "pin-project-lite", + "rand 0.8.5", + "sha1", + "smallvec", + "tracing", + "zstd", +] + +[[package]] +name = "actix-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" +dependencies = [ + "bytestring", + "http", + "regex", + "serde", + "tracing", +] + [[package]] name = "actix-rt" version = "2.7.0" @@ -36,6 +113,97 @@ dependencies = [ "tokio", ] +[[package]] +name = "actix-server" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da34f8e659ea1b077bb4637948b815cd3768ad5a188fdcd74ff4d84240cd824" +dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", + "futures-core", + "futures-util", + "mio", + "num_cpus", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "actix-service" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b894941f818cfdc7ccc4b9e60fa7e53b5042a2e8567270f9147d5591893373a" +dependencies = [ + "futures-core", + "paste", + "pin-project-lite", +] + +[[package]] +name = "actix-utils" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" +dependencies = [ + "local-waker", + "pin-project-lite", +] + +[[package]] +name = "actix-web" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e5ebffd51d50df56a3ae0de0e59487340ca456f05dd0b90c0a7a6dd6a74d31" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-utils", + "actix-web-codegen", + "ahash", + "bytes", + "bytestring", + "cfg-if 1.0.0", + "cookie", + "derive_more", + "encoding_rs", + "futures-core", + "futures-util", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "pin-project-lite", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time 0.3.11", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa9362663c8643d67b2d5eafba49e4cb2c8a053a29ed00a0bea121f17c76b13" +dependencies = [ + "actix-router", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "actix_derive" version = "0.6.0" @@ -47,6 +215,12 @@ dependencies = [ "syn", ] +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "adler32" version = "1.2.0" @@ -73,6 +247,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -588,6 +777,27 @@ dependencies = [ "syn", ] +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bs58" version = "0.4.0" @@ -634,6 +844,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" +[[package]] +name = "bytestring" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7f83e57d9154148e355404702e2694463241880b939570d7c97c014da7a69a1" +dependencies = [ + "bytes", +] + [[package]] name = "c2-chacha" version = "0.3.3" @@ -682,6 +901,9 @@ name = "cc" version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -775,6 +997,17 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "cookie" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" +dependencies = [ + "percent-encoding", + "time 0.3.11", + "version_check", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -1177,6 +1410,16 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1618,6 +1861,7 @@ dependencies = [ name = "indexer-events" version = "0.1.0" dependencies = [ + "actix-web", "anyhow", "atty", "avro-rs", @@ -1632,7 +1876,6 @@ dependencies = [ "dotenv", "futures", "hex", - "hyper", "itertools 0.9.0", "lazy_static", "near-jsonrpc-client", @@ -1706,6 +1949,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.59" @@ -1715,6 +1967,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + [[package]] name = "lazy_static" version = "1.4.0" @@ -1756,6 +2014,24 @@ dependencies = [ "cc", ] +[[package]] +name = "local-channel" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f303ec0e94c6c54447f84f3b0ef7af769858a9c4ef56ef2a986d3dcd4c3fc9c" +dependencies = [ + "futures-core", + "futures-sink", + "futures-util", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" + [[package]] name = "lock_api" version = "0.4.7" @@ -1817,6 +2093,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.4" @@ -2885,6 +3170,17 @@ dependencies = [ "digest 0.10.3", ] +[[package]] +name = "sha1" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.3", +] + [[package]] name = "sha2" version = "0.9.9" @@ -3254,8 +3550,15 @@ dependencies = [ "itoa", "libc", "num_threads", + "time-macros", ] +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "tinyvec" version = "1.6.0" @@ -3869,3 +4172,32 @@ dependencies = [ "syn", "synstructure", ] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.4+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fa202f2ef00074143e219d15b62ffc317d17cc33909feac471c044087cad7b0" +dependencies = [ + "cc", + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index 7450715..908844f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,9 +38,9 @@ quote = "1.0.17" tracing-subscriber = { version = "0.3", features = ["env-filter"] } aws-types= "0.13.0" aws-sdk-s3="0.13.0" -chrono="0.4.23" +chrono = "0.4.19" tracing-bunyan-formatter = "0.3.4" -hyper = { version = "0.14", features = ["full"] } +actix-web = "=4.0.1" near-jsonrpc-primitives = "0.14.0" near-jsonrpc-client = "0.4.0-beta.0" diff --git a/src/main.rs b/src/main.rs index 521a5fc..8d25dab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,7 +11,7 @@ use metrics_server::{ use near_lake_framework::near_indexer_primitives; use near_primitives::utils::from_timestamp; use std::env; -use tokio::{sync::Mutex, task}; +use tokio::sync::Mutex; use tracing_subscriber::EnvFilter; mod configs; mod db_adapters; @@ -59,15 +59,6 @@ async fn main() -> anyhow::Result<()> { tracing::info!(target: LOGGING_PREFIX, "Chain_id: {}", opts.chain_id); - task::spawn_blocking(move || { - if let Err(_val) = init_metrics_server() { - tracing::error!( - target: LOGGING_PREFIX, - "Error setting up the metrics server." - ); - } - }); - let (lake_handle, stream) = near_lake_framework::streamer(config); let json_rpc_client = near_jsonrpc_client::JsonRpcClient::connect(&opts.near_archival_rpc_url); @@ -80,41 +71,44 @@ async fn main() -> anyhow::Result<()> { db_adapters::contracts::ContractsHelper::restore_from_db(&pool, opts.start_block_height) .await?; - let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) - .map(|streamer_message| { - handle_streamer_message( - streamer_message, - &pool, - &json_rpc_client, - &ft_balance_cache, - &contracts, - ) - }) - .buffer_unordered(1usize); - - // let mut time_now = std::time::Instant::now(); - while let Some(handle_message) = handlers.next().await { - match handle_message { - Ok(_block_height) => { - // let elapsed = time_now.elapsed(); - // println!( - // "Elapsed time spent on block {}: {:.3?}", - // block_height, elapsed - // ); - // time_now = std::time::Instant::now(); - } - Err(e) => { - return Err(anyhow::anyhow!(e)); - } - } - } - - // propagate errors from the Lake Framework - match lake_handle.await { - Ok(Ok(())) => Ok(()), - Ok(Err(e)) => Err(e), - Err(e) => Err(anyhow::Error::from(e)), // JoinError - } + tokio::spawn(async move { + let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) + .map(|streamer_message| { + handle_streamer_message( + streamer_message, + &pool, + &json_rpc_client, + &ft_balance_cache, + &contracts, + ) + }) + .buffer_unordered(1usize); + + // let mut time_now = std::time::Instant::now(); + while let Some(_handle_message) = handlers.next().await {} + // match handle_message { + // Ok(_block_height) => { + // // let elapsed = time_now.elapsed(); + // // println!( + // // "Elapsed time spent on block {}: {:.3?}", + // // block_height, elapsed + // // ); + // // time_now = std::time::Instant::now(); + // } + // Err(e) => { + // return Err(anyhow::anyhow!(e)); + // } + // } + // } + }); + init_metrics_server().await?; + Ok(()) + // // propagate errors from the Lake Framework + // match lake_handle.await { + // Ok(Ok(())) => Ok(()), + // Ok(Err(e)) => Err(e), + // Err(e) => Err(anyhow::Error::from(e)), // JoinError + // } } async fn handle_streamer_message( diff --git a/src/metrics_server.rs b/src/metrics_server.rs index 479eda5..d157341 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -1,10 +1,5 @@ -use hyper::{ - header::CONTENT_TYPE, - service::{make_service_fn, service_fn}, - Body, Method, Request, Response, Server, -}; +use actix_web::{get, App, HttpServer, Responder}; use prometheus::{Encoder, Gauge, IntCounter, IntGauge, Opts}; -use std::{convert::Infallible, env}; use crate::LOGGING_PREFIX; @@ -47,82 +42,52 @@ lazy_static! { .unwrap(); } -async fn serve_req(req: Request) -> Result, hyper::Error> { - let response = match (req.method(), req.uri().path()) { - (&Method::GET, "/metrics") => { - let encoder = prometheus::TextEncoder::new(); +#[get("/metrics")] +async fn get_metrics() -> impl Responder { + let encoder = prometheus::TextEncoder::new(); - let mut buffer = Vec::new(); - if let Err(e) = encoder.encode(&prometheus::gather(), &mut buffer) { - eprintln!("could not encode custom metrics: {}", e); - }; - - let res = match String::from_utf8(buffer.clone()) { - Ok(v) => v, - Err(e) => { - eprintln!("custom metrics could not be from_utf8'd: {}", e); - String::default() - } - }; - buffer.clear(); - - Response::builder() - .status(200) - .header(CONTENT_TYPE, encoder.format_type()) - .body(Body::from(res)) - .unwrap() - } - (&Method::GET, "/probe") => { - // shows the last seen block height and difference between last block_timestamp and now - let encoder = prometheus::TextEncoder::new(); - let latest_block_timestamp_diff = LATEST_BLOCK_TIMESTAMP_DIFF.get(); - let last_seen_block_height = LAST_SEEN_BLOCK_HEIGHT.get(); - - let mut res = "".to_owned(); - res.push_str("\n Last seen block height: "); - res.push_str(last_seen_block_height.to_string().as_str()); - res.push_str("\n Last seen block timestamp and current time difference (in seconds): "); - res.push_str(latest_block_timestamp_diff.to_string().as_str()); + let mut buffer = Vec::new(); + if let Err(e) = encoder.encode(&prometheus::gather(), &mut buffer) { + eprintln!("could not encode metrics: {}", e); + }; - Response::builder() - .status(200) - .header(CONTENT_TYPE, encoder.format_type()) - .body(Body::from(res)) - .unwrap() + let res = match String::from_utf8(buffer.clone()) { + Ok(v) => v, + Err(e) => { + eprintln!("custom metrics could not be from_utf8'd: {}", e); + String::default() } - (&Method::GET, "/") => Response::builder() - .status(200) - .body(Body::from("Service is running.")) - .unwrap(), - _ => Response::builder() - .status(404) - .body(Body::from("Missing Page")) - .unwrap(), }; - - Ok(response) + res } -#[tokio::main] -pub async fn init_metrics_server() -> std::result::Result<(), ()> { - // For every connection, we must make a `Service` to handle all - // incoming HTTP requests on said connection. - let make_svc = - make_service_fn(move |_conn| async move { Ok::<_, Infallible>(service_fn(serve_req)) }); - - let port: u16 = match env::var("PORT") { - Ok(val) => val.parse::().unwrap(), - _ => 3000, - }; - - let addr = ([0, 0, 0, 0], port).into(); - - let server = Server::bind(&addr).serve(make_svc); - - tracing::info!(target: LOGGING_PREFIX, "Starting metrics server at http://{}/metrics", addr); +#[get("/probe")] +async fn health_check() -> impl Responder { + // shows the last seen block height and difference between last block_timestamp and now + let latest_block_timestamp_diff = LATEST_BLOCK_TIMESTAMP_DIFF.get(); + let last_seen_block_height = LAST_SEEN_BLOCK_HEIGHT.get(); + + let mut res = "".to_owned(); + res.push_str("\n Last seen block height: "); + res.push_str(last_seen_block_height.to_string().as_str()); + res.push_str("\n Last seen block timestamp and current time difference (in seconds): "); + res.push_str(latest_block_timestamp_diff.to_string().as_str()); + res +} - if let Err(e) = server.await { - eprintln!("server error: {}", e); - } - Ok(()) +pub(crate) async fn init_metrics_server() -> anyhow::Result<(), std::io::Error> { + let port: u16 = std::env::var("HTTP_PORT") + .unwrap_or_else(|_| String::from("3030")) + .parse() + .expect("Unable to parse `HTTP_PORT`"); + + tracing::info!( + target: LOGGING_PREFIX, + "Starting metrics server on http://0.0.0.0:{port}" + ); + + HttpServer::new(|| App::new().service(get_metrics)) + .bind(("0.0.0.0", port))? + .run() + .await } From c05f9262945c13760f132a8a91daf1fbb2fa8f10 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Tue, 29 Nov 2022 08:59:02 +0100 Subject: [PATCH 16/34] undo json logging changes --- Cargo.toml | 6 +-- src/main.rs | 39 +++++++++++++- src/tracing_utils.rs | 125 ++++--------------------------------------- 3 files changed, 50 insertions(+), 120 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 908844f..b07718f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,16 +30,14 @@ syn = "1.0.90" tempfile = "3.3.0" tokio = { version = "1", features = ["full"] } tokio-stream = { version = "0.1" } -tracing = { version = "0.1.36", features = ["std"] } +tracing = "0.1.35" +tracing-subscriber = { version = "0.3.11", features = ["fmt", "std", "env-filter"] } tracing-appender = "0.2.2" prometheus = "0.13.1" -atty = "0.2" quote = "1.0.17" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } aws-types= "0.13.0" aws-sdk-s3="0.13.0" chrono = "0.4.19" -tracing-bunyan-formatter = "0.3.4" actix-web = "=4.0.1" near-jsonrpc-primitives = "0.14.0" diff --git a/src/main.rs b/src/main.rs index 8d25dab..a28b2a7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ use near_primitives::utils::from_timestamp; use std::env; use tokio::sync::Mutex; use tracing_subscriber::EnvFilter; +use tracing_utils::DefaultSubcriberGuard; mod configs; mod db_adapters; mod metrics_server; @@ -54,8 +55,7 @@ async fn main() -> anyhow::Result<()> { let pool = sqlx::PgPool::connect(&env::var("DATABASE_URL")?).await?; - let env_filter = EnvFilter::new("near_lake_framework=info,indexer_events=info"); - let _subscriber = tracing_utils::init_tracing(env_filter).await; + let _writer_guard = init_tracing(); tracing::info!(target: LOGGING_PREFIX, "Chain_id: {}", opts.chain_id); @@ -145,3 +145,38 @@ async fn handle_streamer_message( Ok(streamer_message.block.header.height) } + +fn init_tracing() -> DefaultSubcriberGuard { + let mut env_filter = EnvFilter::new("near_lake_framework=info,indexer_events=info"); + + if let Ok(rust_log) = env::var("RUST_LOG") { + if !rust_log.is_empty() { + for directive in rust_log.split(',').filter_map(|s| match s.parse() { + Ok(directive) => Some(directive), + Err(err) => { + tracing::warn!( + target: crate::LOGGING_PREFIX, + "Ignoring directive `{}`: {}", + s, + err + ); + None + } + }) { + env_filter = env_filter.add_directive(directive); + } + } + } + + let (non_blocking_writer, _guard) = tracing_appender::non_blocking(std::io::stderr()); + + let subscriber = tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(env_filter) + .with_writer(non_blocking_writer) + .finish(); + + DefaultSubcriberGuard { + subscriber_guard: tracing::subscriber::set_default(subscriber), + writer_guard: _guard, + } +} \ No newline at end of file diff --git a/src/tracing_utils.rs b/src/tracing_utils.rs index a0b4ad5..423b667 100644 --- a/src/tracing_utils.rs +++ b/src/tracing_utils.rs @@ -1,119 +1,16 @@ -pub use {tracing, tracing_appender, tracing_subscriber}; - -use std::env; -use tracing_appender::non_blocking::NonBlocking; -use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer}; -use tracing_subscriber::filter::Filtered; -use tracing_subscriber::layer::{Layered, SubscriberExt}; -use tracing_subscriber::registry::LookupSpan; -use tracing_subscriber::{fmt, EnvFilter, Layer}; - -type LogLayer = Layered< - Filtered< - fmt::Layer, - EnvFilter, - Inner, - >, - Inner, ->; - +/// The resource representing a registered subscriber for non_blocking tracing logger +/// +/// since the non_blocking_writer writes to a new thread "at a later point in time", +/// to ensure that all logs get flushed out in case of panics, we return the writer_guard +/// which when dropped, immediately flushes whatever non_blocking_writer has in its +/// cache to its intended place, that could be a file or a std_out. +/// /// https://docs.rs/tracing-appender/0.1.1/tracing_appender/non_blocking/struct.WorkerGuard.html /// /// Once dropped, the subscriber is unregistered, and the output is flushed. Any messages output /// after this value is dropped will be delivered to a previously active subscriber, if any. -pub struct DefaultSubscriberGuard { - // We must first drop the `local_subscriber_guard` so that no new messages are delivered to - // this subscriber while we take care of flushing the messages already in queue. If dropped the - // other way around, the events/spans generated while the subscriber drop guard runs would be - // lost. - subscriber: Option, - #[allow(dead_code)] - writer_guard: Option, -} - -impl DefaultSubscriberGuard { - /// Register this default subscriber globally , for all threads. - /// - /// Must not be called more than once. Mutually exclusive with `Self::local`. - pub fn global(mut self) -> Self { - if let Some(subscriber) = self.subscriber.take() { - tracing::subscriber::set_global_default(subscriber) - .expect("could not set a global subscriber"); - } else { - panic!("trying to set a default subscriber that has been already taken") - } - self - } -} - -fn is_terminal() -> bool { - // Crate `atty` provides a platform-independent way of checking whether the output is a tty. - atty::is(atty::Stream::Stderr) -} - -fn add_non_blocking_log_layer( - filter: EnvFilter, - writer: NonBlocking, - ansi: bool, - subscriber: S, -) -> LogLayer -where - S: tracing::Subscriber + for<'span> LookupSpan<'span> + Send + Sync, -{ - let layer = fmt::layer() - .with_ansi(ansi) - .with_writer(writer) - .with_filter(filter); - - subscriber.with(layer) -} - -pub fn default_subscriber_with_non_blocking_layer( - env_filter: EnvFilter, -) -> DefaultSubscriberGuard { - let color_output = std::env::var_os("NO_COLOR").is_none() && is_terminal(); - - let stderr = std::io::stderr(); - let lined_stderr = std::io::LineWriter::new(stderr); - let (writer, writer_guard) = tracing_appender::non_blocking(lined_stderr); - - let formatting_layer = BunyanFormattingLayer::new("indexer-events".into(), std::io::stdout); - - let subscriber = tracing_subscriber::registry(); - - let subscriber = add_non_blocking_log_layer(env_filter, writer, color_output, subscriber); - - let subscriber = subscriber.with(JsonStorageLayer).with(formatting_layer); - - DefaultSubscriberGuard { - subscriber: Some(subscriber), - writer_guard: Some(writer_guard), - } -} - -pub async fn init_tracing( - mut env_filter: EnvFilter, -) -> DefaultSubscriberGuard { - if let Ok(rust_log) = env::var("RUST_LOG") { - if !rust_log.is_empty() { - for directive in rust_log.split(',').filter_map(|s| match s.parse() { - Ok(directive) => Some(directive), - Err(err) => { - tracing::warn!( - target: crate::LOGGING_PREFIX, - "Ignoring directive `{}`: {}", - s, - err - ); - None - } - }) { - env_filter = env_filter.add_directive(directive); - } - } - } - - let subscriber = default_subscriber_with_non_blocking_layer(env_filter); - - subscriber.global() +#[allow(dead_code)] +pub struct DefaultSubcriberGuard { + pub subscriber_guard: tracing::subscriber::DefaultGuard, + pub writer_guard: tracing_appender::non_blocking::WorkerGuard, } From 056469e3b77d21756d2c7eaf6662f55c9d4a9998 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Tue, 29 Nov 2022 13:25:38 +0100 Subject: [PATCH 17/34] port from env --- Cargo.lock | 29 ----------------------------- Cargo.toml | 2 +- src/configs.rs | 4 ++++ src/main.rs | 6 ------ src/metrics_server.rs | 7 +++---- 5 files changed, 8 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2102e16..73b069e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1567,16 +1567,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "gethostname" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "getrandom" version = "0.1.16" @@ -1863,7 +1853,6 @@ version = "0.1.0" dependencies = [ "actix-web", "anyhow", - "atty", "avro-rs", "aws-sdk-s3", "aws-types", @@ -1896,7 +1885,6 @@ dependencies = [ "tokio-stream", "tracing", "tracing-appender", - "tracing-bunyan-formatter", "tracing-subscriber", ] @@ -3724,23 +3712,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tracing-bunyan-formatter" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2445962f94a813b2aaea29ceeccb6dce9fd3aa5b1cb45595cde755b00d021ad" -dependencies = [ - "gethostname", - "log", - "serde", - "serde_json", - "time 0.3.11", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", -] - [[package]] name = "tracing-core" version = "0.1.30" diff --git a/Cargo.toml b/Cargo.toml index b07718f..45342fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ serde_json = "1.0.55" sqlx = { version = "0.5.13", features = ["runtime-tokio-native-tls", "postgres", "bigdecimal", "json"] } syn = "1.0.90" tempfile = "3.3.0" -tokio = { version = "1", features = ["full"] } +tokio = { version = "1.8", features = ["full"] } tokio-stream = { version = "0.1" } tracing = "0.1.35" tracing-subscriber = { version = "0.3.11", features = ["fmt", "std", "env-filter"] } diff --git a/src/configs.rs b/src/configs.rs index 5ba003d..e72da33 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -32,8 +32,12 @@ pub(crate) struct Opts { pub start_block_height: u64, #[clap(long, short, env)] pub near_archival_rpc_url: String, + // Chain ID: testnet or mainnet #[clap(long, env)] pub chain_id: String, + /// Port to enable metrics/health service + #[clap(long, short, env)] + pub http_port: u16, } impl Opts { diff --git a/src/main.rs b/src/main.rs index 407c819..c692139 100644 --- a/src/main.rs +++ b/src/main.rs @@ -79,12 +79,6 @@ async fn main() -> anyhow::Result<()> { }); init_metrics_server().await?; Ok(()) - // // propagate errors from the Lake Framework - // match lake_handle.await { - // Ok(Ok(())) => Ok(()), - // Ok(Err(e)) => Err(e), - // Err(e) => Err(anyhow::Error::from(e)), // JoinError - // } } async fn handle_streamer_message( diff --git a/src/metrics_server.rs b/src/metrics_server.rs index d157341..f3fea92 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -51,14 +51,13 @@ async fn get_metrics() -> impl Responder { eprintln!("could not encode metrics: {}", e); }; - let res = match String::from_utf8(buffer.clone()) { + match String::from_utf8(buffer.clone()) { Ok(v) => v, Err(e) => { eprintln!("custom metrics could not be from_utf8'd: {}", e); String::default() } - }; - res + } } #[get("/probe")] @@ -77,7 +76,7 @@ async fn health_check() -> impl Responder { pub(crate) async fn init_metrics_server() -> anyhow::Result<(), std::io::Error> { let port: u16 = std::env::var("HTTP_PORT") - .unwrap_or_else(|_| String::from("3030")) + .unwrap_or_else(|_| String::from("3000")) .parse() .expect("Unable to parse `HTTP_PORT`"); From 114fe6d74f9aaa5588a507a9c412a3690f99c4b3 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Tue, 29 Nov 2022 13:38:11 +0100 Subject: [PATCH 18/34] change env var name --- src/configs.rs | 2 +- src/metrics_server.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/configs.rs b/src/configs.rs index e72da33..17706d6 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -37,7 +37,7 @@ pub(crate) struct Opts { pub chain_id: String, /// Port to enable metrics/health service #[clap(long, short, env)] - pub http_port: u16, + pub port: u16, } impl Opts { diff --git a/src/metrics_server.rs b/src/metrics_server.rs index f3fea92..029bc68 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -75,10 +75,10 @@ async fn health_check() -> impl Responder { } pub(crate) async fn init_metrics_server() -> anyhow::Result<(), std::io::Error> { - let port: u16 = std::env::var("HTTP_PORT") + let port: u16 = std::env::var("PORT") .unwrap_or_else(|_| String::from("3000")) .parse() - .expect("Unable to parse `HTTP_PORT`"); + .expect("Unable to parse `PORT`"); tracing::info!( target: LOGGING_PREFIX, From d72f22902b014968c3d44dff053435441874f844 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Tue, 29 Nov 2022 14:15:23 +0100 Subject: [PATCH 19/34] / route --- IMAGE | 0 src/metrics_server.rs | 5 +++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 IMAGE diff --git a/IMAGE b/IMAGE new file mode 100644 index 0000000..e69de29 diff --git a/src/metrics_server.rs b/src/metrics_server.rs index 029bc68..7bbeafb 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -60,7 +60,7 @@ async fn get_metrics() -> impl Responder { } } -#[get("/probe")] +#[get("/")] async fn health_check() -> impl Responder { // shows the last seen block height and difference between last block_timestamp and now let latest_block_timestamp_diff = LATEST_BLOCK_TIMESTAMP_DIFF.get(); @@ -85,7 +85,8 @@ pub(crate) async fn init_metrics_server() -> anyhow::Result<(), std::io::Error> "Starting metrics server on http://0.0.0.0:{port}" ); - HttpServer::new(|| App::new().service(get_metrics)) + HttpServer::new(|| App::new().service(get_metrics) + .service(health_check)) .bind(("0.0.0.0", port))? .run() .await From ae1752220cfcb8da9b89718c7b10f186608c176e Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Tue, 29 Nov 2022 14:34:03 +0100 Subject: [PATCH 20/34] change route back to /probe + remove aws creds setup code --- src/configs.rs | 24 +----------------------- src/main.rs | 13 +++++-------- src/metrics_server.rs | 15 ++++++++++----- 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/src/configs.rs b/src/configs.rs index 17706d6..b4ce7c4 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -38,26 +38,4 @@ pub(crate) struct Opts { /// Port to enable metrics/health service #[clap(long, short, env)] pub port: u16, -} - -impl Opts { - // Creates AWS Credentials for NEAR Lake - fn lake_credentials(&self) -> aws_types::credentials::SharedCredentialsProvider { - let provider = aws_types::Credentials::new( - self.lake_aws_access_key.clone(), - self.lake_aws_secret_access_key.clone(), - None, - None, - "events_indexer", - ); - aws_types::credentials::SharedCredentialsProvider::new(provider) - } - - /// Creates AWS Shared Config for NEAR Lake - pub fn lake_aws_sdk_config(&self) -> aws_types::sdk_config::SdkConfig { - aws_types::sdk_config::SdkConfig::builder() - .credentials_provider(self.lake_credentials()) - .region(aws_types::region::Region::new(self.s3_region_name.clone())) - .build() - } -} +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index c692139..86ee160 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,8 +36,7 @@ async fn main() -> anyhow::Result<()> { dotenv().ok(); let opts: Opts = Opts::parse(); - let s3_config = aws_sdk_s3::config::Builder::from(&opts.lake_aws_sdk_config()).build(); - let config_builder = near_lake_framework::LakeConfigBuilder::default().s3_config(s3_config); + let config_builder = near_lake_framework::LakeConfigBuilder::default(); let config = match opts.chain_id.as_str() { "mainnet" => config_builder.mainnet(), @@ -93,16 +92,14 @@ async fn handle_streamer_message( streamer_message.shards.len() ); } - - db_adapters::events::store_events(pool, &streamer_message).await?; - + LAST_SEEN_BLOCK_HEIGHT.set(streamer_message.block.header.height.try_into().unwrap()); let now = Utc::now(); let block_timestamp = from_timestamp(streamer_message.block.header.timestamp_nanosec); - LATEST_BLOCK_TIMESTAMP_DIFF.set((now - block_timestamp).num_seconds() as f64); - LAST_SEEN_BLOCK_HEIGHT.set(streamer_message.block.header.height.try_into().unwrap()); - BLOCK_PROCESSED_TOTAL.inc(); + db_adapters::events::store_events(pool, &streamer_message).await?; + + BLOCK_PROCESSED_TOTAL.inc(); Ok(streamer_message.block.header.height) } diff --git a/src/metrics_server.rs b/src/metrics_server.rs index 7bbeafb..b209aab 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -60,17 +60,22 @@ async fn get_metrics() -> impl Responder { } } -#[get("/")] +#[get("/probe")] async fn health_check() -> impl Responder { // shows the last seen block height and difference between last block_timestamp and now let latest_block_timestamp_diff = LATEST_BLOCK_TIMESTAMP_DIFF.get(); let last_seen_block_height = LAST_SEEN_BLOCK_HEIGHT.get(); + let num_blocks_processed = BLOCK_PROCESSED_TOTAL.get(); let mut res = "".to_owned(); - res.push_str("\n Last seen block height: "); - res.push_str(last_seen_block_height.to_string().as_str()); - res.push_str("\n Last seen block timestamp and current time difference (in seconds): "); - res.push_str(latest_block_timestamp_diff.to_string().as_str()); + if last_seen_block_height != 0 { + res.push_str("\n Last seen block height: "); + res.push_str(last_seen_block_height.to_string().as_str()); + res.push_str("\n Last seen block timestamp and current time difference (in seconds): "); + res.push_str(latest_block_timestamp_diff.to_string().as_str()); + res.push_str("\n # of Blocks Processed thus far "); + res.push_str(num_blocks_processed.to_string().as_str()); + } res } From e74f6fe8ce62848bb2a8ff0911f2b3e4027cc9ee Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Tue, 29 Nov 2022 16:51:41 +0100 Subject: [PATCH 21/34] remove non blocking tracing_appender --- Cargo.lock | 137 +++--------------------------------------- Cargo.toml | 4 +- src/configs.rs | 2 +- src/main.rs | 41 +++---------- src/metrics_server.rs | 12 +++- src/tracing_utils.rs | 16 ----- 6 files changed, 27 insertions(+), 185 deletions(-) delete mode 100644 src/tracing_utils.rs diff --git a/Cargo.lock b/Cargo.lock index 73b069e..81ab194 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -262,15 +262,6 @@ dependencies = [ "alloc-no-stdlib", ] -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "anyhow" version = "1.0.58" @@ -919,17 +910,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ - "iana-time-zone", - "js-sys", + "libc", "num-integer", "num-traits", "serde", "time 0.1.44", - "wasm-bindgen", "winapi", ] @@ -981,16 +970,6 @@ dependencies = [ "os_str_bytes", ] -[[package]] -name = "codespan-reporting" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] - [[package]] name = "convert_case" version = "0.4.0" @@ -1135,50 +1114,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "cxx" -version = "1.0.82" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.82" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.82" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.82" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "darling" version = "0.13.4" @@ -1777,30 +1712,6 @@ dependencies = [ "tokio-native-tls", ] -[[package]] -name = "iana-time-zone" -version = "0.1.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "winapi", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" -dependencies = [ - "cxx", - "cxx-build", -] - [[package]] name = "ident_case" version = "1.0.1" @@ -1860,7 +1771,6 @@ dependencies = [ "bigdecimal", "borsh", "cached", - "chrono", "clap", "dotenv", "futures", @@ -1884,7 +1794,6 @@ dependencies = [ "tokio", "tokio-stream", "tracing", - "tracing-appender", "tracing-subscriber", ] @@ -1993,15 +1902,6 @@ dependencies = [ "rle-decode-fast", ] -[[package]] -name = "link-cplusplus" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" -dependencies = [ - "cc", -] - [[package]] name = "local-channel" version = "0.1.3" @@ -3059,12 +2959,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "scratch" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" - [[package]] name = "sct" version = "0.6.1" @@ -3679,9 +3573,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" dependencies = [ "cfg-if 1.0.0", "log", @@ -3690,22 +3584,11 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-appender" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" -dependencies = [ - "crossbeam-channel", - "time 0.3.11", - "tracing-subscriber", -] - [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" dependencies = [ "proc-macro2", "quote", @@ -3813,12 +3696,6 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" -[[package]] -name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - [[package]] name = "unicode-xid" version = "0.2.3" diff --git a/Cargo.toml b/Cargo.toml index 45342fc..d2006be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,16 +28,14 @@ serde_json = "1.0.55" sqlx = { version = "0.5.13", features = ["runtime-tokio-native-tls", "postgres", "bigdecimal", "json"] } syn = "1.0.90" tempfile = "3.3.0" -tokio = { version = "1.8", features = ["full"] } +tokio = { version = "1.8", features = ["sync", "time", "macros", "rt-multi-thread"] } tokio-stream = { version = "0.1" } tracing = "0.1.35" tracing-subscriber = { version = "0.3.11", features = ["fmt", "std", "env-filter"] } -tracing-appender = "0.2.2" prometheus = "0.13.1" quote = "1.0.17" aws-types= "0.13.0" aws-sdk-s3="0.13.0" -chrono = "0.4.19" actix-web = "=4.0.1" near-jsonrpc-primitives = "0.14.0" diff --git a/src/configs.rs b/src/configs.rs index b4ce7c4..810d366 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -38,4 +38,4 @@ pub(crate) struct Opts { /// Port to enable metrics/health service #[clap(long, short, env)] pub port: u16, -} \ No newline at end of file +} diff --git a/src/main.rs b/src/main.rs index 86ee160..bea9f30 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ // TODO cleanup imports in all the files in the end use crate::configs::Opts; -use chrono::Utc; +use near_primitives::time::Utc; use clap::Parser; use dotenv::dotenv; use futures::StreamExt; @@ -11,12 +11,11 @@ use near_lake_framework::near_indexer_primitives; use near_primitives::utils::from_timestamp; use std::env; use tracing_subscriber::EnvFilter; -use tracing_utils::DefaultSubcriberGuard; mod configs; mod db_adapters; mod metrics_server; mod models; -mod tracing_utils; + #[macro_use] extern crate lazy_static; @@ -47,36 +46,21 @@ async fn main() -> anyhow::Result<()> { .build()?; let pool = sqlx::PgPool::connect(&env::var("DATABASE_URL")?).await?; - - let _writer_guard = init_tracing(); + init_tracing(); tracing::info!(target: LOGGING_PREFIX, "Chain_id: {}", opts.chain_id); let (_lake_handle, stream) = near_lake_framework::streamer(config); - tokio::spawn(async move { let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) .map(|streamer_message| handle_streamer_message(streamer_message, &pool)) .buffer_unordered(1usize); - // let mut time_now = std::time::Instant::now(); while let Some(_handle_message) = handlers.next().await {} - // match handle_message { - // Ok(_block_height) => { - // // let elapsed = time_now.elapsed(); - // // println!( - // // "Elapsed time spent on block {}: {:.3?}", - // // block_height, elapsed - // // ); - // // time_now = std::time::Instant::now(); - // } - // Err(e) => { - // return Err(anyhow::anyhow!(e)); - // } - // } - // } }); + init_metrics_server().await?; + Ok(()) } @@ -103,7 +87,7 @@ async fn handle_streamer_message( Ok(streamer_message.block.header.height) } -fn init_tracing() -> DefaultSubcriberGuard { +fn init_tracing() { let mut env_filter = EnvFilter::new("near_lake_framework=info,indexer_events=info"); if let Ok(rust_log) = env::var("RUST_LOG") { @@ -125,15 +109,8 @@ fn init_tracing() -> DefaultSubcriberGuard { } } - let (non_blocking_writer, _guard) = tracing_appender::non_blocking(std::io::stderr()); - - let subscriber = tracing_subscriber::FmtSubscriber::builder() + tracing_subscriber::FmtSubscriber::builder() .with_env_filter(env_filter) - .with_writer(non_blocking_writer) - .finish(); - - DefaultSubcriberGuard { - subscriber_guard: tracing::subscriber::set_default(subscriber), - writer_guard: _guard, - } + .with_writer(std::io::stderr) + .init(); } diff --git a/src/metrics_server.rs b/src/metrics_server.rs index b209aab..b370f28 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -75,6 +75,8 @@ async fn health_check() -> impl Responder { res.push_str(latest_block_timestamp_diff.to_string().as_str()); res.push_str("\n # of Blocks Processed thus far "); res.push_str(num_blocks_processed.to_string().as_str()); + } else { + res.push_str("\n Indexer is starting... "); } res } @@ -87,11 +89,15 @@ pub(crate) async fn init_metrics_server() -> anyhow::Result<(), std::io::Error> tracing::info!( target: LOGGING_PREFIX, - "Starting metrics server on http://0.0.0.0:{port}" + "Starting metrics server on http://0.0.0.0:{port}/metrics" ); - HttpServer::new(|| App::new().service(get_metrics) - .service(health_check)) + tracing::info!( + target: LOGGING_PREFIX, + "health probe on http://0.0.0.0:{port}/probe" + ); + + HttpServer::new(|| App::new().service(get_metrics).service(health_check)) .bind(("0.0.0.0", port))? .run() .await diff --git a/src/tracing_utils.rs b/src/tracing_utils.rs deleted file mode 100644 index 423b667..0000000 --- a/src/tracing_utils.rs +++ /dev/null @@ -1,16 +0,0 @@ -/// The resource representing a registered subscriber for non_blocking tracing logger -/// -/// since the non_blocking_writer writes to a new thread "at a later point in time", -/// to ensure that all logs get flushed out in case of panics, we return the writer_guard -/// which when dropped, immediately flushes whatever non_blocking_writer has in its -/// cache to its intended place, that could be a file or a std_out. -/// -/// https://docs.rs/tracing-appender/0.1.1/tracing_appender/non_blocking/struct.WorkerGuard.html -/// -/// Once dropped, the subscriber is unregistered, and the output is flushed. Any messages output -/// after this value is dropped will be delivered to a previously active subscriber, if any. -#[allow(dead_code)] -pub struct DefaultSubcriberGuard { - pub subscriber_guard: tracing::subscriber::DefaultGuard, - pub writer_guard: tracing_appender::non_blocking::WorkerGuard, -} From 64a6d8bf0bf662a7f0130acb83c45747119c61bc Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Wed, 30 Nov 2022 10:07:17 +0100 Subject: [PATCH 22/34] remove /probe + unused dependencies --- Cargo.toml | 4 +--- src/metrics_server.rs | 23 +---------------------- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d2006be..828bd55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ dotenv = "0.15.0" futures = "0.3.5" hex = "0.4" itertools = "0.9.0" -lazy_static = "1.4.0" num-traits = "0.2.11" primitive-types = { version = "0.10.0", default-features = false, features = ["rlp"] } regex = "1.6.0" @@ -32,10 +31,9 @@ tokio = { version = "1.8", features = ["sync", "time", "macros", "rt-multi-threa tokio-stream = { version = "0.1" } tracing = "0.1.35" tracing-subscriber = { version = "0.3.11", features = ["fmt", "std", "env-filter"] } +lazy_static = "1.4.0" prometheus = "0.13.1" quote = "1.0.17" -aws-types= "0.13.0" -aws-sdk-s3="0.13.0" actix-web = "=4.0.1" near-jsonrpc-primitives = "0.14.0" diff --git a/src/metrics_server.rs b/src/metrics_server.rs index b370f28..cc5fb78 100644 --- a/src/metrics_server.rs +++ b/src/metrics_server.rs @@ -60,27 +60,6 @@ async fn get_metrics() -> impl Responder { } } -#[get("/probe")] -async fn health_check() -> impl Responder { - // shows the last seen block height and difference between last block_timestamp and now - let latest_block_timestamp_diff = LATEST_BLOCK_TIMESTAMP_DIFF.get(); - let last_seen_block_height = LAST_SEEN_BLOCK_HEIGHT.get(); - let num_blocks_processed = BLOCK_PROCESSED_TOTAL.get(); - - let mut res = "".to_owned(); - if last_seen_block_height != 0 { - res.push_str("\n Last seen block height: "); - res.push_str(last_seen_block_height.to_string().as_str()); - res.push_str("\n Last seen block timestamp and current time difference (in seconds): "); - res.push_str(latest_block_timestamp_diff.to_string().as_str()); - res.push_str("\n # of Blocks Processed thus far "); - res.push_str(num_blocks_processed.to_string().as_str()); - } else { - res.push_str("\n Indexer is starting... "); - } - res -} - pub(crate) async fn init_metrics_server() -> anyhow::Result<(), std::io::Error> { let port: u16 = std::env::var("PORT") .unwrap_or_else(|_| String::from("3000")) @@ -97,7 +76,7 @@ pub(crate) async fn init_metrics_server() -> anyhow::Result<(), std::io::Error> "health probe on http://0.0.0.0:{port}/probe" ); - HttpServer::new(|| App::new().service(get_metrics).service(health_check)) + HttpServer::new(|| App::new().service(get_metrics)) .bind(("0.0.0.0", port))? .run() .await From 798f9d8cab275ed74c500e753e47618fef0ce473 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Wed, 30 Nov 2022 11:29:31 +0100 Subject: [PATCH 23/34] aws creds from env var --- Cargo.toml | 4 +++- src/configs.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 16 ++++------------ 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 828bd55..99941d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,9 +31,11 @@ tokio = { version = "1.8", features = ["sync", "time", "macros", "rt-multi-threa tokio-stream = { version = "0.1" } tracing = "0.1.35" tracing-subscriber = { version = "0.3.11", features = ["fmt", "std", "env-filter"] } -lazy_static = "1.4.0" prometheus = "0.13.1" quote = "1.0.17" +lazy_static = "1.4.0" +aws-types= "0.13.0" +aws-sdk-s3="0.13.0" actix-web = "=4.0.1" near-jsonrpc-primitives = "0.14.0" diff --git a/src/configs.rs b/src/configs.rs index 810d366..655a0e4 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -1,5 +1,7 @@ use clap::Parser; +use crate::LOGGING_PREFIX; + /// NEAR Indexer for Explorer /// Watches for stream of blocks from the chain #[derive(Parser, Debug)] @@ -39,3 +41,44 @@ pub(crate) struct Opts { #[clap(long, short, env)] pub port: u16, } + +impl Opts { + // Creates AWS Credentials for NEAR Lake + fn lake_credentials(&self) -> aws_types::credentials::SharedCredentialsProvider { + let provider = aws_types::Credentials::new( + self.lake_aws_access_key.clone(), + self.lake_aws_secret_access_key.clone(), + None, + None, + "events_indexer", + ); + aws_types::credentials::SharedCredentialsProvider::new(provider) + } + + /// Creates AWS Shared Config for NEAR Lake + pub fn lake_aws_sdk_config(&self) -> aws_types::sdk_config::SdkConfig { + aws_types::sdk_config::SdkConfig::builder() + .credentials_provider(self.lake_credentials()) + .region(aws_types::region::Region::new(self.s3_region_name.clone())) + .build() + } + + pub async fn get_lake_config(&self) -> near_lake_framework::LakeConfig { + let s3_config = aws_sdk_s3::config::Builder::from(&self.lake_aws_sdk_config()).build(); + let config_builder = near_lake_framework::LakeConfigBuilder::default().s3_config(s3_config); + + tracing::info!(target: LOGGING_PREFIX, "Chain_id: {}", self.chain_id); + + match self.chain_id.as_str() { + "mainnet" => config_builder + .mainnet() + .start_block_height(self.start_block_height), + "testnet" => config_builder + .testnet() + .start_block_height(self.start_block_height), + _ => panic!("CHAIN_ID is not set to a valid enviornment name. Try `mainnet` or `testnet`") + } + .build() + .expect("Failed to build LakeConfig") + } +} diff --git a/src/main.rs b/src/main.rs index bea9f30..a118ff5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,22 +35,14 @@ async fn main() -> anyhow::Result<()> { dotenv().ok(); let opts: Opts = Opts::parse(); - let config_builder = near_lake_framework::LakeConfigBuilder::default(); - - let config = match opts.chain_id.as_str() { - "mainnet" => config_builder.mainnet(), - "testnet" => config_builder.testnet(), - _ => panic!(), - } - .start_block_height(opts.start_block_height) - .build()?; - let pool = sqlx::PgPool::connect(&env::var("DATABASE_URL")?).await?; + init_tracing(); - tracing::info!(target: LOGGING_PREFIX, "Chain_id: {}", opts.chain_id); - + // create a lake configuration with S3 information passed in as ENV vars + let config = opts.get_lake_config().await; let (_lake_handle, stream) = near_lake_framework::streamer(config); + tokio::spawn(async move { let mut handlers = tokio_stream::wrappers::ReceiverStream::new(stream) .map(|streamer_message| handle_streamer_message(streamer_message, &pool)) From 959f82902103507c188d0a9e471d4ec3d67f3da0 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Wed, 30 Nov 2022 17:08:15 +0100 Subject: [PATCH 24/34] json logging support --- Cargo.toml | 2 +- src/configs.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 33 ++------------------------------- 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 99941d1..419ad9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ tempfile = "3.3.0" tokio = { version = "1.8", features = ["sync", "time", "macros", "rt-multi-thread"] } tokio-stream = { version = "0.1" } tracing = "0.1.35" -tracing-subscriber = { version = "0.3.11", features = ["fmt", "std", "env-filter"] } +tracing-subscriber = "0.2.4" prometheus = "0.13.1" quote = "1.0.17" lazy_static = "1.4.0" diff --git a/src/configs.rs b/src/configs.rs index 655a0e4..b106f4a 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -1,4 +1,7 @@ +use std::env; + use clap::Parser; +use tracing_subscriber::EnvFilter; use crate::LOGGING_PREFIX; @@ -82,3 +85,43 @@ impl Opts { .expect("Failed to build LakeConfig") } } + +pub(crate) fn init_tracing(debug: bool) -> anyhow::Result<()> { + let mut env_filter = EnvFilter::new("indexer_events=info"); + + if debug { + env_filter = env_filter + .add_directive("near_lake_framework=debug".parse()?); + } + + if let Ok(rust_log) = env::var("RUST_LOG") { + if !rust_log.is_empty() { + for directive in rust_log.split(',').filter_map(|s| match s.parse() { + Ok(directive) => Some(directive), + Err(err) => { + tracing::warn!( + target: crate::LOGGING_PREFIX, + "Ignoring directive `{}`: {}", + s, + err + ); + None + } + }) { + env_filter = env_filter.add_directive(directive); + } + } + } + + let subscriber = tracing_subscriber::fmt::Subscriber::builder() + .with_env_filter(env_filter) + .with_writer(std::io::stderr); + + if std::env::var("ENABLE_JSON_LOGS").is_ok() { + subscriber.json().init(); + } else { + subscriber.compact().init(); + } + + Ok(()) +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index a118ff5..bfacc0d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,5 @@ // TODO cleanup imports in all the files in the end -use crate::configs::Opts; +use crate::configs::{Opts, init_tracing}; use near_primitives::time::Utc; use clap::Parser; use dotenv::dotenv; @@ -10,7 +10,6 @@ use metrics_server::{ use near_lake_framework::near_indexer_primitives; use near_primitives::utils::from_timestamp; use std::env; -use tracing_subscriber::EnvFilter; mod configs; mod db_adapters; mod metrics_server; @@ -37,7 +36,7 @@ async fn main() -> anyhow::Result<()> { let pool = sqlx::PgPool::connect(&env::var("DATABASE_URL")?).await?; - init_tracing(); + init_tracing(opts.debug)?; // create a lake configuration with S3 information passed in as ENV vars let config = opts.get_lake_config().await; @@ -78,31 +77,3 @@ async fn handle_streamer_message( BLOCK_PROCESSED_TOTAL.inc(); Ok(streamer_message.block.header.height) } - -fn init_tracing() { - let mut env_filter = EnvFilter::new("near_lake_framework=info,indexer_events=info"); - - if let Ok(rust_log) = env::var("RUST_LOG") { - if !rust_log.is_empty() { - for directive in rust_log.split(',').filter_map(|s| match s.parse() { - Ok(directive) => Some(directive), - Err(err) => { - tracing::warn!( - target: crate::LOGGING_PREFIX, - "Ignoring directive `{}`: {}", - s, - err - ); - None - } - }) { - env_filter = env_filter.add_directive(directive); - } - } - } - - tracing_subscriber::FmtSubscriber::builder() - .with_env_filter(env_filter) - .with_writer(std::io::stderr) - .init(); -} From b5545a5c48691a55e0739b2965471aae98ffc787 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Thu, 1 Dec 2022 11:02:37 +0100 Subject: [PATCH 25/34] use default aws provider --- src/configs.rs | 34 +++++----------------------------- src/main.rs | 3 +-- 2 files changed, 6 insertions(+), 31 deletions(-) diff --git a/src/configs.rs b/src/configs.rs index b106f4a..38e27c9 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -23,12 +23,6 @@ pub(crate) struct Opts { /// AWS S3 bucket name to get the stream from #[clap(long, env)] pub s3_bucket_name: String, - /// AWS Access Key with the rights to read from AWS S3 - #[clap(long, env)] - pub lake_aws_access_key: String, - #[clap(long, env)] - /// AWS Secret Access Key with the rights to read from AWS S3 - pub lake_aws_secret_access_key: String, /// AWS S3 bucket region #[clap(long, env)] pub s3_region_name: String, @@ -46,29 +40,11 @@ pub(crate) struct Opts { } impl Opts { - // Creates AWS Credentials for NEAR Lake - fn lake_credentials(&self) -> aws_types::credentials::SharedCredentialsProvider { - let provider = aws_types::Credentials::new( - self.lake_aws_access_key.clone(), - self.lake_aws_secret_access_key.clone(), - None, - None, - "events_indexer", - ); - aws_types::credentials::SharedCredentialsProvider::new(provider) - } - - /// Creates AWS Shared Config for NEAR Lake - pub fn lake_aws_sdk_config(&self) -> aws_types::sdk_config::SdkConfig { - aws_types::sdk_config::SdkConfig::builder() - .credentials_provider(self.lake_credentials()) - .region(aws_types::region::Region::new(self.s3_region_name.clone())) - .build() - } - - pub async fn get_lake_config(&self) -> near_lake_framework::LakeConfig { - let s3_config = aws_sdk_s3::config::Builder::from(&self.lake_aws_sdk_config()).build(); - let config_builder = near_lake_framework::LakeConfigBuilder::default().s3_config(s3_config); + // returns a Lake Config object with AWS credentials passed in. + // Will try to source the AWS credentials from .env file first, and then from .aws/credentials if not found. + // https://docs.aws.amazon.com/sdk-for-rust/latest/dg/credentials.html + pub async fn to_lake_config(&self) -> near_lake_framework::LakeConfig { + let config_builder = near_lake_framework::LakeConfigBuilder::default(); tracing::info!(target: LOGGING_PREFIX, "Chain_id: {}", self.chain_id); diff --git a/src/main.rs b/src/main.rs index bfacc0d..450d3e0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,8 +38,7 @@ async fn main() -> anyhow::Result<()> { init_tracing(opts.debug)?; - // create a lake configuration with S3 information passed in as ENV vars - let config = opts.get_lake_config().await; + let config: near_lake_framework::LakeConfig = opts.to_lake_config().await; let (_lake_handle, stream) = near_lake_framework::streamer(config); tokio::spawn(async move { From d324479d3a6817c01cd731338535618cc6f9cf80 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Thu, 1 Dec 2022 11:09:08 +0100 Subject: [PATCH 26/34] non-blocking logging --- Cargo.toml | 1 + src/configs.rs | 24 ++++++++++++++---------- src/main.rs | 8 ++++---- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 419ad9d..8d3b280 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ tokio = { version = "1.8", features = ["sync", "time", "macros", "rt-multi-threa tokio-stream = { version = "0.1" } tracing = "0.1.35" tracing-subscriber = "0.2.4" +tracing-appender = "0.1.2" prometheus = "0.13.1" quote = "1.0.17" lazy_static = "1.4.0" diff --git a/src/configs.rs b/src/configs.rs index 38e27c9..af3eae2 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -40,8 +40,7 @@ pub(crate) struct Opts { } impl Opts { - // returns a Lake Config object with AWS credentials passed in. - // Will try to source the AWS credentials from .env file first, and then from .aws/credentials if not found. + // returns a Lake Config object where AWS credentials are sourced from .env file first, and then from .aws/credentials if not found. // https://docs.aws.amazon.com/sdk-for-rust/latest/dg/credentials.html pub async fn to_lake_config(&self) -> near_lake_framework::LakeConfig { let config_builder = near_lake_framework::LakeConfigBuilder::default(); @@ -55,19 +54,22 @@ impl Opts { "testnet" => config_builder .testnet() .start_block_height(self.start_block_height), - _ => panic!("CHAIN_ID is not set to a valid enviornment name. Try `mainnet` or `testnet`") + _ => panic!( + "CHAIN_ID is not set to a valid enviornment name. Try `mainnet` or `testnet`" + ), } .build() .expect("Failed to build LakeConfig") } } -pub(crate) fn init_tracing(debug: bool) -> anyhow::Result<()> { +pub(crate) fn init_tracing( + debug: bool, +) -> anyhow::Result { let mut env_filter = EnvFilter::new("indexer_events=info"); if debug { - env_filter = env_filter - .add_directive("near_lake_framework=debug".parse()?); + env_filter = env_filter.add_directive("near_lake_framework=debug".parse()?); } if let Ok(rust_log) = env::var("RUST_LOG") { @@ -89,9 +91,11 @@ pub(crate) fn init_tracing(debug: bool) -> anyhow::Result<()> { } } + let (non_blocking, guard) = tracing_appender::non_blocking(std::io::stdout()); + let subscriber = tracing_subscriber::fmt::Subscriber::builder() - .with_env_filter(env_filter) - .with_writer(std::io::stderr); + .with_writer(non_blocking) + .with_env_filter(env_filter); if std::env::var("ENABLE_JSON_LOGS").is_ok() { subscriber.json().init(); @@ -99,5 +103,5 @@ pub(crate) fn init_tracing(debug: bool) -> anyhow::Result<()> { subscriber.compact().init(); } - Ok(()) -} \ No newline at end of file + Ok(guard) +} diff --git a/src/main.rs b/src/main.rs index 450d3e0..9fc17c9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,5 @@ // TODO cleanup imports in all the files in the end -use crate::configs::{Opts, init_tracing}; -use near_primitives::time::Utc; +use crate::configs::{init_tracing, Opts}; use clap::Parser; use dotenv::dotenv; use futures::StreamExt; @@ -8,6 +7,7 @@ use metrics_server::{ init_metrics_server, BLOCK_PROCESSED_TOTAL, LAST_SEEN_BLOCK_HEIGHT, LATEST_BLOCK_TIMESTAMP_DIFF, }; use near_lake_framework::near_indexer_primitives; +use near_primitives::time::Utc; use near_primitives::utils::from_timestamp; use std::env; mod configs; @@ -35,8 +35,8 @@ async fn main() -> anyhow::Result<()> { let opts: Opts = Opts::parse(); let pool = sqlx::PgPool::connect(&env::var("DATABASE_URL")?).await?; - - init_tracing(opts.debug)?; + + let _worker_guard = init_tracing(opts.debug)?; let config: near_lake_framework::LakeConfig = opts.to_lake_config().await; let (_lake_handle, stream) = near_lake_framework::streamer(config); From 44079711a842dc943c349bc499ec9d1da29353b8 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Thu, 1 Dec 2022 11:54:12 +0100 Subject: [PATCH 27/34] chore: refactor metric names + remove blocktime_diff metric --- src/main.rs | 17 +++++----------- src/{metrics_server.rs => metrics.rs} | 29 +++++++-------------------- 2 files changed, 12 insertions(+), 34 deletions(-) rename src/{metrics_server.rs => metrics.rs} (64%) diff --git a/src/main.rs b/src/main.rs index 9fc17c9..ace7ceb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,16 +3,11 @@ use crate::configs::{init_tracing, Opts}; use clap::Parser; use dotenv::dotenv; use futures::StreamExt; -use metrics_server::{ - init_metrics_server, BLOCK_PROCESSED_TOTAL, LAST_SEEN_BLOCK_HEIGHT, LATEST_BLOCK_TIMESTAMP_DIFF, -}; use near_lake_framework::near_indexer_primitives; -use near_primitives::time::Utc; -use near_primitives::utils::from_timestamp; use std::env; mod configs; mod db_adapters; -mod metrics_server; +mod metrics; mod models; #[macro_use] @@ -49,7 +44,7 @@ async fn main() -> anyhow::Result<()> { while let Some(_handle_message) = handlers.next().await {} }); - init_metrics_server().await?; + metrics::init_metrics_server().await?; Ok(()) } @@ -58,6 +53,9 @@ async fn handle_streamer_message( streamer_message: near_indexer_primitives::StreamerMessage, pool: &sqlx::Pool, ) -> anyhow::Result { + metrics::BLOCK_PROCESSED_TOTAL.inc(); + metrics::LATEST_BLOCK_HEIGHT.set(streamer_message.block.header.height.try_into().unwrap()); + if streamer_message.block.header.height % 100 == 0 { tracing::info!( target: crate::LOGGING_PREFIX, @@ -66,13 +64,8 @@ async fn handle_streamer_message( streamer_message.shards.len() ); } - LAST_SEEN_BLOCK_HEIGHT.set(streamer_message.block.header.height.try_into().unwrap()); - let now = Utc::now(); - let block_timestamp = from_timestamp(streamer_message.block.header.timestamp_nanosec); - LATEST_BLOCK_TIMESTAMP_DIFF.set((now - block_timestamp).num_seconds() as f64); db_adapters::events::store_events(pool, &streamer_message).await?; - BLOCK_PROCESSED_TOTAL.inc(); Ok(streamer_message.block.header.height) } diff --git a/src/metrics_server.rs b/src/metrics.rs similarity index 64% rename from src/metrics_server.rs rename to src/metrics.rs index cc5fb78..7a90402 100644 --- a/src/metrics_server.rs +++ b/src/metrics.rs @@ -1,5 +1,5 @@ use actix_web::{get, App, HttpServer, Responder}; -use prometheus::{Encoder, Gauge, IntCounter, IntGauge, Opts}; +use prometheus::{Encoder, IntCounter, IntGauge, Opts}; use crate::LOGGING_PREFIX; @@ -19,25 +19,15 @@ pub fn try_create_int_gauge(name: &str, help: &str) -> Result Result { - let opts = Opts::new(name, help); - let gauge = Gauge::with_opts(opts)?; - prometheus::register(Box::new(gauge.clone()))?; - Ok(gauge) -} - lazy_static! { - pub static ref BLOCK_PROCESSED_TOTAL: IntCounter = - try_create_int_counter("total_blocks_processed", "Total number of blocks processed") - .unwrap(); - pub static ref LAST_SEEN_BLOCK_HEIGHT: IntGauge = try_create_int_gauge( - "last_seen_block_height", - "latest block height seen by indexer." + pub static ref BLOCK_PROCESSED_TOTAL: IntCounter = try_create_int_counter( + "indexer_events_total_blocks_processed", + "Total number of blocks processed" ) .unwrap(); - pub static ref LATEST_BLOCK_TIMESTAMP_DIFF: Gauge = try_create_gauge( - "latest_block_timestamp", - "Difference between latest block timestamp and current time." + pub static ref LATEST_BLOCK_HEIGHT: IntGauge = try_create_int_gauge( + "indexer_events_latest_block_height", + "Number of indexed blocks" ) .unwrap(); } @@ -71,11 +61,6 @@ pub(crate) async fn init_metrics_server() -> anyhow::Result<(), std::io::Error> "Starting metrics server on http://0.0.0.0:{port}/metrics" ); - tracing::info!( - target: LOGGING_PREFIX, - "health probe on http://0.0.0.0:{port}/probe" - ); - HttpServer::new(|| App::new().service(get_metrics)) .bind(("0.0.0.0", port))? .run() From 57b01c6b6f94db9cf5103ede008e4fd94d975487 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Thu, 1 Dec 2022 12:28:38 +0100 Subject: [PATCH 28/34] remove unused dependencies --- Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8d3b280..63842f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,8 +35,6 @@ tracing-appender = "0.1.2" prometheus = "0.13.1" quote = "1.0.17" lazy_static = "1.4.0" -aws-types= "0.13.0" -aws-sdk-s3="0.13.0" actix-web = "=4.0.1" near-jsonrpc-primitives = "0.14.0" From 8a2b026cd07aeef80e5de9d6dd3136c206201990 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Thu, 1 Dec 2022 12:37:41 +0100 Subject: [PATCH 29/34] chore: remove accidentally checked in file --- Cargo.lock | 1088 +++++++++++++++++++++++++++++++++++----------------- IMAGE | 0 2 files changed, 738 insertions(+), 350 deletions(-) delete mode 100644 IMAGE diff --git a/Cargo.lock b/Cargo.lock index 81ab194..f059a75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,7 +54,7 @@ dependencies = [ "actix-service", "actix-utils", "ahash", - "base64 0.13.0", + "base64 0.13.1", "bitflags", "brotli", "bytes", @@ -188,7 +188,7 @@ dependencies = [ "serde_urlencoded", "smallvec", "socket2", - "time 0.3.11", + "time 0.3.17", "url", ] @@ -233,16 +233,16 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", "once_cell", "version_check", ] [[package]] name = "aho-corasick" -version = "0.7.18" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] @@ -262,11 +262,29 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anyhow" -version = "1.0.58" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" +checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "arrayref" @@ -295,11 +313,32 @@ dependencies = [ "event-listener", ] +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" -version = "0.1.56" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" +checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" dependencies = [ "proc-macro2", "quote", @@ -350,7 +389,7 @@ dependencies = [ "strum_macros 0.18.0", "thiserror", "typed-builder", - "uuid", + "uuid 0.8.2", "zerocopy", ] @@ -508,7 +547,7 @@ dependencies = [ "percent-encoding", "regex", "ring", - "time 0.3.11", + "time 0.3.17", "tracing", ] @@ -624,7 +663,7 @@ dependencies = [ "itoa", "num-integer", "ryu", - "time 0.3.11", + "time 0.3.17", ] [[package]] @@ -660,9 +699,9 @@ checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" [[package]] name = "base64" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "bigdecimal" @@ -716,9 +755,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" dependencies = [ "generic-array", ] @@ -797,15 +836,15 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bumpalo" -version = "3.10.0" +version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "byte-slice-cast" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87c5fdd0166095e1d463fc6cc01aa8ce547ad77a4e84d42eb6762b084e28067e" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" [[package]] name = "byteorder" @@ -815,15 +854,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0b3de4a0c5e67e16066a0715723abd91edc2f9001d09c46e1dca929351e130e" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "bytes-utils" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1934a3ef9cac8efde4966a92781e77713e1ba329f1d42e446c7d7eba340d8ef1" +checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" dependencies = [ "bytes", "either", @@ -889,9 +928,9 @@ checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663" [[package]] name = "cc" -version = "1.0.73" +version = "1.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" dependencies = [ "jobserver", ] @@ -910,15 +949,17 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ - "libc", + "iana-time-zone", + "js-sys", "num-integer", "num-traits", "serde", - "time 0.1.44", + "time 0.1.45", + "wasm-bindgen", "winapi", ] @@ -933,9 +974,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.15" +version = "3.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44bbe24bbd31a185bc2c4f7c2abe80bea13a20d57ee4e55be70ac512bdc76417" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", "bitflags", @@ -950,9 +991,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.2.15" +version = "3.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba52acd3b0a5c33aeada5cdaa3267cdc7c594a98731d4268cdc1532f4264cb4" +checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" dependencies = [ "heck 0.4.0", "proc-macro-error", @@ -970,6 +1011,16 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -983,7 +1034,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" dependencies = [ "percent-encoding", - "time 0.3.11", + "time 0.3.17", "version_check", ] @@ -1005,9 +1056,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" dependencies = [ "libc", ] @@ -1048,9 +1099,9 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -1058,12 +1109,11 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.11" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" dependencies = [ "cfg-if 1.0.0", - "once_cell", ] [[package]] @@ -1114,6 +1164,50 @@ dependencies = [ "zeroize", ] +[[package]] +name = "cxx" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "darling" version = "0.13.4" @@ -1126,12 +1220,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02" +checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" dependencies = [ - "darling_core 0.14.1", - "darling_macro 0.14.1", + "darling_core 0.14.2", + "darling_macro 0.14.2", ] [[package]] @@ -1150,9 +1244,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f" +checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" dependencies = [ "fnv", "ident_case", @@ -1175,11 +1269,11 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5" +checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" dependencies = [ - "darling_core 0.14.1", + "darling_core 0.14.2", "quote", "syn", ] @@ -1199,7 +1293,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ - "darling 0.14.1", + "darling 0.14.2", "proc-macro2", "quote", "syn", @@ -1239,11 +1333,11 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.3" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ - "block-buffer 0.10.2", + "block-buffer 0.10.3", "crypto-common", "subtle", ] @@ -1305,9 +1399,9 @@ dependencies = [ [[package]] name = "either" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "encoding_rs" @@ -1378,11 +1472,10 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" dependencies = [ - "matches", "percent-encoding", ] @@ -1394,9 +1487,9 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" [[package]] name = "futures" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" dependencies = [ "futures-channel", "futures-core", @@ -1409,9 +1502,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ "futures-core", "futures-sink", @@ -1419,15 +1512,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" [[package]] name = "futures-executor" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" dependencies = [ "futures-core", "futures-task", @@ -1436,9 +1529,9 @@ dependencies = [ [[package]] name = "futures-intrusive" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62007592ac46aa7c2b6416f7deb9a8a8f63a01e0f1d6e1787d5630170db2b63e" +checksum = "a604f7a68fbf8103337523b1fadc8ade7361ee3f112f7c680ad179651616aed5" dependencies = [ "futures-core", "lock_api", @@ -1447,15 +1540,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" [[package]] name = "futures-macro" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" dependencies = [ "proc-macro2", "quote", @@ -1464,21 +1557,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" [[package]] name = "futures-task" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" [[package]] name = "futures-util" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-channel", "futures-core", @@ -1494,9 +1587,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" dependencies = [ "typenum", "version_check", @@ -1515,9 +1608,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1526,9 +1619,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" dependencies = [ "bytes", "fnv", @@ -1621,7 +1714,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.3", + "digest 0.10.6", ] [[package]] @@ -1648,9 +1741,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" @@ -1660,9 +1753,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.20" +version = "0.14.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" dependencies = [ "bytes", "futures-channel", @@ -1712,6 +1805,30 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1720,11 +1837,10 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" dependencies = [ - "matches", "unicode-bidi", "unicode-normalization", ] @@ -1765,8 +1881,6 @@ dependencies = [ "actix-web", "anyhow", "avro-rs", - "aws-sdk-s3", - "aws-types", "base64 0.11.0", "bigdecimal", "borsh", @@ -1778,9 +1892,9 @@ dependencies = [ "itertools 0.9.0", "lazy_static", "near-jsonrpc-client", - "near-jsonrpc-primitives", + "near-jsonrpc-primitives 0.14.0", "near-lake-framework", - "near-primitives", + "near-primitives 0.14.0", "num-traits", "primitive-types", "prometheus", @@ -1794,14 +1908,15 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "tracing-appender", "tracing-subscriber", ] [[package]] name = "indexmap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown 0.12.3", @@ -1818,9 +1933,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" [[package]] name = "itertools" @@ -1833,18 +1948,18 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "jobserver" @@ -1857,9 +1972,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.59" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" dependencies = [ "wasm-bindgen", ] @@ -1878,9 +1993,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.126" +version = "0.2.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" [[package]] name = "libflate" @@ -1902,6 +2017,15 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "link-cplusplus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +dependencies = [ + "cc", +] + [[package]] name = "local-channel" version = "0.1.3" @@ -1922,9 +2046,9 @@ checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" [[package]] name = "lock_api" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg", "scopeguard", @@ -1941,26 +2065,20 @@ dependencies = [ [[package]] name = "matchers" -version = "0.1.0" +version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" dependencies = [ "regex-automata", ] -[[package]] -name = "matches" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" - [[package]] name = "md-5" -version = "0.10.1" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ - "digest 0.10.3", + "digest 0.10.6", ] [[package]] @@ -1992,21 +2110,21 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "native-tls" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" dependencies = [ "lazy_static", "libc", @@ -2030,6 +2148,16 @@ dependencies = [ "serde", ] +[[package]] +name = "near-account-id" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d924011380de759c3dc6fdbcda37a19a5c061f56dab69d28a34ecee765e23e4" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "near-chain-configs" version = "0.14.0" @@ -2039,12 +2167,31 @@ dependencies = [ "anyhow", "chrono", "derive_more", - "near-crypto", - "near-primitives", + "near-crypto 0.14.0", + "near-primitives 0.14.0", + "num-rational", + "serde", + "serde_json", + "sha2 0.10.6", + "smart-default", + "tracing", +] + +[[package]] +name = "near-chain-configs" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1faf676a95bd1718b06e5957e01a9415fedf7900f32d94d5bcf70abd678b10a2" +dependencies = [ + "anyhow", + "chrono", + "derive_more", + "near-crypto 0.15.0", + "near-primitives 0.15.0", "num-rational", "serde", "serde_json", - "sha2 0.10.2", + "sha2 0.10.6", "smart-default", "tracing", ] @@ -2056,8 +2203,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a734353027698b21633a49d478e564c61ae0171c32f6912bb8844add15d72ebe" dependencies = [ "chrono", - "near-crypto", - "near-primitives", + "near-crypto 0.14.0", + "near-primitives 0.14.0", "thiserror", "tracing", ] @@ -2069,7 +2216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c17f6f22f1ab710731dfba4101f12a99cac120d6af80b99899bd335bb8971477" dependencies = [ "near-chain-primitives", - "near-primitives", + "near-primitives 0.14.0", ] [[package]] @@ -2080,12 +2227,12 @@ checksum = "e1065d86012eeea838661434549f33bb6267c9950fd2aadd2af617fe773def38" dependencies = [ "actix", "chrono", - "near-chain-configs", + "near-chain-configs 0.14.0", "near-chain-primitives", "near-chunks-primitives", - "near-crypto", + "near-crypto 0.14.0", "near-network-primitives", - "near-primitives", + "near-primitives 0.14.0", "serde", "serde_json", "strum 0.24.1", @@ -2106,7 +2253,7 @@ dependencies = [ "curve25519-dalek", "derive_more", "ed25519-dalek", - "near-account-id", + "near-account-id 0.14.0", "once_cell", "parity-secp256k1", "primitive-types", @@ -2118,35 +2265,60 @@ dependencies = [ "thiserror", ] +[[package]] +name = "near-crypto" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7754612b47737d277fb818e9fdbb1406e90f9e57151c55c3584d714421976cb6" +dependencies = [ + "arrayref", + "blake2", + "borsh", + "bs58", + "c2-chacha", + "curve25519-dalek", + "derive_more", + "ed25519-dalek", + "near-account-id 0.15.0", + "once_cell", + "primitive-types", + "rand 0.7.3", + "secp256k1", + "serde", + "serde_json", + "subtle", + "thiserror", +] + [[package]] name = "near-indexer-primitives" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1c089edfb295575737e94f8bad60d125f632919a742d6648644939b1b847665" dependencies = [ - "near-primitives", + "near-primitives 0.14.0", "serde", "serde_json", ] [[package]] name = "near-jsonrpc-client" -version = "0.4.0-beta.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bba462f54bc35289a1013ed3a2ebfa67cc8b12699f81c12dd67687f200c7b871" +checksum = "d1335ffce1476da6516dcd22b26cece1a495fc725c0e8fec1879073752ac068d" dependencies = [ "borsh", "lazy_static", "log", - "near-chain-configs", - "near-crypto", - "near-jsonrpc-primitives", - "near-primitives", + "near-chain-configs 0.15.0", + "near-crypto 0.15.0", + "near-jsonrpc-primitives 0.15.0", + "near-primitives 0.15.0", "reqwest", "serde", "serde_json", "thiserror", - "uuid", + "uuid 1.2.2", ] [[package]] @@ -2155,24 +2327,40 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34a14ee8ca393c0140cb232789259ebc61b13b4cceb177267d0131f50d0eda6c" dependencies = [ - "near-chain-configs", + "near-chain-configs 0.14.0", "near-client-primitives", - "near-crypto", - "near-primitives", - "near-rpc-error-macro", + "near-crypto 0.14.0", + "near-primitives 0.14.0", + "near-rpc-error-macro 0.14.0", + "serde", + "serde_json", + "thiserror", + "uuid 0.8.2", +] + +[[package]] +name = "near-jsonrpc-primitives" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada226c74f05508c516f109a97b9f23335120d0bfda208f0d187b6bbfe6eef5a" +dependencies = [ + "near-chain-configs 0.15.0", + "near-crypto 0.15.0", + "near-primitives 0.15.0", + "near-rpc-error-macro 0.15.0", "serde", "serde_json", "thiserror", - "uuid", ] [[package]] name = "near-lake-framework" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0316ee594f99e37604a0ec839e0130519e8e42110cfe4049d64d8f4b021c49e" +checksum = "5984129964c79102a489ccd9f2af31b4bb8e9f78b0e757f480cb13d426e1915f" dependencies = [ "anyhow", + "async-stream", "aws-config", "aws-sdk-s3", "aws-types", @@ -2196,8 +2384,8 @@ dependencies = [ "anyhow", "borsh", "chrono", - "near-crypto", - "near-primitives", + "near-crypto 0.14.0", + "near-primitives 0.14.0", "serde", "strum 0.24.1", "tokio", @@ -2217,10 +2405,40 @@ dependencies = [ "derive_more", "easy-ext", "hex", - "near-crypto", - "near-primitives-core", - "near-rpc-error-macro", - "near-vm-errors", + "near-crypto 0.14.0", + "near-primitives-core 0.14.0", + "near-rpc-error-macro 0.14.0", + "near-vm-errors 0.14.0", + "num-rational", + "once_cell", + "primitive-types", + "rand 0.7.3", + "reed-solomon-erasure", + "serde", + "serde_json", + "smart-default", + "strum 0.24.1", + "thiserror", +] + +[[package]] +name = "near-primitives" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97670b302dce15f09bba50f24c67aa08130fd01528cc61d4415892401e88e974" +dependencies = [ + "borsh", + "byteorder", + "bytesize", + "cfg-if 1.0.0", + "chrono", + "derive_more", + "easy-ext", + "hex", + "near-crypto 0.15.0", + "near-primitives-core 0.15.0", + "near-rpc-error-macro 0.15.0", + "near-vm-errors 0.15.0", "num-rational", "once_cell", "primitive-types", @@ -2243,10 +2461,28 @@ dependencies = [ "borsh", "bs58", "derive_more", - "near-account-id", + "near-account-id 0.14.0", "num-rational", "serde", - "sha2 0.10.2", + "sha2 0.10.6", + "strum 0.24.1", +] + +[[package]] +name = "near-primitives-core" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7929e19d862221949734c4a0063a8f55e7069de3a2ebc2d4f4c13497a5e953cb" +dependencies = [ + "base64 0.13.1", + "borsh", + "bs58", + "derive_more", + "near-account-id 0.15.0", + "num-rational", + "serde", + "serde_repr", + "sha2 0.10.6", "strum 0.24.1", ] @@ -2261,13 +2497,35 @@ dependencies = [ "syn", ] +[[package]] +name = "near-rpc-error-core" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36addf90cc04bd547a627b3a292f59d7de4dd6fb5042115419ae901b93ce6c2d" +dependencies = [ + "quote", + "serde", + "syn", +] + [[package]] name = "near-rpc-error-macro" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e837bd4bacd807073ec5ceb85708da7f721b46a4c2a978de86027fb0034ce31" dependencies = [ - "near-rpc-error-core", + "near-rpc-error-core 0.14.0", + "serde", + "syn", +] + +[[package]] +name = "near-rpc-error-macro" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5beb352f3b91d8c491646c2fa4fdbbbf463c7b9c0226951c28f0197de44f99" +dependencies = [ + "near-rpc-error-core 0.15.0", "serde", "syn", ] @@ -2279,29 +2537,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0da466a30f0446639cbd788c30865086fac3e8dcb07a79e51d2b0775ed4261e" dependencies = [ "borsh", - "near-account-id", - "near-rpc-error-macro", + "near-account-id 0.14.0", + "near-rpc-error-macro 0.14.0", "serde", ] [[package]] -name = "nom" -version = "7.1.1" +name = "near-vm-errors" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "5591c9c8afa83a040cb5c3f29bc52b2efae2c32d4bcaee1bba723738da1a5cf6" dependencies = [ - "memchr", - "minimal-lexical", + "borsh", + "near-account-id 0.15.0", + "near-rpc-error-macro 0.15.0", + "serde", + "strum 0.24.1", ] [[package]] -name = "nu-ansi-term" -version = "0.46.0" +name = "nom" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" dependencies = [ - "overload", - "winapi", + "memchr", + "minimal-lexical", ] [[package]] @@ -2360,28 +2621,19 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ "hermit-abi", "libc", ] -[[package]] -name = "num_threads" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" -dependencies = [ - "libc", -] - [[package]] name = "once_cell" -version = "1.13.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "opaque-debug" @@ -2391,9 +2643,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.41" +version = "0.10.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" +checksum = "020433887e44c27ff16365eaa2d380547a94544ad509aff6eb5b6e3e0b27b376" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -2423,9 +2675,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.75" +version = "0.9.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" +checksum = "07d5c8cb6e57b3a3612064d7b18b117912b4ce70955c2504d4b741c9e244b132" dependencies = [ "autocfg", "cc", @@ -2436,15 +2688,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.2.0" +version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4" - -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" [[package]] name = "parity-scale-codec" @@ -2466,7 +2712,7 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1557010476e0595c9b568d16dcfb81b93cdeb157612726f5170d31aa707bed27" dependencies = [ - "proc-macro-crate 1.1.3", + "proc-macro-crate 1.2.1", "proc-macro2", "quote", "syn", @@ -2502,7 +2748,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.3", + "parking_lot_core 0.9.5", ] [[package]] @@ -2521,43 +2767,43 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "paste" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" +checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" [[package]] name = "percent-encoding" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pin-project" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", @@ -2578,15 +2824,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "primitive-types" @@ -2611,10 +2857,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "1.1.3" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" +checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" dependencies = [ + "once_cell", "thiserror", "toml", ] @@ -2645,9 +2892,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.42" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ "unicode-ident", ] @@ -2675,9 +2922,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "quote" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" dependencies = [ "proc-macro2", ] @@ -2709,7 +2956,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha 0.3.1", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -2729,7 +2976,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -2743,11 +2990,11 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", ] [[package]] @@ -2774,7 +3021,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", "redox_syscall", "thiserror", ] @@ -2790,9 +3037,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "aho-corasick", "memchr", @@ -2810,9 +3057,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "remove_dir_all" @@ -2825,11 +3072,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.11" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" +checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ - "base64 0.13.0", + "base64 0.13.1", "bytes", "encoding_rs", "futures-core", @@ -2841,10 +3088,10 @@ dependencies = [ "hyper-tls", "ipnet", "js-sys", - "lazy_static", "log", "mime", "native-tls", + "once_cell", "percent-encoding", "pin-project-lite", "serde", @@ -2883,9 +3130,9 @@ checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" [[package]] name = "rlp" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "999508abb0ae792aabed2460c45b89106d97fe4adac593bdaef433c2605847b5" +checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" dependencies = [ "bytes", "rustc-hex", @@ -2912,7 +3159,7 @@ version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ - "base64 0.13.0", + "base64 0.13.1", "log", "ring", "sct", @@ -2933,15 +3180,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24c8ad4f0c00e1eb5bc7614d236a7f1300e3dbd76b68cac8e06fb00b015ad8d8" +checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" [[package]] name = "ryu" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" [[package]] name = "schannel" @@ -2950,7 +3197,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -2959,6 +3206,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" + [[package]] name = "sct" version = "0.6.1" @@ -2969,11 +3222,30 @@ dependencies = [ "untrusted", ] +[[package]] +name = "secp256k1" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff55dc09d460954e9ef2fa8a7ced735a964be9981fd50e870b2b3b0705e14964" +dependencies = [ + "rand 0.8.5", + "secp256k1-sys", +] + +[[package]] +name = "secp256k1-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83080e2c2fc1006e625be82e5d1eb6a43b7fd9578b617fcc55814daf286bba4b" +dependencies = [ + "cc", +] + [[package]] name = "security-framework" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" dependencies = [ "bitflags", "core-foundation", @@ -2994,24 +3266,24 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" +checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" [[package]] name = "serde" -version = "1.0.140" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" +checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.140" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" +checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" dependencies = [ "proc-macro2", "quote", @@ -3020,15 +3292,26 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.82" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" +checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" dependencies = [ "itoa", "ryu", "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3043,24 +3326,24 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.3", + "digest 0.10.6", ] [[package]] name = "sha1" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.3", + "digest 0.10.6", ] [[package]] @@ -3078,13 +3361,13 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.2" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.3", + "digest 0.10.6", ] [[package]] @@ -3107,9 +3390,9 @@ dependencies = [ [[package]] name = "signature" -version = "1.5.0" +version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f054c6c1a6e95179d6f23ed974060dcefb2d9388bb7256900badad682c499de4" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" [[package]] name = "slab" @@ -3122,9 +3405,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "smart-default" @@ -3139,9 +3422,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.4" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" dependencies = [ "libc", "winapi", @@ -3159,7 +3442,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4b7922be017ee70900be125523f38bdd644f4f06a1b16e8fa5a8ee8c34bffd4" dependencies = [ - "itertools 0.10.3", + "itertools 0.10.5", "nom", "unicode_categories", ] @@ -3182,7 +3465,7 @@ checksum = "e48c61941ccf5ddcada342cd59e3e5173b007c509e1e8e990dafc830294d9dc5" dependencies = [ "ahash", "atoi", - "base64 0.13.0", + "base64 0.13.1", "bigdecimal", "bitflags", "byteorder", @@ -3214,7 +3497,7 @@ dependencies = [ "serde", "serde_json", "sha-1", - "sha2 0.10.2", + "sha2 0.10.6", "smallvec", "sqlformat", "sqlx-rt", @@ -3238,7 +3521,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "sha2 0.10.2", + "sha2 0.10.6", "sqlx-core", "sqlx-rt", "syn", @@ -3291,7 +3574,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ - "strum_macros 0.24.2", + "strum_macros 0.24.3", ] [[package]] @@ -3308,9 +3591,9 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.24.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4faebde00e8ff94316c01800f9054fd2ba77d30d9e922541913051d1d978918b" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck 0.4.0", "proc-macro2", @@ -3327,9 +3610,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.98" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" dependencies = [ "proc-macro2", "quote", @@ -3379,24 +3662,24 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.31" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.31" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" dependencies = [ "proc-macro2", "quote", @@ -3414,9 +3697,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -3425,21 +3708,30 @@ dependencies = [ [[package]] name = "time" -version = "0.3.11" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c91f41dcb2f096c05f0873d667dceec1087ce5bcf984ec8ffb19acddbb3217" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ "itoa", - "libc", - "num_threads", + "serde", + "time-core", "time-macros", ] +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + [[package]] name = "time-macros" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +dependencies = [ + "time-core", +] [[package]] name = "tinyvec" @@ -3458,9 +3750,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.20.1" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" +checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" dependencies = [ "autocfg", "bytes", @@ -3468,7 +3760,6 @@ dependencies = [ "memchr", "mio", "num_cpus", - "once_cell", "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", @@ -3479,9 +3770,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", @@ -3511,9 +3802,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" +checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" dependencies = [ "futures-core", "pin-project-lite", @@ -3522,9 +3813,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" dependencies = [ "bytes", "futures-core", @@ -3561,9 +3852,9 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" [[package]] name = "tower-service" @@ -3573,9 +3864,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.35" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "log", @@ -3584,11 +3875,22 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9965507e507f12c8901432a33e31131222abac31edd90cabbcf85cf544b7127a" +dependencies = [ + "chrono", + "crossbeam-channel", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", @@ -3616,22 +3918,36 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" dependencies = [ + "ansi_term", + "chrono", + "lazy_static", "matchers", - "nu-ansi-term", - "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -3659,9 +3975,9 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "uint" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f03af7ccf01dd611cc450a0d10dbc9b745770d096473e2faf0ca6e2d66d1e0" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" dependencies = [ "byteorder", "crunchy", @@ -3677,30 +3993,36 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] name = "unicode-normalization" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.9.0" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" + +[[package]] +name = "unicode-width" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode-xid" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unicode_categories" @@ -3716,21 +4038,20 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.2.2" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", "idna", - "matches", "percent-encoding", ] [[package]] name = "urlencoding" -version = "2.1.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821" +checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" [[package]] name = "uuid" @@ -3738,10 +4059,19 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", "serde", ] +[[package]] +name = "uuid" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +dependencies = [ + "getrandom 0.2.8", +] + [[package]] name = "valuable" version = "0.1.0" @@ -3790,9 +4120,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3800,9 +4130,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" dependencies = [ "bumpalo", "log", @@ -3815,9 +4145,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3827,9 +4157,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3837,9 +4167,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", @@ -3850,15 +4180,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" [[package]] name = "web-sys" -version = "0.3.59" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" dependencies = [ "js-sys", "wasm-bindgen", @@ -3876,10 +4206,11 @@ dependencies = [ [[package]] name = "whoami" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524b58fa5a20a2fb3014dd6358b70e6579692a56ef6fce928834e488f42f65e8" +checksum = "d6631b6a2fd59b1841b622e8f1a7ad241ef0a46f2d580464ce8140ac94cbd571" dependencies = [ + "bumpalo", "wasm-bindgen", "web-sys", ] @@ -3921,43 +4252,100 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + [[package]] name = "windows_i686_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + [[package]] name = "windows_i686_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + [[package]] name = "winreg" version = "0.10.1" @@ -4011,9 +4399,9 @@ dependencies = [ [[package]] name = "zeroize_derive" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" +checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ "proc-macro2", "quote", diff --git a/IMAGE b/IMAGE deleted file mode 100644 index e69de29..0000000 From 1db7a7d2b2c0d2e1166bb205c569123cbeb894b4 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Fri, 2 Dec 2022 08:17:49 +0100 Subject: [PATCH 30/34] update rust builder --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b004679..ab7c657 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.62 AS builder +FROM rust:1.64 AS builder WORKDIR /tmp/ # this build step will cache your dependencies From 457b6686e496ee59711f785197b00986f2c5f518 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Fri, 2 Dec 2022 16:47:07 +0100 Subject: [PATCH 31/34] suggestions and refactoring --- Cargo.toml | 2 +- src/configs.rs | 11 ++++------- src/main.rs | 2 +- src/metrics.rs | 4 ++-- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 63842f4..ac045ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "indexer-events" version = "0.1.0" edition = "2021" -rust-version = "1.62.1" +rust-version = "1.64" [lib] proc-macro = true diff --git a/src/configs.rs b/src/configs.rs index af3eae2..22f0c4f 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -48,16 +48,13 @@ impl Opts { tracing::info!(target: LOGGING_PREFIX, "Chain_id: {}", self.chain_id); match self.chain_id.as_str() { - "mainnet" => config_builder - .mainnet() - .start_block_height(self.start_block_height), - "testnet" => config_builder - .testnet() - .start_block_height(self.start_block_height), + "mainnet" => config_builder.mainnet(), + "testnet" => config_builder.testnet(), _ => panic!( - "CHAIN_ID is not set to a valid enviornment name. Try `mainnet` or `testnet`" + "CHAIN_ID is not set to a valid environment name. Try `mainnet` or `testnet`" ), } + .start_block_height(start_block_height) .build() .expect("Failed to build LakeConfig") } diff --git a/src/main.rs b/src/main.rs index ace7ceb..7b776f0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -54,7 +54,7 @@ async fn handle_streamer_message( pool: &sqlx::Pool, ) -> anyhow::Result { metrics::BLOCK_PROCESSED_TOTAL.inc(); - metrics::LATEST_BLOCK_HEIGHT.set(streamer_message.block.header.height.try_into().unwrap()); + metrics::LATEST_BLOCK_HEIGHT.set(i64::try_from(streamer_message.block.header.height)?); if streamer_message.block.header.height % 100 == 0 { tracing::info!( diff --git a/src/metrics.rs b/src/metrics.rs index 7a90402..b6d0ea8 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -22,12 +22,12 @@ pub fn try_create_int_gauge(name: &str, help: &str) -> Result Date: Fri, 2 Dec 2022 16:55:37 +0100 Subject: [PATCH 32/34] comment for clarity + fix build err --- src/configs.rs | 2 +- src/main.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/configs.rs b/src/configs.rs index 22f0c4f..b4e68bd 100644 --- a/src/configs.rs +++ b/src/configs.rs @@ -54,7 +54,7 @@ impl Opts { "CHAIN_ID is not set to a valid environment name. Try `mainnet` or `testnet`" ), } - .start_block_height(start_block_height) + .start_block_height(self.start_block_height) .build() .expect("Failed to build LakeConfig") } diff --git a/src/main.rs b/src/main.rs index 7b776f0..636dd45 100644 --- a/src/main.rs +++ b/src/main.rs @@ -54,6 +54,7 @@ async fn handle_streamer_message( pool: &sqlx::Pool, ) -> anyhow::Result { metrics::BLOCK_PROCESSED_TOTAL.inc(); + // Prometheus Guage Metric type only supports i64 and f64 types for the time being, so we need to cast the type into i64 metrics::LATEST_BLOCK_HEIGHT.set(i64::try_from(streamer_message.block.header.height)?); if streamer_message.block.header.height % 100 == 0 { From 5dde7f241c08bfea8926f812630448b61dc98e01 Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Fri, 2 Dec 2022 17:01:45 +0100 Subject: [PATCH 33/34] cargo.toml EOF --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index ac045ee..f98ba86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,3 +41,4 @@ near-jsonrpc-primitives = "0.14.0" near-jsonrpc-client = "0.4.0-beta.0" near-lake-framework = "0.5.0" near-primitives = "0.14.0" + From 3228e6f4356cd7d8da11b084ba4e9453d9da757e Mon Sep 17 00:00:00 2001 From: Roshaan Siddiqui Date: Mon, 5 Dec 2022 11:39:08 +0100 Subject: [PATCH 34/34] order cargo.toml alphabetically --- Cargo.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f98ba86..654acc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ rust-version = "1.64" proc-macro = true [dependencies] +actix-web = "=4.0.1" anyhow = "1.0.51" avro-rs = "0.13.0" base64 = "0.11" @@ -19,8 +20,11 @@ dotenv = "0.15.0" futures = "0.3.5" hex = "0.4" itertools = "0.9.0" +lazy_static = "1.4.0" num-traits = "0.2.11" primitive-types = { version = "0.10.0", default-features = false, features = ["rlp"] } +prometheus = "0.13.1" +quote = "1.0.17" regex = "1.6.0" serde = { version = "1", features = ["derive"] } serde_json = "1.0.55" @@ -30,12 +34,8 @@ tempfile = "3.3.0" tokio = { version = "1.8", features = ["sync", "time", "macros", "rt-multi-thread"] } tokio-stream = { version = "0.1" } tracing = "0.1.35" -tracing-subscriber = "0.2.4" tracing-appender = "0.1.2" -prometheus = "0.13.1" -quote = "1.0.17" -lazy_static = "1.4.0" -actix-web = "=4.0.1" +tracing-subscriber = "0.2.4" near-jsonrpc-primitives = "0.14.0" near-jsonrpc-client = "0.4.0-beta.0"