diff --git a/.env.example b/.env.example index 6d8ec66..c237a14 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ archiver_pk="..." backfill_pk="..." +backfill_start_block="0" network="./networks/your_network.json" DATABASE_HOST="" diff --git a/README.md b/README.md index db03ef7..df3ce55 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ While a WeaveVM Archiver node can run without web2 component dependencies, this ```js archiver_pk="" // WeaveVM archiver PK backfill_pk="" // WeaveVM backfill PK +backfill_start_block="0" // it defaults to 0 (genesis), but it's dynamic, so you can specify from which block number you want to start backfilling network="./networks/your_network.json" DATABASE_HOST="" // planetscale @@ -52,6 +53,15 @@ To start archiving your network block data on WeaveVM: 6. set `start_block` value to the most recent network's blockheight. That will facilitate the archiver to start in sync with live blockheight while, in parallel, reindexing from genesis using the `backfill_address`. 7. Set up your PlanetScale DB according to `db_schema.sql`. +#### Parallel Threads of Archiving + +As mentioned previously, `archiver_address` is responsible for archiving blocks starting from the `start_block` specified in your `network.json` config file, while also keeping up with the network’s current blockheight (live sync). Meanwhile, `backfill_address` handles archiving blocks from `backfill_start_block` up to `start_block`. + +```txt +backfill thread: backfill_start_block -> start_block +live sync thread: start_block -> network's live blockheight +``` + ### RPC Proxy and Caching You can use [eRPC](https://github.com/erpc/erpc) to cache, load-balance and failover between as many RPC endpoints and use eRPC's proxy URL in each network's config for WeaveVM. This will increase performance and resiliency and reduce RPC usage cost while fetching network's block data via WeaveVM. diff --git a/src/utils/backfill_genesis.rs b/src/utils/backfill_genesis.rs index 8202b56..f5072be 100644 --- a/src/utils/backfill_genesis.rs +++ b/src/utils/backfill_genesis.rs @@ -1,12 +1,14 @@ use crate::utils::archive_block::archive; use crate::utils::planetscale::ps_archive_block; use crate::utils::schema::Network; +use crate::utils::env_var::get_env_var; use anyhow::{Error, Ok}; pub async fn backfill_from_genesis() -> Result<(), Error> { let network = Network::config(); let config_start_block = network.start_block; - let backfill_blocks: Vec = (0..=config_start_block).collect(); + let backfill_block_start: String = get_env_var("backfill_start_block").unwrap_or(0.to_string()); + let backfill_blocks: Vec = (backfill_block_start.parse::().unwrap()..=config_start_block).collect(); if config_start_block == 0 { return Ok(());