feat(rust/cardano-chain-follower): Auto-sync mithril snapshot

.config/dictionaries/project.dic

@@ -1,4 +1,5 @@
 aarch
+abcz
 ABNF
 abnf
 addrr
@@ -43,12 +44,11 @@ crontagged
 cstring
 dalek
 dashmap
-dashmap
-Datelike
 Datelike
 DBSTATUS
 dbsync
 dcbor
+decompressor
 delegators
 dockerhub
 Dominik
@@ -72,6 +72,7 @@ filestat
 filestorage
 filesystems
 fkey
+fmmap
 fmtchk
 fmtfix
 fontawesome
@@ -86,17 +87,19 @@ gmtime
 gossipsub
 happ
 hardano
+Hardlink
 hasher
+hexdigit
 highwater
 hmod
+humantime
 ideascale
 idents
 IFMT
 Intellij
 ioerr
 iohk
 ipfs
-ipfs
 ipld
 jetbrains
 jorm
@@ -116,13 +119,16 @@ lintfix
 lipsum
 livedocs
 localizable
+logcall
 lookaside
 maindbname
 mapref
 mdlint
 mdns
+memx
 Metadatum
 metadatum
+mimalloc
 minicbor
 miniprotocol
 miniprotocols
@@ -131,7 +137,10 @@ mitigations
 mkcron
 mkdelay
 mkdirat
+Mmap
 moderations
+moka
+MPMC
 msvc
 Multiaddr
 multiera
@@ -177,6 +186,7 @@ redoc
 REMOVEDIR
 renameat
 Replayability
+Repr
 reqwest
 retriggering
 rlib
@@ -199,7 +209,9 @@ scanstatus
 Sched
 scrollability
 seckey
+Signdata
 sitedocs
+skiplist
 slotno
 smac
 stevenj
@@ -221,12 +233,14 @@ tinyvec
 toobig
 toolsets
 Traceback
+txmonitor
 txns
 typenum
 unfinalized
 unixfs
 unlinkat
 upnp
+ureq
 userid
 utimensat
 UTXO
@@ -236,11 +250,10 @@ vkey
 vkeywitness
 voteplan
 voteplans
+wallclock
 WASI
 wasi
 wasip
-wasip
-wasip
 wasmtime
 webasm
 webassembly

rust/cardano-chain-follower/testbed/Cargo.toml → integration_tests/rust/Cargo.toml

@@ -23,11 +23,9 @@ bare_urls = "deny"
 unescaped_backticks = "deny"
 
 [workspace.lints.clippy]
-pedantic = { level = "deny", priority = -1 }
+pedantic = "deny"
 unwrap_used = "deny"
 expect_used = "deny"
-todo = "deny"
-unimplemented = "deny"
 exit = "deny"
 get_unwrap = "deny"
 index_refutable_slice = "deny"

integration_tests/rust/Earthfile

@@ -0,0 +1,25 @@
+VERSION 0.8
+
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/mithril_snapshot:v3.1.8 AS mithril-snapshot-ci
+IMPORT ../../rust AS rust-workspace
+
+local-build:
+    FROM rust-workspace+builder
+
+    WORKDIR crates/cardano-chain-follower/testbed
+    RUN cargo build -p overhead_benchmark --release
+
+    SAVE ARTIFACT target/release/overhead_benchmark overhead_benchmark
+
+local-run-preprod:
+    ARG --required BENCH_NAME
+
+    FROM +local-build
+
+    COPY --dir mithril-snapshot-ci+package-preprod-snapshot/snapshot/immutable mithril_snapshot 
+    COPY +build/overhead_benchmark overhead_benchmark_bin
+    RUN ./overhead_benchmark_bin --bench-name $BENCH_NAME --mithril-snapshot-path ./mithril_snapshot
+
+local-save-preprod-snapshot:
+    FROM mithril-snapshot-ci+package-preprod-snapshot
+    SAVE ARTIFACT immutable AS LOCAL local_preprod_mithril_snapshot 

rust/cardano-chain-follower/testbed/overhead_benchmark/Cargo.toml → integration_tests/rust/overhead_benchmark/Cargo.toml

@@ -11,6 +11,6 @@ cardano-chain-follower = { path = "../.." }
 
 anyhow = "1.0.82"
 clap = { version = "4.5.4", features = ["derive", "help", "usage", "std"], default-features = false }
-pallas-traverse = "0.24.0"
-pallas-hardano = "0.24.0"
+pallas-traverse = "0.30.1"
+pallas-hardano = "0.30.1"
 tokio = { version = "1.37.0", features = ["macros", "sync", "rt-multi-thread", "rt", "net"] }

rust/cardano-chain-follower/testbed/overhead_benchmark/src/benchs/cardano_chain_follower.rs → integration_tests/rust/overhead_benchmark/src/benchs/cardano_chain_follower.rs

@@ -32,7 +32,10 @@ pub async fn run(params: BenchmarkParams) -> anyhow::Result<()> {
         let update = follower.next().await?;
 
         match update {
-            ChainUpdate::Block(raw_block_data) => {
+            ChainUpdate::ImmutableBlockRollback(data)
+            | ChainUpdate::BlockTip(data)
+            | ChainUpdate::ImmutableBlock(data)
+            | ChainUpdate::Block(raw_block_data) => {
                 let block_data = raw_block_data.decode()?;
 
                 monitor_task_handle.send_update(monitor::BenchmarkStats {

rust/cardano-chain-follower/testbed/rust-toolchain.toml → integration_tests/rust/rust-toolchain.toml

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "1.78"
+channel = "1.80"
 profile = "default"

rust/Earthfile

@@ -1,6 +1,7 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.2.00 AS rust-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:feat/cardano-chain-follower-changes AS rust-ci
+#IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.2.00 AS rust-ci
 
 # Use when debugging cat-ci locally.
 # IMPORT ../../catalyst-ci/earthly/rust AS rust-ci

rust/cardano-chain-follower/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cardano-chain-follower"
-version = "0.0.1"
+version = "0.0.2"
 edition.workspace = true
 authors.workspace = true
 homepage.workspace = true
@@ -11,13 +11,85 @@ license.workspace = true
 workspace = true
 
 [dependencies]
-pallas = { git = "https://github.com/input-output-hk/catalyst-pallas.git", rev = "9b5183c8b90b90fe2cc319d986e933e9518957b3", version = "0.30.1" }
-pallas-hardano = { git = "https://github.com/input-output-hk/catalyst-pallas.git", rev = "9b5183c8b90b90fe2cc319d986e933e9518957b3", version = "0.30.1" }
+pallas = { version = "0.30.1", git = "https://github.com/input-output-hk/catalyst-pallas.git", rev = "9b5183c8b90b90fe2cc319d986e933e9518957b3" }
+pallas-hardano = { version = "0.30.1", git = "https://github.com/input-output-hk/catalyst-pallas.git", rev = "9b5183c8b90b90fe2cc319d986e933e9518957b3" }
+pallas-crypto = { version = "0.30.1", git = "https://github.com/input-output-hk/catalyst-pallas.git", rev = "9b5183c8b90b90fe2cc319d986e933e9518957b3" }
+
+# cspell: words licence
+mithril-client = { version = "0.8.16", git = "https://github.com/input-output-hk/catalyst-mithril.git", branch = "fix/lgpl-licence", default-features = false, features = [
+    "full",
+    "num-integer-backend",
+] }
+
+c509-certificate = { version = "0.0.1", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "v0.0.1" }
 
 thiserror = "1.0.56"
-tokio = { version = "1.36.0", features = ["macros", "rt", "net", "rt-multi-thread"] }
+tokio = { version = "1.40.0", features = [
+    "macros",
+    "rt",
+    "net",
+    "rt-multi-thread",
+] }
 tracing = "0.1.40"
+tracing-log = "0.2.0"
+dashmap = "6.0.1"
+url = "2.5.0"
+anyhow = "1.0.86"
+chrono = "0.4.38"
+async-trait = "0.1.82"
+dirs = "5.0.1"
+futures = "0.3.30"
+humantime = "2.1.0"
+crossbeam-skiplist = "0.1.3"
+crossbeam-channel = "0.5.13"
+crossbeam-epoch = "0.9.18"
+strum = "0.26.3"
+ouroboros = "0.18.4"
+hex = "0.4.3"
+rayon = "1.10.0"
+serde = "1.0.209"
+serde_json = "1.0.128"
+mimalloc = { version = "0.1.43", optional = true }
+memx = "0.1.32"
+fmmap = { version = "0.3.3", features = ["sync", "tokio-async"] }
+minicbor = { version = "0.24.4", features = ["alloc", "derive", "half"] }
+brotli = "6.0.0"
+zstd = "0.13.2"
+x509-cert = "0.2.5"
+ed25519-dalek = "2.1.1"
+blake2b_simd = "1.0.2"
+num-traits = "0.2.19"
+logcall = "0.1.9"
+tar = "0.4.41"
+ureq = { version = "2.10.1", features = ["native-certs"] }
+http = "1.1.0"
+hickory-resolver = { version = "0.24.1", features = ["dns-over-rustls"] }
+moka = { version = "0.12.8", features = ["sync"] }
 
 [dev-dependencies]
 hex = "0.4.3"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+test-log = { version = "0.2.16", default-features = false, features = [
+    "trace",
+] }
+clap = "4.5.17"
+
+# Note, these features are for support of features exposed by dependencies.
+[features]
+default = ["rustls-tls-native-roots"]
+
+# Enable the MiMalloc global allocator
+# Only used for examples.
+mimalloc = ["dep:mimalloc"]
+
+# These features are for support of dependent crates only.
+# They do not change the operation of the main crate.
+# cspell: words alpn
+native-tls = ["mithril-client/native-tls"]
+native-tls-alpn = ["mithril-client/native-tls-alpn"]
+native-tls-vendored = ["mithril-client/native-tls-vendored"]
+
+rustls-tls = ["mithril-client/rustls-tls"]
+rustls-tls-manual-roots = ["mithril-client/rustls-tls-manual-roots"]
+rustls-tls-webpki-roots = ["mithril-client/rustls-tls-webpki-roots"]
+rustls-tls-native-roots = ["mithril-client/rustls-tls-native-roots"]

rust/cardano-chain-follower/Readme.md

@@ -0,0 +1,59 @@
+# Things that need fixing
+
+The following fixes would be nice to have to:
+
+1. Improve sync times.
+2. Decrease disk utilization.
+3. Eliminate external dependencies.
+
+## Parallel downloading requires external tool
+
+We currently require an external tool `aria2c` to download the Mithril snapshot.
+We should have a native version to remove this external tool dependency.
+
+See: <https://ochagavia.nl/blog/download-accelerator-async-rust-edition/>
+For a simple version of such we could adapt.
+
+The first version should just replace `Aria2c` and download to a file.
+
+Ideally, we would have an in-memory queue that downloads in parallel, rather than saving to disk.
+This would need to use something like a skip-map to re-order the blocks, and a pool of workers to download the next blocks.
+It's not trivial, but it would remove the necessity to store the actual snapshot archive on-disk.
+
+It's not possible to download the snapshot archive to ram because it is enormous.
+
+## Zstd decompress and tar extraction optimization
+
+Currently, an async zstd decompress and tar extraction is used.
+This is known to be slow, and we are CPU bound doing it.
+
+Change this to run in a Thread outside async and use the zstd library, which links to the C zstd library directly.
+And the non async tar extraction library.  
+
+This will speed up extracting files from the archive.
+
+This would be better also if we had synchronous piped downloading as mentioned above.
+
+## Block Decode Optimization
+
+Currently, to enforce and validate chain integrity, we need to decode the blocks all over the place.
+Decoding blocks is expensive, and this is wasteful.
+As the application will almost certainly require the block to be decoded, it makes sense for it to happen once in a uniform way.  
+We would then pass the decoded block to the application saving it the effort of doing it, itself.
+
+We should Decode LIVE blocks once when we receive them from the network,
+and then keep the decoded as well as raw block data in memory.
+
+For Immutable blocks, we should decode them ONCE when we read them from disk.
+
+## Immutable Queue Optimization
+
+The Immutable follower reads from disk, inline.
+Disk IO is relatively expensive.
+Decoding blocks is also expensive, it's better to do that in parallel with an application processing a previous block.
+
+What we should do is have a read ahead queue, where a second task is reading ahead of the application following,
+reading the next blocks from disk, and decoding them.
+
+The main follower used by the application then reads from this red ahead queue.
+This would help us better utilize disk and CPU resources, which would result in improved sync times.