test: add benchmarks to compare stable memory vs btreemap performance

.github/workflows/ci.yml

@@ -88,13 +88,16 @@ jobs:
           - btreemap
           - memory-manager
           - vec
+          - compare
         include:
           - name: btreemap
             project_dir: ./benchmarks/btreemap
           - name: memory-manager
             project_dir: ./benchmarks/memory_manager
           - name: vec
             project_dir: ./benchmarks/vec
+          - name: compare
+            project_dir: ./benchmarks/compare
 
     env:
       PROJECT_DIR: ${{ matrix.project_dir }}

benchmarks/Cargo.toml

@@ -29,3 +29,7 @@ path = "memory_manager/src/main.rs"
 [[bin]]
 name = "vec"
 path = "vec/src/main.rs"
+
+[[bin]]
+name = "compare"
+path = "compare/src/main.rs"

benchmarks/compare/canbench.yml

@@ -0,0 +1,3 @@
+build_cmd: cargo build -p benchmarks --release --target wasm32-unknown-unknown
+
+wasm_path: ../../target/wasm32-unknown-unknown/release/compare.wasm

benchmarks/compare/canbench_results.yml

@@ -0,0 +1,86 @@
+benches:
+  read_chunks_btreemap_1:
+    total:
+      calls: 1
+      instructions: 1220197642
+      heap_increase: 3233
+      stable_memory_increase: 1665
+    scopes: {}
+  read_chunks_btreemap_1k:
+    total:
+      calls: 1
+      instructions: 5418812624
+      heap_increase: 1604
+      stable_memory_increase: 1665
+    scopes: {}
+  read_chunks_btreemap_1m:
+    total:
+      calls: 1
+      instructions: 133684080756
+      heap_increase: 1892
+      stable_memory_increase: 3201
+    scopes: {}
+  read_chunks_stable_1:
+    total:
+      calls: 1
+      instructions: 812767514
+      heap_increase: 1601
+      stable_memory_increase: 1665
+    scopes: {}
+  read_chunks_stable_1k:
+    total:
+      calls: 1
+      instructions: 525926853
+      heap_increase: 1600
+      stable_memory_increase: 1665
+    scopes: {}
+  read_chunks_stable_1m:
+    total:
+      calls: 1
+      instructions: 1307625987
+      heap_increase: 1892
+      stable_memory_increase: 1665
+    scopes: {}
+  write_chunks_btreemap_1:
+    total:
+      calls: 1
+      instructions: 1070838059
+      heap_increase: 3233
+      stable_memory_increase: 1665
+    scopes: {}
+  write_chunks_btreemap_1k:
+    total:
+      calls: 1
+      instructions: 4918823602
+      heap_increase: 1604
+      stable_memory_increase: 1665
+    scopes: {}
+  write_chunks_btreemap_1m:
+    total:
+      calls: 1
+      instructions: 89917688426
+      heap_increase: 1892
+      stable_memory_increase: 3201
+    scopes: {}
+  write_chunks_stable_1:
+    total:
+      calls: 1
+      instructions: 418914609
+      heap_increase: 1601
+      stable_memory_increase: 1665
+    scopes: {}
+  write_chunks_stable_1k:
+    total:
+      calls: 1
+      instructions: 420017351
+      heap_increase: 1600
+      stable_memory_increase: 1665
+    scopes: {}
+  write_chunks_stable_1m:
+    total:
+      calls: 1
+      instructions: 1076987632
+      heap_increase: 1892
+      stable_memory_increase: 1665
+    scopes: {}
+version: 0.1.15

benchmarks/compare/src/main.rs

@@ -0,0 +1,105 @@
+use canbench_rs::{bench, bench_fn};
+use ic_cdk::api::stable::WASM_PAGE_SIZE_IN_BYTES;
+use ic_stable_structures::{
+    memory_manager::{MemoryId, MemoryManager},
+    BTreeMap, DefaultMemoryImpl, Memory,
+};
+
+const TOTAL_SIZE: usize = 100 * 1024 * 1024; // 100 MiB
+const K: usize = 1_000;
+const M: usize = 1_000_000;
+
+fn init_memory(id: u8) -> impl Memory {
+    MemoryManager::init(DefaultMemoryImpl::default()).get(MemoryId::new(id))
+}
+
+fn ensure_memory_size(memory: &impl Memory, size: usize) {
+    let required = size.div_ceil(WASM_PAGE_SIZE_IN_BYTES) as u64;
+    if memory.size() < required {
+        memory.grow(required - memory.size());
+    }
+}
+
+fn chunk_data(n: usize) -> Vec<Vec<u8>> {
+    let chunk_size = TOTAL_SIZE / n;
+    (0..n).map(|_| vec![37; chunk_size]).collect()
+}
+
+// Stable memory benchmarks
+
+fn write_chunks_stable(mem_id: u8, n: usize) {
+    let memory = init_memory(mem_id);
+    let chunks = chunk_data(n);
+    let chunk_size = TOTAL_SIZE / n;
+
+    bench_fn(|| {
+        ensure_memory_size(&memory, TOTAL_SIZE);
+        for (i, chunk) in chunks.iter().enumerate() {
+            memory.write((i * chunk_size) as u64, chunk);
+        }
+    });
+}
+
+fn read_chunks_stable(mem_id: u8, n: usize) {
+    write_chunks_stable(mem_id, n);
+    let memory = init_memory(mem_id);
+    let chunk_size = TOTAL_SIZE / n;
+    let mut buf = vec![0u8; chunk_size];
+
+    bench_fn(|| {
+        for i in 0..n {
+            memory.read((i * chunk_size) as u64, &mut buf);
+        }
+    });
+}
+
+// BTreeMap benchmarks
+
+fn write_chunks_btreemap(mem_id: u8, n: usize) {
+    let mut map = BTreeMap::init(init_memory(mem_id));
+    let chunks = chunk_data(n);
+
+    bench_fn(|| {
+        for (i, chunk) in chunks.into_iter().enumerate() {
+            map.insert(i as u32, chunk);
+        }
+    });
+}
+
+fn read_chunks_btreemap(mem_id: u8, n: usize) {
+    write_chunks_btreemap(mem_id, n);
+    let map: BTreeMap<_, Vec<u8>, _> = BTreeMap::init(init_memory(mem_id));
+    bench_fn(|| {
+        for i in 0..n {
+            let _ = map.get(&(i as u32));
+        }
+    });
+}
+
+// Macro to define a single benchmark function
+macro_rules! bench_case {
+    ($name:ident, $func:ident, $mem_id:expr, $n:expr) => {
+        #[bench]
+        fn $name() {
+            $func($mem_id, $n);
+        }
+    };
+}
+
+// Stable Memory benchmarks
+bench_case!(write_chunks_stable_1, write_chunks_stable, 10, 1);
+bench_case!(write_chunks_stable_1k, write_chunks_stable, 11, K);
+bench_case!(write_chunks_stable_1m, write_chunks_stable, 12, M);
+bench_case!(read_chunks_stable_1, read_chunks_stable, 20, 1);
+bench_case!(read_chunks_stable_1k, read_chunks_stable, 21, K);
+bench_case!(read_chunks_stable_1m, read_chunks_stable, 22, M);
+
+// BTreeMap benchmarks
+bench_case!(write_chunks_btreemap_1, write_chunks_btreemap, 30, 1);
+bench_case!(write_chunks_btreemap_1k, write_chunks_btreemap, 31, K);
+bench_case!(write_chunks_btreemap_1m, write_chunks_btreemap, 32, M);
+bench_case!(read_chunks_btreemap_1, read_chunks_btreemap, 40, 1);
+bench_case!(read_chunks_btreemap_1k, read_chunks_btreemap, 41, K);
+bench_case!(read_chunks_btreemap_1m, read_chunks_btreemap, 42, M);
+
+fn main() {}