From 5a0f777669e0d354c3a6e6e8b19a94f2a982f118 Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 09:31:21 +0200 Subject: [PATCH 01/15] refactor: use name reclaim_memory --- src/btreemap.rs | 10 +-- src/btreeset.rs | 10 +-- src/memory_manager.rs | 59 ++++++++----- .../bucket_release_btreemap_tests.rs | 38 ++++----- .../bucket_release_core_tests.rs | 84 +++++++++---------- .../bucket_release_vec_tests.rs | 34 ++++---- 6 files changed, 125 insertions(+), 110 deletions(-) diff --git a/src/btreemap.rs b/src/btreemap.rs index 7b079cb4..cdb6364e 100644 --- a/src/btreemap.rs +++ b/src/btreemap.rs @@ -710,14 +710,14 @@ where // TODO: In next major release (v1.0), rename this method to `clear` to follow // standard Rust collection naming conventions. /// - /// # Safety Note for Bucket Release - /// If using manual bucket release via `MemoryManager::release_virtual_memory_buckets()`: + /// # Safety Note for Memory Reclamation + /// If using manual memory reclamation via `MemoryManager::reclaim_memory()`: /// 1. **MANDATORY**: Drop this BTreeMap object first (let it go out of scope) - /// 2. Call `release_virtual_memory_buckets()` on the memory manager + /// 2. Call `reclaim_memory()` on the memory manager /// 3. Create new structures as needed /// - /// Using this BTreeMap after bucket release causes data corruption. - /// Note: You can still call `clear_new()` if you need to clear data without bucket release. + /// Using this BTreeMap after memory reclamation causes data corruption. + /// Note: You can still call `clear_new()` if you need to clear data without memory reclamation. pub fn clear_new(&mut self) { self.root_addr = NULL; self.length = 0; diff --git a/src/btreeset.rs b/src/btreeset.rs index f5b3edad..781c5531 100644 --- a/src/btreeset.rs +++ b/src/btreeset.rs @@ -353,14 +353,14 @@ where /// assert!(set.is_empty()); /// ``` /// - /// # Safety Note for Bucket Release - /// If using manual bucket release via `MemoryManager::release_virtual_memory_buckets()`: + /// # Safety Note for Memory Reclamation + /// If using manual memory reclamation via `MemoryManager::reclaim_memory()`: /// 1. **MANDATORY**: Drop this BTreeSet object first (let it go out of scope) - /// 2. Call `release_virtual_memory_buckets()` on the memory manager + /// 2. Call `reclaim_memory()` on the memory manager /// 3. Create new structures as needed /// - /// Using this BTreeSet after bucket release causes data corruption. - /// Note: You can still call `clear()` if you need to clear data without bucket release. + /// Using this BTreeSet after memory reclamation causes data corruption. + /// Note: You can still call `clear()` if you need to clear data without memory reclamation. pub fn clear(&mut self) { self.map.clear_new(); } diff --git a/src/memory_manager.rs b/src/memory_manager.rs index aa64f287..af751f17 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -130,11 +130,11 @@ const HEADER_RESERVED_BYTES: usize = 32; /// /// # Current Limitations & Safety Requirements /// -/// - **Manual bucket release** - call `release_virtual_memory_buckets()` after dropping structures -/// - **Structure invalidation** - original structures become invalid after bucket release -/// - **Mandatory drop** - you MUST drop original structures BEFORE releasing buckets +/// - **Manual memory reclamation** - call `reclaim_memory()` after dropping structures +/// - **Structure invalidation** - original structures become invalid after memory reclamation +/// - **Mandatory drop** - you MUST drop original structures BEFORE reclaiming memory /// - **No safety verification** - user discipline required to prevent data corruption -/// - **Incorrect usage** - using structures after bucket release causes data corruption +/// - **Incorrect usage** - using structures after memory reclamation causes data corruption pub struct MemoryManager { inner: Rc>>, } @@ -164,7 +164,7 @@ impl MemoryManager { } } - /// Releases buckets allocated to the specified virtual memory for reuse. + /// Reclaims memory allocated to the specified virtual memory for reuse. /// /// **CRITICAL SAFETY REQUIREMENT**: Before calling this method: /// 1. You MUST drop the original structure object first @@ -176,15 +176,15 @@ impl MemoryManager { /// **Correct Usage Pattern:** /// ```rust,ignore /// drop(map); // 1. MANDATORY: Drop the structure object first - /// let count = memory_manager.release_virtual_memory_buckets(memory_id); // 2. Release buckets + /// let pages_reclaimed = memory_manager.reclaim_memory(memory_id); // 2. Reclaim memory /// let new_map = BTreeMap::new(memory_manager.get(memory_id)); // 3. Create fresh structure /// ``` /// - /// **DANGER**: Using the original structure after bucket release causes data corruption. + /// **DANGER**: Using the original structure after memory reclamation causes data corruption. /// - /// Returns the number of buckets released and added to the free pool (0 if none allocated). - pub fn release_virtual_memory_buckets(&self, id: MemoryId) -> usize { - self.inner.borrow_mut().release_virtual_memory_buckets(id) + /// Returns the number of pages reclaimed and made available for reuse (0 if none allocated). + pub fn reclaim_memory(&self, id: MemoryId) -> u64 { + self.inner.borrow_mut().reclaim_memory(id) } /// Returns the underlying memory. @@ -439,7 +439,7 @@ impl MemoryManagerInner { old_size as i64 } - /// Releases buckets for the specified memory, marking them unallocated in stable storage + /// Reclaims memory for the specified virtual memory, marking buckets unallocated in stable storage /// and adding them to the free pool. Resets memory size to 0. /// /// **CRITICAL**: This invalidates any data structures using this memory ID. @@ -447,8 +447,8 @@ impl MemoryManagerInner { /// 1. Original structure object is dropped BEFORE calling this /// 2. New structure is created if memory ID needs to be reused /// - /// Returns the number of buckets released and added to the free pool (0 if none allocated). - fn release_virtual_memory_buckets(&mut self, id: MemoryId) -> usize { + /// Returns the number of pages reclaimed and made available for reuse (0 if none allocated). + fn reclaim_memory(&mut self, id: MemoryId) -> u64 { let memory_buckets = &mut self.memory_buckets[id.0 as usize]; let bucket_count = memory_buckets.len(); @@ -481,7 +481,8 @@ impl MemoryManagerInner { self.save_header(); - bucket_count + // Return pages reclaimed (bucket count * pages per bucket) + bucket_count as u64 * self.bucket_size_in_pages as u64 } fn write(&self, id: MemoryId, offset: u64, src: &[u8], bucket_cache: &BucketCache) { @@ -1257,16 +1258,20 @@ mod test { let size_after_allocation = mem.size(); // Manually release first memory's buckets - let released_count = mem_mgr.release_virtual_memory_buckets(MemoryId::new(0)); - assert_eq!(released_count, 1, "Should release exactly 1 bucket"); + let pages_reclaimed = mem_mgr.reclaim_memory(MemoryId::new(0)); + assert_eq!( + pages_reclaimed, BUCKET_SIZE_IN_PAGES, + "Should reclaim exactly {} pages", + BUCKET_SIZE_IN_PAGES + ); - // CRITICAL: Drop the memory_0 after bucket release as it's now invalid + // CRITICAL: Drop the memory_0 after memory reclamation as it's now invalid drop(memory_0); // Verify memory size didn't change (buckets marked free, not deallocated) assert_eq!(mem.size(), size_after_allocation); - // Create second memory AFTER manual release - should reuse bucket 0 + // Create second memory AFTER manual memory reclamation - should reuse bucket 0 let memory_1 = mem_mgr.get(MemoryId::new(1)); memory_1.grow(BUCKET_SIZE_IN_PAGES); @@ -1305,10 +1310,20 @@ mod test { memory_1.write(BUCKET_SIZE_IN_PAGES * WASM_PAGE_SIZE, b"bucket_id_4"); // Release all buckets in reverse order to test sorting - let released_1 = mem_mgr.release_virtual_memory_buckets(MemoryId::new(1)); // releases 3, 4 - let released_0 = mem_mgr.release_virtual_memory_buckets(MemoryId::new(0)); // releases 0, 1, 2 - assert_eq!(released_1, 2, "Should release 2 buckets from memory 1"); - assert_eq!(released_0, 3, "Should release 3 buckets from memory 0"); + let pages_reclaimed_1 = mem_mgr.reclaim_memory(MemoryId::new(1)); // releases 3, 4 + let pages_reclaimed_0 = mem_mgr.reclaim_memory(MemoryId::new(0)); // releases 0, 1, 2 + assert_eq!( + pages_reclaimed_1, + BUCKET_SIZE_IN_PAGES * 2, + "Should reclaim {} pages from memory 1", + BUCKET_SIZE_IN_PAGES * 2 + ); + assert_eq!( + pages_reclaimed_0, + BUCKET_SIZE_IN_PAGES * 3, + "Should reclaim {} pages from memory 0", + BUCKET_SIZE_IN_PAGES * 3 + ); // Allocate new memories - should reuse buckets in increasing order (0, 1, then 2) let memory_2 = mem_mgr.get(MemoryId::new(2)); diff --git a/src/memory_manager/bucket_release_btreemap_tests.rs b/src/memory_manager/bucket_release_btreemap_tests.rs index db965d19..4fdd34ee 100644 --- a/src/memory_manager/bucket_release_btreemap_tests.rs +++ b/src/memory_manager/bucket_release_btreemap_tests.rs @@ -1,13 +1,13 @@ -//! Migration scenario tests for BTreeMap with bucket release. +//! Migration scenario tests for BTreeMap with memory reclamation. //! //! These tests demonstrate real-world migration patterns where users move data -//! from one structure to another. They show how bucket release prevents memory +//! from one structure to another. They show how memory reclamation prevents memory //! waste during common migration scenarios, and most importantly, demonstrate //! the data corruption bug and its safe usage solution. //! //! **CRITICAL SAFETY REQUIREMENTS**: -//! All bucket release operations require mandatory Rust object drop BEFORE release. -//! Using original data structures after bucket release causes data corruption. +//! All memory reclamation operations require mandatory Rust object drop BEFORE reclamation. +//! Using original data structures after memory reclamation causes data corruption. //! See MemoryManager documentation for proper usage patterns. use super::{MemoryId, MemoryManager}; @@ -24,8 +24,8 @@ fn large_value(id: u64) -> Vec { } #[test] -fn migration_without_release_wastes_buckets() { - // Scenario: Populate A → Drop A without bucket release → Populate B +fn migration_without_reclaim_wastes_buckets() { + // Scenario: Populate A → Drop A without memory reclamation → Populate B // Result: B cannot reuse A's buckets, causing memory waste (growth) let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); @@ -69,15 +69,15 @@ fn migration_with_release_reuses_buckets() { } assert_eq!(a_map.len(), 50); - // MANDATORY: Drop the Rust object before releasing buckets + // MANDATORY: Drop the Rust object before reclaiming memory drop(a_map); - // Release the buckets after dropping the object - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0); + // Reclaim the memory after dropping the object + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0); let stable_before = mock_stable_memory.size(); - // Allocate in B → should reuse A's released buckets + // Allocate in B → should reuse A's reclaimed buckets let mut b_map = BTreeMap::init(mm.get(b)); for i in 0u64..50 { b_map.insert(i, large_value(i + 100)); @@ -104,10 +104,10 @@ fn data_corruption_without_mandatory_drop() { map_a.insert(1u64, 100u64); assert_eq!(map_a.get(&1u64).unwrap(), 100u64); - // DANGEROUS: Release buckets but keep map_a alive - mm.release_virtual_memory_buckets(a); + // DANGEROUS: Reclaim memory but keep map_a alive + mm.reclaim_memory(a); - // Create BTreeMap B - reuses A's released buckets + // Create BTreeMap B - reuses A's reclaimed buckets let mut map_b = BTreeMap::init(mm.get(b)); map_b.insert(2u64, 200u64); assert_eq!(map_b.get(&2u64).unwrap(), 200u64); @@ -159,14 +159,14 @@ fn safe_usage_with_mandatory_drop() { map_a.insert(1u64, 100u64); assert_eq!(map_a.get(&1u64).unwrap(), 100u64); - // MANDATORY: Drop the Rust object before releasing buckets + // MANDATORY: Drop the Rust object before reclaiming memory drop(map_a); - // Release the buckets after dropping the object - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0); + // Reclaim the memory after dropping the object + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0); - // Create BTreeMap B - safely reuses A's released buckets + // Create BTreeMap B - safely reuses A's reclaimed buckets let mut map_b = BTreeMap::init(mm.get(b)); map_b.insert(2u64, 200u64); assert_eq!(map_b.get(&2u64).unwrap(), 200u64); diff --git a/src/memory_manager/bucket_release_core_tests.rs b/src/memory_manager/bucket_release_core_tests.rs index 8e436cf0..06988ff1 100644 --- a/src/memory_manager/bucket_release_core_tests.rs +++ b/src/memory_manager/bucket_release_core_tests.rs @@ -1,12 +1,12 @@ -//! Core bucket release functionality tests for MemoryManager. +//! Core memory reclamation functionality tests for MemoryManager. //! //! These tests verify the basic memory management operations without dependency //! on specific data structures. They test the fundamental bucket allocation, -//! release, and reuse mechanisms. +//! reclamation, and reuse mechanisms. //! //! **CRITICAL SAFETY REQUIREMENTS**: -//! All bucket release operations require mandatory Rust object drop BEFORE release. -//! Using original data structures after bucket release causes data corruption. +//! All memory reclamation operations require mandatory Rust object drop BEFORE reclamation. +//! Using original data structures after memory reclamation causes data corruption. //! See MemoryManager documentation for proper usage patterns. use super::{MemoryId, MemoryManager}; @@ -22,26 +22,26 @@ fn allocate_buckets_via_btreemap( for i in 0..count { map.insert(i, vec![42u8; 2000]); // 2KB blob to trigger bucket allocation } - drop(map); // Drop the structure before bucket release + drop(map); // Drop the structure before memory reclamation } #[test] -fn release_empty_memory_returns_zero() { - // Test: Releasing buckets from unused memory should return 0 +fn reclaim_empty_memory_returns_zero() { + // Test: Reclaiming memory from unused memory should return 0 let memory_id = MemoryId::new(0); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory); - // Try to release buckets from empty memory - let released_buckets = mm.release_virtual_memory_buckets(memory_id); + // Try to reclaim memory from empty memory + let pages_reclaimed = mm.reclaim_memory(memory_id); - // Should return 0 since no buckets were allocated - assert_eq!(released_buckets, 0, "Empty memory should release 0 buckets"); + // Should return 0 since no pages were allocated + assert_eq!(pages_reclaimed, 0, "Empty memory should reclaim 0 pages"); } #[test] -fn double_release_returns_zero_second_time() { - // Test: Releasing the same memory twice should return 0 on second attempt +fn double_reclaim_returns_zero_second_time() { + // Test: Reclaiming the same memory twice should return 0 on second attempt let memory_id = MemoryId::new(0); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory); @@ -49,18 +49,18 @@ fn double_release_returns_zero_second_time() { // Allocate buckets and clear structure allocate_buckets_via_btreemap(&mm, memory_id, 50); - // Release buckets - let first_release = mm.release_virtual_memory_buckets(memory_id); - assert!(first_release > 0, "First release should return >0 buckets"); + // First release should return non-zero + let first_release = mm.reclaim_memory(memory_id); + assert!(first_release > 0, "First reclaim should return pages > 0"); - // Try to release again - let second_release = mm.release_virtual_memory_buckets(memory_id); - assert_eq!(second_release, 0, "Second release should return 0 buckets"); + // Second release should return 0 (nothing left to release) + let second_release = mm.reclaim_memory(memory_id); + assert_eq!(second_release, 0, "Second reclaim should return 0 pages"); } #[test] -fn release_only_affects_target_memory() { - // Test: Releasing memory A should not affect memory B's buckets +fn reclaim_only_affects_target_memory() { + // Test: Reclaiming memory A should not affect memory B's buckets let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory); @@ -69,45 +69,45 @@ fn release_only_affects_target_memory() { allocate_buckets_via_btreemap(&mm, a, 30); allocate_buckets_via_btreemap(&mm, b, 30); - // Release A's buckets - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0, "Should release A's buckets"); + // Reclaim A's memory + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0, "Should reclaim A's pages"); - // Verify B still has its buckets (can't be released again) - let b_release_attempt = mm.release_virtual_memory_buckets(b); + // Verify B still has its memory (can't be reclaimed again) + let b_reclaim_attempt = mm.reclaim_memory(b); assert!( - b_release_attempt > 0, - "B should still have releasable buckets" + b_reclaim_attempt > 0, + "B should still have reclaimable pages" ); } #[test] -fn multiple_release_cycles() { - // Test: Multiple allocate→release cycles should work consistently +fn multiple_reclaim_cycles() { + // Test: Multiple allocate→reclaim cycles should work consistently let memory_id = MemoryId::new(0); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory); - // Perform several cycles of allocate→release + // Perform several cycles of allocate→reclaim for cycle in 0..5 { // Allocate buckets allocate_buckets_via_btreemap(&mm, memory_id, 40); - // Release buckets - let released_buckets = mm.release_virtual_memory_buckets(memory_id); + // Reclaim memory + let pages_reclaimed = mm.reclaim_memory(memory_id); assert!( - released_buckets > 0, - "Cycle {} should release >0 buckets, got {}", + pages_reclaimed > 0, + "Cycle {} should reclaim >0 pages, got {}", cycle, - released_buckets + pages_reclaimed ); } } #[test] fn bucket_reuse_prevents_memory_growth() { - // Test: Released buckets are actually reused by subsequent allocations + // Test: Reclaimed buckets are actually reused by subsequent allocations let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory.clone()); @@ -115,18 +115,18 @@ fn bucket_reuse_prevents_memory_growth() { // Allocate buckets in A allocate_buckets_via_btreemap(&mm, a, 50); - // Release A's buckets - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0); + // Reclaim A's memory + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0); let stable_before = mock_stable_memory.size(); - // Allocate same amount in B → should reuse A's released buckets + // Allocate same amount in B → should reuse A's reclaimed buckets allocate_buckets_via_btreemap(&mm, b, 50); let stable_after = mock_stable_memory.size(); // Verify: stable memory should not grow significantly (reuse occurred) assert!( stable_after <= stable_before + 1, // Allow minimal growth for structure overhead - "Stable memory should not grow when reusing released buckets" + "Stable memory should not grow when reusing reclaimed buckets" ); } diff --git a/src/memory_manager/bucket_release_vec_tests.rs b/src/memory_manager/bucket_release_vec_tests.rs index f90daa0f..51a539aa 100644 --- a/src/memory_manager/bucket_release_vec_tests.rs +++ b/src/memory_manager/bucket_release_vec_tests.rs @@ -1,17 +1,17 @@ -//! Migration scenario tests for Vec with bucket release. +//! Migration scenario tests for Vec with memory reclamation. //! //! These tests demonstrate real-world migration patterns where users move data -//! from one structure to another. They show how bucket release prevents memory +//! from one structure to another. They show how memory reclamation prevents memory //! waste during common migration scenarios, and most importantly, demonstrate //! the data corruption bug and its safe usage solution for Vec structures. //! //! **CRITICAL SAFETY REQUIREMENTS**: -//! All bucket release operations require mandatory Rust object drop BEFORE release. -//! Using original data structures after bucket release causes data corruption. +//! All memory reclamation operations require mandatory Rust object drop BEFORE reclamation. +//! Using original data structures after memory reclamation causes data corruption. //! See MemoryManager documentation for proper usage patterns. //! //! **Vec-Specific Notes**: -//! Vec doesn't have a clear() method. Since we drop the object before bucket release, +//! Vec doesn't have a clear() method. Since we drop the object before memory reclamation, //! there's no need to clear the data first. use super::{MemoryId, MemoryManager}; @@ -75,15 +75,15 @@ fn migration_with_release_reuses_buckets() { } assert_eq!(vec_a_original.len(), 50); - // MANDATORY: Drop the Rust object before releasing buckets + // MANDATORY: Drop the Rust object before reclaiming memory drop(vec_a_original); - // Release the buckets after dropping the object - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0); + // Reclaim the memory after dropping the object + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0); let stable_before = mock_stable_memory.size(); - // Allocate in B → should reuse A's released buckets + // Allocate in B → should reuse A's reclaimed buckets let vec_b = StableVec::init(mm.get(b)); for i in 0u64..50 { vec_b.push(&large_data(i + 100)); @@ -113,9 +113,9 @@ fn data_corruption_without_mandatory_drop() { // "Clear" by creating new instance, but keep vec_a alive (DANGEROUS!) let vec_a: StableVec = StableVec::new(mm.get(a)); assert_eq!(vec_a.len(), 0); - mm.release_virtual_memory_buckets(a); + mm.reclaim_memory(a); - // Create Vec B - reuses A's released buckets + // Create Vec B - reuses A's reclaimed buckets let vec_b = StableVec::init(mm.get(b)); vec_b.push(&2u64); assert_eq!(vec_b.get(0).unwrap(), 2u64); @@ -167,14 +167,14 @@ fn safe_usage_with_mandatory_drop() { vec_a.push(&1u64); assert_eq!(vec_a.get(0).unwrap(), 1u64); - // MANDATORY: Drop the Rust object before releasing buckets + // MANDATORY: Drop the Rust object before reclaiming memory drop(vec_a); - // Release the buckets after dropping the object - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0); + // Reclaim the memory after dropping the object + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0); - // Create Vec B - safely reuses A's released buckets + // Create Vec B - safely reuses A's reclaimed buckets let vec_b = StableVec::init(mm.get(b)); vec_b.push(&2u64); assert_eq!(vec_b.get(0).unwrap(), 2u64); From baa0cd17d3ceb99e23a562e187e9cc6131fda4f9 Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 09:40:05 +0200 Subject: [PATCH 02/15] release -> reclaim --- src/memory_manager.rs | 22 +++++++++---------- .../bucket_release_btreemap_tests.rs | 18 +++++++-------- .../bucket_release_core_tests.rs | 12 +++++----- .../bucket_release_vec_tests.rs | 22 +++++++++---------- 4 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/memory_manager.rs b/src/memory_manager.rs index af751f17..d7dbfb7b 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -457,18 +457,18 @@ impl MemoryManagerInner { } // Mark all buckets as unallocated in stable storage and collect them - let mut released_buckets = Vec::new(); + let mut reclaimed_buckets = Vec::new(); for &bucket_id in memory_buckets.iter() { write( &self.memory, bucket_allocations_address(bucket_id).get(), &[UNALLOCATED_BUCKET_MARKER], ); - released_buckets.push(bucket_id); + reclaimed_buckets.push(bucket_id); } - // Add released buckets to free pool and sort to maintain increasing order - self.free_buckets.extend(released_buckets); + // Add reclaimed buckets to free pool and sort to maintain increasing order + self.free_buckets.extend(reclaimed_buckets); let mut sorted_buckets: Vec<_> = self.free_buckets.drain(..).collect(); sorted_buckets.sort_by_key(|bucket| bucket.0); self.free_buckets = VecDeque::from(sorted_buckets); @@ -1210,7 +1210,7 @@ mod test { } #[test] - fn memory_grows_without_manual_release() { + fn memory_grows_without_manual_reclaim() { let mem = make_memory(); let mem_mgr = MemoryManager::init(mem.clone()); let initial_size = mem.size(); @@ -1224,7 +1224,7 @@ mod test { // Verify first allocation grew memory assert_eq!(size_after_first, initial_size + BUCKET_SIZE_IN_PAGES); - // Create second memory WITHOUT releasing first - should allocate bucket 1 + // Create second memory WITHOUT reclaiming first - should allocate bucket 1 let memory_1 = mem_mgr.get(MemoryId::new(1)); memory_1.grow(BUCKET_SIZE_IN_PAGES); memory_1.write(0, b"bucket_id_1"); @@ -1247,7 +1247,7 @@ mod test { } #[test] - fn memory_reuses_buckets_with_manual_release() { + fn memory_reuses_buckets_with_manual_reclaim() { let mem = make_memory(); let mem_mgr = MemoryManager::init(mem.clone()); @@ -1257,7 +1257,7 @@ mod test { memory_0.write(0, b"bucket_id_0"); let size_after_allocation = mem.size(); - // Manually release first memory's buckets + // Manually reclaim first memory let pages_reclaimed = mem_mgr.reclaim_memory(MemoryId::new(0)); assert_eq!( pages_reclaimed, BUCKET_SIZE_IN_PAGES, @@ -1309,9 +1309,9 @@ mod test { memory_1.write(0, b"bucket_id_3"); memory_1.write(BUCKET_SIZE_IN_PAGES * WASM_PAGE_SIZE, b"bucket_id_4"); - // Release all buckets in reverse order to test sorting - let pages_reclaimed_1 = mem_mgr.reclaim_memory(MemoryId::new(1)); // releases 3, 4 - let pages_reclaimed_0 = mem_mgr.reclaim_memory(MemoryId::new(0)); // releases 0, 1, 2 + // Reclaim all buckets in reverse order to test sorting + let pages_reclaimed_1 = mem_mgr.reclaim_memory(MemoryId::new(1)); // reclaims 3, 4 + let pages_reclaimed_0 = mem_mgr.reclaim_memory(MemoryId::new(0)); // reclaims 0, 1, 2 assert_eq!( pages_reclaimed_1, BUCKET_SIZE_IN_PAGES * 2, diff --git a/src/memory_manager/bucket_release_btreemap_tests.rs b/src/memory_manager/bucket_release_btreemap_tests.rs index 4fdd34ee..ff28d4d2 100644 --- a/src/memory_manager/bucket_release_btreemap_tests.rs +++ b/src/memory_manager/bucket_release_btreemap_tests.rs @@ -42,7 +42,7 @@ fn migration_without_reclaim_wastes_buckets() { drop(a_map); let stable_before = mock_stable_memory.size(); - // Allocate in B → should need new buckets since A's aren't released + // Allocate in B → should need new buckets since A's aren't reclaimed let mut b_map = BTreeMap::init(mm.get(b)); for i in 0u64..50 { b_map.insert(i, large_value(i + 100)); @@ -55,9 +55,9 @@ fn migration_without_reclaim_wastes_buckets() { } #[test] -fn migration_with_release_reuses_buckets() { - // Scenario: Populate A → Drop A with bucket release → Populate B - // Result: B reuses A's released buckets, preventing memory waste (no growth) +fn migration_with_reclaim_reuses_buckets() { + // Scenario: Populate A → Drop A with memory reclamation → Populate B + // Result: B reuses A's reclaimed buckets, preventing memory waste (no growth) let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory.clone()); @@ -92,8 +92,8 @@ fn migration_with_release_reuses_buckets() { ); } -/// **DANGER**: This test demonstrates data corruption from unsafe bucket release usage. -/// This shows what happens when you DON'T drop the original object before bucket release. +/// **DANGER**: This test demonstrates data corruption from unsafe memory reclamation usage. +/// This shows what happens when you DON'T drop the original object before memory reclamation. #[test] fn data_corruption_without_mandatory_drop() { let (a, b) = (MemoryId::new(0), MemoryId::new(1)); @@ -135,7 +135,7 @@ fn data_corruption_without_mandatory_drop() { Ok(_) => { // If it succeeds, check if map_b can see the new data (shared allocation) if map_b.get(&3u64).is_some() { - println!("CORRUPTION: Both maps operating on the same released memory space"); + println!("CORRUPTION: Both maps operating on the same reclaimed memory space"); } } Err(_) => { @@ -144,10 +144,10 @@ fn data_corruption_without_mandatory_drop() { } } - // This test proves why objects MUST be dropped before bucket release + // This test proves why objects MUST be dropped before memory reclamation } -/// **SAFE**: This test demonstrates the correct way to use bucket release. +/// **SAFE**: This test demonstrates the correct way to use memory reclamation. /// This shows how mandatory drop prevents data corruption. #[test] fn safe_usage_with_mandatory_drop() { diff --git a/src/memory_manager/bucket_release_core_tests.rs b/src/memory_manager/bucket_release_core_tests.rs index 06988ff1..c5088e28 100644 --- a/src/memory_manager/bucket_release_core_tests.rs +++ b/src/memory_manager/bucket_release_core_tests.rs @@ -49,13 +49,13 @@ fn double_reclaim_returns_zero_second_time() { // Allocate buckets and clear structure allocate_buckets_via_btreemap(&mm, memory_id, 50); - // First release should return non-zero - let first_release = mm.reclaim_memory(memory_id); - assert!(first_release > 0, "First reclaim should return pages > 0"); + // First reclamation should return non-zero + let first_reclaim = mm.reclaim_memory(memory_id); + assert!(first_reclaim > 0, "First reclaim should return pages > 0"); - // Second release should return 0 (nothing left to release) - let second_release = mm.reclaim_memory(memory_id); - assert_eq!(second_release, 0, "Second reclaim should return 0 pages"); + // Second reclamation should return 0 (nothing left to reclaim) + let second_reclaim = mm.reclaim_memory(memory_id); + assert_eq!(second_reclaim, 0, "Second reclaim should return 0 pages"); } #[test] diff --git a/src/memory_manager/bucket_release_vec_tests.rs b/src/memory_manager/bucket_release_vec_tests.rs index 51a539aa..d5bd00b4 100644 --- a/src/memory_manager/bucket_release_vec_tests.rs +++ b/src/memory_manager/bucket_release_vec_tests.rs @@ -28,8 +28,8 @@ fn large_data(id: u64) -> [u8; 1024] { } #[test] -fn migration_without_release_wastes_buckets() { - // Scenario: Populate A → Clear A without bucket release → Populate B +fn migration_without_reclaim_wastes_buckets() { + // Scenario: Populate A → Clear A without memory reclamation → Populate B // Result: B cannot reuse A's buckets, causing memory waste (growth) let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); @@ -47,7 +47,7 @@ fn migration_without_release_wastes_buckets() { assert_eq!(vec_a.len(), 0); let stable_before = mock_stable_memory.size(); - // Allocate in B → should need new buckets since A's aren't released + // Allocate in B → should need new buckets since A's aren't reclaimed let vec_b = StableVec::init(mm.get(b)); for i in 0u64..50 { vec_b.push(&large_data(i + 100)); @@ -61,9 +61,9 @@ fn migration_without_release_wastes_buckets() { } #[test] -fn migration_with_release_reuses_buckets() { - // Scenario: Populate A → Clear A with bucket release → Populate B - // Result: B reuses A's released buckets, preventing memory waste (no growth) +fn migration_with_reclaim_reuses_buckets() { + // Scenario: Populate A → Clear A with memory reclamation → Populate B + // Result: B reuses A's reclaimed buckets, preventing memory waste (no growth) let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory.clone()); @@ -98,8 +98,8 @@ fn migration_with_release_reuses_buckets() { ); } -/// **DANGER**: This test demonstrates data corruption from unsafe bucket release usage. -/// This shows what happens when you DON'T drop the original object after bucket release. +/// **DANGER**: This test demonstrates data corruption from unsafe memory reclamation usage. +/// This shows what happens when you DON'T drop the original object after memory reclamation. #[test] fn data_corruption_without_mandatory_drop() { let (a, b) = (MemoryId::new(0), MemoryId::new(1)); @@ -143,7 +143,7 @@ fn data_corruption_without_mandatory_drop() { // If it succeeds, check if vec_b can see the new data (shared allocation) if !vec_a.is_empty() && vec_b.len() > vec_a.len() { // Check if data appears in unexpected places - println!("CORRUPTION: Both vecs operating on the same released memory space"); + println!("CORRUPTION: Both vecs operating on the same reclaimed memory space"); } } Err(_) => { @@ -152,10 +152,10 @@ fn data_corruption_without_mandatory_drop() { } } - // This test proves why objects MUST be dropped after bucket release + // This test proves why objects MUST be dropped after memory reclamation } -/// **SAFE**: This test demonstrates the correct way to use bucket release. +/// **SAFE**: This test demonstrates the correct way to use memory reclamation. /// This shows how mandatory drop prevents data corruption. #[test] fn safe_usage_with_mandatory_drop() { From 2021633521ca2159ed5c2e48194cafeaa07e263f Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 09:42:46 +0200 Subject: [PATCH 03/15] . --- ...release_btreemap_tests.rs => memory_reclaim_btreemap_tests.rs} | 0 ...{bucket_release_core_tests.rs => memory_reclaim_core_tests.rs} | 0 .../{bucket_release_vec_tests.rs => memory_reclaim_vec_tests.rs} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename src/memory_manager/{bucket_release_btreemap_tests.rs => memory_reclaim_btreemap_tests.rs} (100%) rename src/memory_manager/{bucket_release_core_tests.rs => memory_reclaim_core_tests.rs} (100%) rename src/memory_manager/{bucket_release_vec_tests.rs => memory_reclaim_vec_tests.rs} (100%) diff --git a/src/memory_manager/bucket_release_btreemap_tests.rs b/src/memory_manager/memory_reclaim_btreemap_tests.rs similarity index 100% rename from src/memory_manager/bucket_release_btreemap_tests.rs rename to src/memory_manager/memory_reclaim_btreemap_tests.rs diff --git a/src/memory_manager/bucket_release_core_tests.rs b/src/memory_manager/memory_reclaim_core_tests.rs similarity index 100% rename from src/memory_manager/bucket_release_core_tests.rs rename to src/memory_manager/memory_reclaim_core_tests.rs diff --git a/src/memory_manager/bucket_release_vec_tests.rs b/src/memory_manager/memory_reclaim_vec_tests.rs similarity index 100% rename from src/memory_manager/bucket_release_vec_tests.rs rename to src/memory_manager/memory_reclaim_vec_tests.rs From 849e1273c0b20ec3ce23608608d5ead593474191 Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:02:12 +0200 Subject: [PATCH 04/15] cleanup --- README.md | 36 ++++++++++++++++++------------------ src/memory_manager.rs | 33 +++++++++++++-------------------- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 79ca7ad5..fda018b5 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,10 @@ Here are some basic examples: ```rust use ic_stable_structures::{BTreeMap, DefaultMemoryImpl}; -let mut map: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); +let mut map: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); -map.insert(1, 2); -assert_eq!(map.get(&1), Some(2)); +map.insert(1, "hello".to_string()); +assert_eq!(map.get(&1), Some("hello".to_string())); ``` Memories are abstracted with the [Memory] trait, and stable structures can work with any storage @@ -77,17 +77,17 @@ Note that **stable structures cannot share memories.** Each memory must belong to only one stable structure. For example, this fails when run in a canister: -```no_run +```rust,ignore use ic_stable_structures::{BTreeMap, DefaultMemoryImpl}; -let mut map_1: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); -let mut map_2: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); +let mut map_a: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); +let mut map_b: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); -map_1.insert(1, 2); -map_2.insert(1, 3); -assert_eq!(map_1.get(&1), Some(2)); // This assertion fails. +map_a.insert(1, "value_a".to_string()); +map_b.insert(1, "value_b".to_string()); +assert_eq!(map_a.get(&1), Some("value_a".to_string())); // This assertion fails. ``` -It fails because both `map_1` and `map_2` are using the same stable memory under the hood, and so changes in `map_1` end up changing or corrupting `map_2`. +It fails because both `map_a` and `map_b` are using the same stable memory under the hood, and so changes in `map_a` end up changing or corrupting `map_b`. To address this issue, we make use of the [MemoryManager](memory_manager::MemoryManager), which takes a single memory and creates up to 255 virtual memories for our disposal. Here's the above failing example, but fixed by using the [MemoryManager](memory_manager::MemoryManager): @@ -98,12 +98,12 @@ use ic_stable_structures::{ BTreeMap, DefaultMemoryImpl, }; let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); -let mut map_1: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); -let mut map_2: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); -map_1.insert(1, 2); -map_2.insert(1, 3); -assert_eq!(map_1.get(&1), Some(2)); // Succeeds, as expected. +map_a.insert(1, "value_a".to_string()); +map_b.insert(1, "value_b".to_string()); +assert_eq!(map_a.get(&1), Some("value_a".to_string())); // Succeeds, as expected. ``` ## Example Canister @@ -135,7 +135,7 @@ thread_local! { RefCell::new(MemoryManager::init(DefaultMemoryImpl::default())); // Initialize a `StableBTreeMap` with `MemoryId(0)`. - static MAP: RefCell> = RefCell::new( + static MAP: RefCell> = RefCell::new( StableBTreeMap::init( MEMORY_MANAGER.with(|m| m.borrow().get(MemoryId::new(0))), ) @@ -144,13 +144,13 @@ thread_local! { // Retrieves the value associated with the given key if it exists. #[ic_cdk_macros::query] -fn get(key: u128) -> Option { +fn get(key: u64) -> Option { MAP.with(|p| p.borrow().get(&key)) } // Inserts an entry into the map and returns the previous value of the key if it exists. #[ic_cdk_macros::update] -fn insert(key: u128, value: u128) -> Option { +fn insert(key: u64, value: String) -> Option { MAP.with(|p| p.borrow_mut().insert(key, value)) } ``` diff --git a/src/memory_manager.rs b/src/memory_manager.rs index d7dbfb7b..5d76705e 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -166,23 +166,18 @@ impl MemoryManager { /// Reclaims memory allocated to the specified virtual memory for reuse. /// - /// **CRITICAL SAFETY REQUIREMENT**: Before calling this method: - /// 1. You MUST drop the original structure object first + /// **CRITICAL SAFETY REQUIREMENT**: Drop the original structure object first! /// - /// **After calling this method**: - /// 2. The data structure using this memory ID becomes INVALID - /// 3. You MUST create a new structure if you want to reuse the memory ID - /// - /// **Correct Usage Pattern:** + /// **Usage Pattern:** /// ```rust,ignore - /// drop(map); // 1. MANDATORY: Drop the structure object first - /// let pages_reclaimed = memory_manager.reclaim_memory(memory_id); // 2. Reclaim memory - /// let new_map = BTreeMap::new(memory_manager.get(memory_id)); // 3. Create fresh structure + /// drop(map); // 1. Drop the structure object first + /// let pages = memory_manager.reclaim_memory(memory_id); // 2. Reclaim memory + /// let new_map = BTreeMap::new(memory_manager.get(memory_id)); // 3. Create new structure /// ``` /// - /// **DANGER**: Using the original structure after memory reclamation causes data corruption. + /// **DANGER**: Using the original structure after reclamation causes data corruption. /// - /// Returns the number of pages reclaimed and made available for reuse (0 if none allocated). + /// Returns the number of pages reclaimed (0 if none allocated). pub fn reclaim_memory(&self, id: MemoryId) -> u64 { self.inner.borrow_mut().reclaim_memory(id) } @@ -439,15 +434,13 @@ impl MemoryManagerInner { old_size as i64 } - /// Reclaims memory for the specified virtual memory, marking buckets unallocated in stable storage + /// Reclaims memory for the specified virtual memory, marking buckets unallocated /// and adding them to the free pool. Resets memory size to 0. /// /// **CRITICAL**: This invalidates any data structures using this memory ID. - /// Caller must ensure: - /// 1. Original structure object is dropped BEFORE calling this - /// 2. New structure is created if memory ID needs to be reused + /// Drop the original structure object BEFORE calling this method. /// - /// Returns the number of pages reclaimed and made available for reuse (0 if none allocated). + /// Returns the number of pages reclaimed (0 if none allocated). fn reclaim_memory(&mut self, id: MemoryId) -> u64 { let memory_buckets = &mut self.memory_buckets[id.0 as usize]; let bucket_count = memory_buckets.len(); @@ -1357,10 +1350,10 @@ mod test { } #[cfg(test)] -mod bucket_release_core_tests; +mod memory_reclaim_core_tests; #[cfg(test)] -mod bucket_release_btreemap_tests; +mod memory_reclaim_btreemap_tests; #[cfg(test)] -mod bucket_release_vec_tests; +mod memory_reclaim_vec_tests; From b7dea864d517cfa97eeb2fe23f5696bc137e53ea Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:06:49 +0200 Subject: [PATCH 05/15] . --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fda018b5..09c08e6f 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,8 @@ let mut map_b: BTreeMap = BTreeMap::init(DefaultMemoryImpl::defa map_a.insert(1, "value_a".to_string()); map_b.insert(1, "value_b".to_string()); -assert_eq!(map_a.get(&1), Some("value_a".to_string())); // This assertion fails. +assert_eq!(map_a.get(&1), Some("value_a".to_string())); // ❌ FAILS: Returns "value_b" due to shared memory! +assert_eq!(map_b.get(&1), Some("value_b".to_string())); // ✅ Succeeds, but corrupted map_a ``` It fails because both `map_a` and `map_b` are using the same stable memory under the hood, and so changes in `map_a` end up changing or corrupting `map_b`. @@ -103,7 +104,8 @@ let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::n map_a.insert(1, "value_a".to_string()); map_b.insert(1, "value_b".to_string()); -assert_eq!(map_a.get(&1), Some("value_a".to_string())); // Succeeds, as expected. +assert_eq!(map_a.get(&1), Some("value_a".to_string())); // ✅ Succeeds: Each map has its own memory +assert_eq!(map_b.get(&1), Some("value_b".to_string())); // ✅ Succeeds: No data corruption ``` ## Example Canister From d7cc41b4ed4dd526fe4d9ef6496d122cc6211ed9 Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:11:49 +0200 Subject: [PATCH 06/15] . --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 09c08e6f..26cab8ea 100644 --- a/README.md +++ b/README.md @@ -79,13 +79,13 @@ For example, this fails when run in a canister: ```rust,ignore use ic_stable_structures::{BTreeMap, DefaultMemoryImpl}; -let mut map_a: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); -let mut map_b: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); +let mut map_a: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); +let mut map_b: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); -map_a.insert(1, "value_a".to_string()); -map_b.insert(1, "value_b".to_string()); -assert_eq!(map_a.get(&1), Some("value_a".to_string())); // ❌ FAILS: Returns "value_b" due to shared memory! -assert_eq!(map_b.get(&1), Some("value_b".to_string())); // ✅ Succeeds, but corrupted map_a +map_a.insert(1, b'A'); +map_b.insert(1, b'B'); +assert_eq!(map_a.get(&1), Some(b'A')); // ❌ FAILS: Returns b'B' due to shared memory! +assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds, but corrupted map_a ``` It fails because both `map_a` and `map_b` are using the same stable memory under the hood, and so changes in `map_a` end up changing or corrupting `map_b`. @@ -99,13 +99,13 @@ use ic_stable_structures::{ BTreeMap, DefaultMemoryImpl, }; let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); -let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); -let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); -map_a.insert(1, "value_a".to_string()); -map_b.insert(1, "value_b".to_string()); -assert_eq!(map_a.get(&1), Some("value_a".to_string())); // ✅ Succeeds: Each map has its own memory -assert_eq!(map_b.get(&1), Some("value_b".to_string())); // ✅ Succeeds: No data corruption +map_a.insert(1, b'A'); +map_b.insert(1, b'B'); +assert_eq!(map_a.get(&1), Some(b'A')); // ✅ Succeeds: Each map has its own memory +assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds: No data corruption ``` ## Example Canister From 6b3021d933d67a8775c64b428505b8fdec803d1f Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:27:57 +0200 Subject: [PATCH 07/15] memory reclamation section --- README.md | 56 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 26cab8ea..50c217c6 100644 --- a/README.md +++ b/README.md @@ -58,21 +58,6 @@ This includes stable memory, a vector ([VectorMemory]), or even a flat file ([Fi The example above initializes a [BTreeMap] with a [DefaultMemoryImpl], which maps to stable memory when used in a canister and to a [VectorMemory] otherwise. -### Example: BTreeSet - -The `BTreeSet` is a stable set implementation based on a B-Tree. It allows efficient insertion, deletion, and lookup of unique elements. - -```rust -use ic_stable_structures::{BTreeSet, DefaultMemoryImpl}; -let mut set: BTreeSet = BTreeSet::new(DefaultMemoryImpl::default()); - -set.insert(42); -assert!(set.contains(&42)); -assert_eq!(set.pop_first(), Some(42)); -assert!(set.is_empty()); -``` - - Note that **stable structures cannot share memories.** Each memory must belong to only one stable structure. For example, this fails when run in a canister: @@ -108,6 +93,47 @@ assert_eq!(map_a.get(&1), Some(b'A')); // ✅ Succeeds: Each map has its own mem assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds: No data corruption ``` +### Memory Reclamation + +Virtual memories remain assigned to their memory IDs even after the data structure is dropped. This can cause memory waste during data migration scenarios. For example, when migrating from structure A to structure B using different memory IDs, the underlying memory grows 2x even though only one structure contains data. + +**Without memory reclamation (memory waste):** +```rust +let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); + +// Structure A uses memory +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); +map_a.insert(1, b'A'); + +// Migrate data and create structure B with different ID +let data = map_a.get(&1); +drop(map_a); // A's memory remains allocated to ID 0 +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); // Allocates NEW memory +map_b.insert(1, data.unwrap()); +// Result: 2x memory usage (A's unused memory + B's new memory) +``` + +**With memory reclamation (memory reuse):** +```rust +let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); + +// Structure A uses memory +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); +map_a.insert(1, b'A'); + +// Migrate data, drop structure, and reclaim memory +let data = map_a.get(&1); +drop(map_a); +mem_mgr.reclaim_memory(MemoryId::new(0)); // Free A's memory for reuse + +// Structure B reuses A's memory +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); // Reuses A's memory +map_b.insert(1, data.unwrap()); +// Result: Same memory usage (B reuses A's memory) +``` + +**Important**: Always ensure the original structure is dropped before calling `reclaim_memory`. + ## Example Canister Here's a fully working canister example that ties everything together. From 464615a40e3bd45179335a27dafd30b217d269c1 Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:33:16 +0200 Subject: [PATCH 08/15] . --- README.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 50c217c6..cd8ce349 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Stable structures are able to work directly in stable memory because each data s its own memory. When initializing a stable structure, a memory is provided that the data structure can use to store its data. -Here are some basic examples: +Here's a basic example: ### Example: BTreeMap @@ -75,8 +75,8 @@ assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds, but corrupted map_a It fails because both `map_a` and `map_b` are using the same stable memory under the hood, and so changes in `map_a` end up changing or corrupting `map_b`. -To address this issue, we make use of the [MemoryManager](memory_manager::MemoryManager), which takes a single memory and creates up to 255 virtual memories for our disposal. -Here's the above failing example, but fixed by using the [MemoryManager](memory_manager::MemoryManager): +To address this issue, we use the [MemoryManager](memory_manager::MemoryManager), which takes a single memory and creates up to 255 virtual memories for our use. +Here's the above failing example, but fixed: ```rust use ic_stable_structures::{ @@ -99,6 +99,10 @@ Virtual memories remain assigned to their memory IDs even after the data structu **Without memory reclamation (memory waste):** ```rust +use ic_stable_structures::{ + memory_manager::{MemoryId, MemoryManager}, + BTreeMap, DefaultMemoryImpl, +}; let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); // Structure A uses memory @@ -115,6 +119,10 @@ map_b.insert(1, data.unwrap()); **With memory reclamation (memory reuse):** ```rust +use ic_stable_structures::{ + memory_manager::{MemoryId, MemoryManager}, + BTreeMap, DefaultMemoryImpl, +}; let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); // Structure A uses memory @@ -144,7 +152,7 @@ Dependencies: [dependencies] ic-cdk = "0.18.3" ic-cdk-macros = "0.18.3" -ic-stable-structures = "0.5.6" +ic-stable-structures = "0.7.0" ``` Code: @@ -185,19 +193,19 @@ fn insert(key: u64, value: String) -> Option { ### More Examples -- [Basic Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/basic_example) (the one above) +- [Basic Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/basic_example): Simple usage patterns - [Quickstart Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/quick_start): Ideal as a template when developing a new canister - [Custom Types Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/custom_types_example): Showcases storing your own custom types ## Combined Persistence -If your project exclusively relies on stable structures, the memory can expand in size without the requirement of `pre_upgrade`/`post_upgrade` hooks. +If your project uses only stable structures, memory can expand in size without requiring `pre_upgrade`/`post_upgrade` hooks. -However, it's important to note that if you also intend to perform serialization/deserialization of the heap data, utilizing the memory manager becomes necessary. To effectively combine both approaches, refer to the [Quickstart Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/quick_start) for guidance. +However, if you also need to serialize/deserialize heap data, you must use the memory manager to avoid conflicts. To combine both approaches effectively, refer to the [Quickstart Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/quick_start) for guidance. ## Fuzzing -Stable structures requires strong guarantees to work reliably and scale over millions of operations. To that extent, we use fuzzing to emulate such operations on the available data structures. +Stable structures require strong guarantees to work reliably and scale over millions of operations. To that extent, we use fuzzing to emulate such operations on the available data structures. To run a fuzzer locally, ```sh From c2122340c1aced623b7a2f23f788f4b5558ca766 Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:36:16 +0200 Subject: [PATCH 09/15] . --- README.md | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index cd8ce349..322a2a63 100644 --- a/README.md +++ b/README.md @@ -95,9 +95,8 @@ assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds: No data corruption ### Memory Reclamation -Virtual memories remain assigned to their memory IDs even after the data structure is dropped. This can cause memory waste during data migration scenarios. For example, when migrating from structure A to structure B using different memory IDs, the underlying memory grows 2x even though only one structure contains data. +Virtual memories remain assigned to their memory IDs even after structures are dropped, which can waste memory during data migrations. For example: -**Without memory reclamation (memory waste):** ```rust use ic_stable_structures::{ memory_manager::{MemoryId, MemoryManager}, @@ -105,42 +104,23 @@ use ic_stable_structures::{ }; let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); -// Structure A uses memory +// Without reclamation: migrating A→B doubles memory usage let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); map_a.insert(1, b'A'); - -// Migrate data and create structure B with different ID let data = map_a.get(&1); -drop(map_a); // A's memory remains allocated to ID 0 -let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); // Allocates NEW memory -map_b.insert(1, data.unwrap()); -// Result: 2x memory usage (A's unused memory + B's new memory) -``` +drop(map_a); // Memory stays allocated to ID 0 +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); // Uses NEW memory -**With memory reclamation (memory reuse):** -```rust -use ic_stable_structures::{ - memory_manager::{MemoryId, MemoryManager}, - BTreeMap, DefaultMemoryImpl, -}; -let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); - -// Structure A uses memory +// With reclamation: B reuses A's memory let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); map_a.insert(1, b'A'); - -// Migrate data, drop structure, and reclaim memory let data = map_a.get(&1); drop(map_a); -mem_mgr.reclaim_memory(MemoryId::new(0)); // Free A's memory for reuse - -// Structure B reuses A's memory -let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); // Reuses A's memory -map_b.insert(1, data.unwrap()); -// Result: Same memory usage (B reuses A's memory) +mem_mgr.reclaim_memory(MemoryId::new(0)); // Free memory for reuse +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); // Reuses memory ``` -**Important**: Always ensure the original structure is dropped before calling `reclaim_memory`. +**Important**: Always drop the original structure before calling `reclaim_memory`. ## Example Canister From b752cfd4b645d4445366c19271297f5af0bb2d2d Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:47:50 +0200 Subject: [PATCH 10/15] . --- README.md | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 322a2a63..760bf811 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,9 @@ assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds: No data corruption ### Memory Reclamation -Virtual memories remain assigned to their memory IDs even after structures are dropped, which can waste memory during data migrations. For example: +During data migration scenarios, you often need to create a new data structure (B) and populate it with data from an existing structure (A). Without memory reclamation, this process doubles memory usage even after A is no longer needed. + +Consider this migration scenario: ```rust use ic_stable_structures::{ @@ -103,21 +105,30 @@ use ic_stable_structures::{ BTreeMap, DefaultMemoryImpl, }; let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); - -// Without reclamation: migrating A→B doubles memory usage -let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); -map_a.insert(1, b'A'); -let data = map_a.get(&1); -drop(map_a); // Memory stays allocated to ID 0 -let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); // Uses NEW memory - -// With reclamation: B reuses A's memory -let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); -map_a.insert(1, b'A'); -let data = map_a.get(&1); -drop(map_a); -mem_mgr.reclaim_memory(MemoryId::new(0)); // Free memory for reuse -let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); // Reuses memory +let (mem_id_a, mem_id_b) = (MemoryId::new(0), MemoryId::new(1)); + +// Scenario 1: WITHOUT reclamation - doubles memory usage +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_a)); +map_a.insert(1, b'A'); // A is populated with data +let data = map_a.get(&1); // Extract data for migration +map_a.clear_new(); // A is now empty +drop(map_a); // but still holds allocated memory, even after drop. + +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_b)); +map_b.insert(1, data.unwrap()); // B gets new memory allocation +// Result: 2x memory usage (A's unused memory + B's new memory) + +// Scenario 2: WITH reclamation - reuses memory efficiently +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_a)); +map_a.insert(1, b'A'); // A is populated with data +let data = map_a.get(&1); // Extract data for migration +map_a.clear_new(); // A is now empty +drop(map_a); // Drop A completely +mem_mgr.reclaim_memory(mem_id_a); // Free A's memory buckets for reuse + +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_b)); +map_b.insert(1, data.unwrap()); // B reuses A's reclaimed memory buckets +// Result: 1x memory usage (B reuses A's freed memory) ``` **Important**: Always drop the original structure before calling `reclaim_memory`. From 5c6c45dae307a02c896902567978488b7f53fe0c Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:51:33 +0200 Subject: [PATCH 11/15] . --- README.md | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 760bf811..adb51041 100644 --- a/README.md +++ b/README.md @@ -104,31 +104,35 @@ use ic_stable_structures::{ memory_manager::{MemoryId, MemoryManager}, BTreeMap, DefaultMemoryImpl, }; + let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); let (mem_id_a, mem_id_b) = (MemoryId::new(0), MemoryId::new(1)); -// Scenario 1: WITHOUT reclamation - doubles memory usage +// ======================================== +// Scenario 1: WITHOUT reclamation +// ======================================== let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_a)); -map_a.insert(1, b'A'); // A is populated with data -let data = map_a.get(&1); // Extract data for migration -map_a.clear_new(); // A is now empty -drop(map_a); // but still holds allocated memory, even after drop. +map_a.insert(1, b'A'); // Populate map A with data +let data = map_a.get(&1); // Extract data for migration +map_a.clear_new(); // A is now empty +drop(map_a); // Memory stays allocated to mem_id_a let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_b)); -map_b.insert(1, data.unwrap()); // B gets new memory allocation -// Result: 2x memory usage (A's unused memory + B's new memory) +map_b.insert(1, data.unwrap()); // B allocates NEW memory + // Result: 2x memory usage -// Scenario 2: WITH reclamation - reuses memory efficiently +// ======================================== +// Scenario 2: WITH reclamation +// ======================================== let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_a)); -map_a.insert(1, b'A'); // A is populated with data -let data = map_a.get(&1); // Extract data for migration -map_a.clear_new(); // A is now empty -drop(map_a); // Drop A completely -mem_mgr.reclaim_memory(mem_id_a); // Free A's memory buckets for reuse +map_a.insert(1, b'A'); // Populate map A with data +let data = map_a.get(&1); // Extract data for migration +drop(map_a); // Drop A completely +mem_mgr.reclaim_memory(mem_id_a); // Free A's memory buckets for reuse let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_b)); -map_b.insert(1, data.unwrap()); // B reuses A's reclaimed memory buckets -// Result: 1x memory usage (B reuses A's freed memory) +map_b.insert(1, data.unwrap()); // B reuses A's reclaimed memory buckets + // Result: 1x memory usage ``` **Important**: Always drop the original structure before calling `reclaim_memory`. From 11dd3c11d55d2dea90b8684183420cd73eb05776 Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:53:56 +0200 Subject: [PATCH 12/15] . --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index adb51041..45e88ba1 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,9 @@ Stable structures are able to work directly in stable memory because each data s its own memory. When initializing a stable structure, a memory is provided that the data structure can use to store its data. -Here's a basic example: +### Basic Usage -### Example: BTreeMap +Here's a basic example: ```rust use ic_stable_structures::{BTreeMap, DefaultMemoryImpl}; @@ -58,6 +58,8 @@ This includes stable memory, a vector ([VectorMemory]), or even a flat file ([Fi The example above initializes a [BTreeMap] with a [DefaultMemoryImpl], which maps to stable memory when used in a canister and to a [VectorMemory] otherwise. +### Memory Isolation Requirement + Note that **stable structures cannot share memories.** Each memory must belong to only one stable structure. For example, this fails when run in a canister: @@ -75,6 +77,8 @@ assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds, but corrupted map_a It fails because both `map_a` and `map_b` are using the same stable memory under the hood, and so changes in `map_a` end up changing or corrupting `map_b`. +### Using MemoryManager + To address this issue, we use the [MemoryManager](memory_manager::MemoryManager), which takes a single memory and creates up to 255 virtual memories for our use. Here's the above failing example, but fixed: From 8271daf84be5d36296d5cd6a20ffd0be72f3c7f6 Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:56:14 +0200 Subject: [PATCH 13/15] . --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 45e88ba1..89ab41a1 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,8 @@ The example above initializes a [BTreeMap] with a [DefaultMemoryImpl], which map ### Memory Isolation Requirement -Note that **stable structures cannot share memories.** -Each memory must belong to only one stable structure. +> **⚠️ CRITICAL:** Stable structures **MUST NOT** share memories! +> Each memory must belong to only one stable structure. For example, this fails when run in a canister: ```rust,ignore From b93b3e6f888982b484fddf75edb4c67a8991306d Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 10:57:57 +0200 Subject: [PATCH 14/15] clippy --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 89ab41a1..835aa216 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ The example above initializes a [BTreeMap] with a [DefaultMemoryImpl], which map > **⚠️ CRITICAL:** Stable structures **MUST NOT** share memories! > Each memory must belong to only one stable structure. + For example, this fails when run in a canister: ```rust,ignore From 2160c23ba47b32971248a0de52a8baa7291b1b65 Mon Sep 17 00:00:00 2001 From: Maksym Arutyunyan Date: Tue, 19 Aug 2025 12:54:56 +0200 Subject: [PATCH 15/15] update book --- README.md | 2 +- docs/src/concepts/memory-manager.md | 62 +++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 835aa216..15649f55 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ assert_eq!(map_a.get(&1), Some(b'A')); // ❌ FAILS: Returns b'B' due to shared assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds, but corrupted map_a ``` -It fails because both `map_a` and `map_b` are using the same stable memory under the hood, and so changes in `map_a` end up changing or corrupting `map_b`. +It fails because both `map_a` and `map_b` are using the same stable memory under the hood, and so changes in `map_b` end up changing or corrupting `map_a`. ### Using MemoryManager diff --git a/docs/src/concepts/memory-manager.md b/docs/src/concepts/memory-manager.md index 888fd317..5b665f23 100644 --- a/docs/src/concepts/memory-manager.md +++ b/docs/src/concepts/memory-manager.md @@ -22,16 +22,64 @@ use ic_stable_structures::{ let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); // Create two separate BTreeMaps, each with its own virtual memory -let mut map_1: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); -let mut map_2: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); // Demonstrate independent operation of the two maps -map_1.insert(1, 2); -map_2.insert(1, 3); -assert_eq!(map_1.get(&1), Some(2)); // Succeeds as expected +map_a.insert(1, b'A'); +map_b.insert(1, b'B'); +assert_eq!(map_a.get(&1), Some(b'A')); // ✅ Succeeds: Each map has its own memory +assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds: No data corruption ``` ```admonish warning "" -Virtual memories from the `MemoryManager` cannot be shared between stable structures. -Each memory instance should be assigned to exactly one stable structure. +**⚠️ CRITICAL:** Stable structures **MUST NOT** share memories! +Each memory instance must be assigned to exactly one stable structure. +``` + +## Memory Reclamation + +During data migration scenarios, you often need to create a new data structure and populate it with data from an existing structure. Without memory reclamation, this process doubles memory usage even after the original structure is no longer needed. + +The `MemoryManager` provides a `reclaim_memory` method to efficiently handle these scenarios: + +```rust +use ic_stable_structures::{ + memory_manager::{MemoryId, MemoryManager}, + BTreeMap, DefaultMemoryImpl, +}; + +let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); +let (mem_id_a, mem_id_b) = (MemoryId::new(0), MemoryId::new(1)); + +// ======================================== +// Scenario 1: WITHOUT reclamation +// ======================================== +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_a)); +map_a.insert(1, b'A'); // Populate map A with data +let data = map_a.get(&1); // Extract data for migration +map_a.clear_new(); // A is now empty +drop(map_a); // Memory stays allocated to mem_id_a + +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_b)); +map_b.insert(1, data.unwrap()); // B allocates NEW memory + // Result: 2x memory usage + +// ======================================== +// Scenario 2: WITH reclamation +// ======================================== +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_a)); +map_a.insert(1, b'A'); // Populate map A with data +let data = map_a.get(&1); // Extract data for migration +drop(map_a); // Drop A completely +mem_mgr.reclaim_memory(mem_id_a); // Free A's memory buckets for reuse + +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_b)); +map_b.insert(1, data.unwrap()); // B reuses A's reclaimed memory buckets + // Result: 1x memory usage +``` + +```admonish info "" +**Important**: Always drop the original structure before calling `reclaim_memory`. +The method returns the number of pages that were reclaimed and made available for reuse. ```