diff --git a/README.md b/README.md index 79ca7ad5..15649f55 100644 --- a/README.md +++ b/README.md @@ -40,16 +40,16 @@ Stable structures are able to work directly in stable memory because each data s its own memory. When initializing a stable structure, a memory is provided that the data structure can use to store its data. -Here are some basic examples: +### Basic Usage -### Example: BTreeMap +Here's a basic example: ```rust use ic_stable_structures::{BTreeMap, DefaultMemoryImpl}; -let mut map: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); +let mut map: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); -map.insert(1, 2); -assert_eq!(map.get(&1), Some(2)); +map.insert(1, "hello".to_string()); +assert_eq!(map.get(&1), Some("hello".to_string())); ``` Memories are abstracted with the [Memory] trait, and stable structures can work with any storage @@ -58,39 +58,30 @@ This includes stable memory, a vector ([VectorMemory]), or even a flat file ([Fi The example above initializes a [BTreeMap] with a [DefaultMemoryImpl], which maps to stable memory when used in a canister and to a [VectorMemory] otherwise. -### Example: BTreeSet +### Memory Isolation Requirement -The `BTreeSet` is a stable set implementation based on a B-Tree. It allows efficient insertion, deletion, and lookup of unique elements. +> **⚠️ CRITICAL:** Stable structures **MUST NOT** share memories! +> Each memory must belong to only one stable structure. -```rust -use ic_stable_structures::{BTreeSet, DefaultMemoryImpl}; -let mut set: BTreeSet = BTreeSet::new(DefaultMemoryImpl::default()); - -set.insert(42); -assert!(set.contains(&42)); -assert_eq!(set.pop_first(), Some(42)); -assert!(set.is_empty()); -``` - - -Note that **stable structures cannot share memories.** -Each memory must belong to only one stable structure. For example, this fails when run in a canister: -```no_run +```rust,ignore use ic_stable_structures::{BTreeMap, DefaultMemoryImpl}; -let mut map_1: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); -let mut map_2: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); +let mut map_a: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); +let mut map_b: BTreeMap = BTreeMap::init(DefaultMemoryImpl::default()); -map_1.insert(1, 2); -map_2.insert(1, 3); -assert_eq!(map_1.get(&1), Some(2)); // This assertion fails. +map_a.insert(1, b'A'); +map_b.insert(1, b'B'); +assert_eq!(map_a.get(&1), Some(b'A')); // ❌ FAILS: Returns b'B' due to shared memory! +assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds, but corrupted map_a ``` -It fails because both `map_1` and `map_2` are using the same stable memory under the hood, and so changes in `map_1` end up changing or corrupting `map_2`. +It fails because both `map_a` and `map_b` are using the same stable memory under the hood, and so changes in `map_b` end up changing or corrupting `map_a`. -To address this issue, we make use of the [MemoryManager](memory_manager::MemoryManager), which takes a single memory and creates up to 255 virtual memories for our disposal. -Here's the above failing example, but fixed by using the [MemoryManager](memory_manager::MemoryManager): +### Using MemoryManager + +To address this issue, we use the [MemoryManager](memory_manager::MemoryManager), which takes a single memory and creates up to 255 virtual memories for our use. +Here's the above failing example, but fixed: ```rust use ic_stable_structures::{ @@ -98,14 +89,59 @@ use ic_stable_structures::{ BTreeMap, DefaultMemoryImpl, }; let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); -let mut map_1: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); -let mut map_2: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); -map_1.insert(1, 2); -map_2.insert(1, 3); -assert_eq!(map_1.get(&1), Some(2)); // Succeeds, as expected. +map_a.insert(1, b'A'); +map_b.insert(1, b'B'); +assert_eq!(map_a.get(&1), Some(b'A')); // ✅ Succeeds: Each map has its own memory +assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds: No data corruption ``` +### Memory Reclamation + +During data migration scenarios, you often need to create a new data structure (B) and populate it with data from an existing structure (A). Without memory reclamation, this process doubles memory usage even after A is no longer needed. + +Consider this migration scenario: + +```rust +use ic_stable_structures::{ + memory_manager::{MemoryId, MemoryManager}, + BTreeMap, DefaultMemoryImpl, +}; + +let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); +let (mem_id_a, mem_id_b) = (MemoryId::new(0), MemoryId::new(1)); + +// ======================================== +// Scenario 1: WITHOUT reclamation +// ======================================== +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_a)); +map_a.insert(1, b'A'); // Populate map A with data +let data = map_a.get(&1); // Extract data for migration +map_a.clear_new(); // A is now empty +drop(map_a); // Memory stays allocated to mem_id_a + +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_b)); +map_b.insert(1, data.unwrap()); // B allocates NEW memory + // Result: 2x memory usage + +// ======================================== +// Scenario 2: WITH reclamation +// ======================================== +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_a)); +map_a.insert(1, b'A'); // Populate map A with data +let data = map_a.get(&1); // Extract data for migration +drop(map_a); // Drop A completely +mem_mgr.reclaim_memory(mem_id_a); // Free A's memory buckets for reuse + +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_b)); +map_b.insert(1, data.unwrap()); // B reuses A's reclaimed memory buckets + // Result: 1x memory usage +``` + +**Important**: Always drop the original structure before calling `reclaim_memory`. + ## Example Canister Here's a fully working canister example that ties everything together. @@ -116,7 +152,7 @@ Dependencies: [dependencies] ic-cdk = "0.18.3" ic-cdk-macros = "0.18.3" -ic-stable-structures = "0.5.6" +ic-stable-structures = "0.7.0" ``` Code: @@ -135,7 +171,7 @@ thread_local! { RefCell::new(MemoryManager::init(DefaultMemoryImpl::default())); // Initialize a `StableBTreeMap` with `MemoryId(0)`. - static MAP: RefCell> = RefCell::new( + static MAP: RefCell> = RefCell::new( StableBTreeMap::init( MEMORY_MANAGER.with(|m| m.borrow().get(MemoryId::new(0))), ) @@ -144,32 +180,32 @@ thread_local! { // Retrieves the value associated with the given key if it exists. #[ic_cdk_macros::query] -fn get(key: u128) -> Option { +fn get(key: u64) -> Option { MAP.with(|p| p.borrow().get(&key)) } // Inserts an entry into the map and returns the previous value of the key if it exists. #[ic_cdk_macros::update] -fn insert(key: u128, value: u128) -> Option { +fn insert(key: u64, value: String) -> Option { MAP.with(|p| p.borrow_mut().insert(key, value)) } ``` ### More Examples -- [Basic Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/basic_example) (the one above) +- [Basic Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/basic_example): Simple usage patterns - [Quickstart Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/quick_start): Ideal as a template when developing a new canister - [Custom Types Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/custom_types_example): Showcases storing your own custom types ## Combined Persistence -If your project exclusively relies on stable structures, the memory can expand in size without the requirement of `pre_upgrade`/`post_upgrade` hooks. +If your project uses only stable structures, memory can expand in size without requiring `pre_upgrade`/`post_upgrade` hooks. -However, it's important to note that if you also intend to perform serialization/deserialization of the heap data, utilizing the memory manager becomes necessary. To effectively combine both approaches, refer to the [Quickstart Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/quick_start) for guidance. +However, if you also need to serialize/deserialize heap data, you must use the memory manager to avoid conflicts. To combine both approaches effectively, refer to the [Quickstart Example](https://github.com/dfinity/stable-structures/tree/main/examples/src/quick_start) for guidance. ## Fuzzing -Stable structures requires strong guarantees to work reliably and scale over millions of operations. To that extent, we use fuzzing to emulate such operations on the available data structures. +Stable structures require strong guarantees to work reliably and scale over millions of operations. To that extent, we use fuzzing to emulate such operations on the available data structures. To run a fuzzer locally, ```sh diff --git a/docs/src/concepts/memory-manager.md b/docs/src/concepts/memory-manager.md index 888fd317..5b665f23 100644 --- a/docs/src/concepts/memory-manager.md +++ b/docs/src/concepts/memory-manager.md @@ -22,16 +22,64 @@ use ic_stable_structures::{ let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); // Create two separate BTreeMaps, each with its own virtual memory -let mut map_1: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); -let mut map_2: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(0))); +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(MemoryId::new(1))); // Demonstrate independent operation of the two maps -map_1.insert(1, 2); -map_2.insert(1, 3); -assert_eq!(map_1.get(&1), Some(2)); // Succeeds as expected +map_a.insert(1, b'A'); +map_b.insert(1, b'B'); +assert_eq!(map_a.get(&1), Some(b'A')); // ✅ Succeeds: Each map has its own memory +assert_eq!(map_b.get(&1), Some(b'B')); // ✅ Succeeds: No data corruption ``` ```admonish warning "" -Virtual memories from the `MemoryManager` cannot be shared between stable structures. -Each memory instance should be assigned to exactly one stable structure. +**⚠️ CRITICAL:** Stable structures **MUST NOT** share memories! +Each memory instance must be assigned to exactly one stable structure. +``` + +## Memory Reclamation + +During data migration scenarios, you often need to create a new data structure and populate it with data from an existing structure. Without memory reclamation, this process doubles memory usage even after the original structure is no longer needed. + +The `MemoryManager` provides a `reclaim_memory` method to efficiently handle these scenarios: + +```rust +use ic_stable_structures::{ + memory_manager::{MemoryId, MemoryManager}, + BTreeMap, DefaultMemoryImpl, +}; + +let mem_mgr = MemoryManager::init(DefaultMemoryImpl::default()); +let (mem_id_a, mem_id_b) = (MemoryId::new(0), MemoryId::new(1)); + +// ======================================== +// Scenario 1: WITHOUT reclamation +// ======================================== +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_a)); +map_a.insert(1, b'A'); // Populate map A with data +let data = map_a.get(&1); // Extract data for migration +map_a.clear_new(); // A is now empty +drop(map_a); // Memory stays allocated to mem_id_a + +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_b)); +map_b.insert(1, data.unwrap()); // B allocates NEW memory + // Result: 2x memory usage + +// ======================================== +// Scenario 2: WITH reclamation +// ======================================== +let mut map_a: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_a)); +map_a.insert(1, b'A'); // Populate map A with data +let data = map_a.get(&1); // Extract data for migration +drop(map_a); // Drop A completely +mem_mgr.reclaim_memory(mem_id_a); // Free A's memory buckets for reuse + +let mut map_b: BTreeMap = BTreeMap::init(mem_mgr.get(mem_id_b)); +map_b.insert(1, data.unwrap()); // B reuses A's reclaimed memory buckets + // Result: 1x memory usage +``` + +```admonish info "" +**Important**: Always drop the original structure before calling `reclaim_memory`. +The method returns the number of pages that were reclaimed and made available for reuse. ``` diff --git a/src/btreemap.rs b/src/btreemap.rs index 7b079cb4..cdb6364e 100644 --- a/src/btreemap.rs +++ b/src/btreemap.rs @@ -710,14 +710,14 @@ where // TODO: In next major release (v1.0), rename this method to `clear` to follow // standard Rust collection naming conventions. /// - /// # Safety Note for Bucket Release - /// If using manual bucket release via `MemoryManager::release_virtual_memory_buckets()`: + /// # Safety Note for Memory Reclamation + /// If using manual memory reclamation via `MemoryManager::reclaim_memory()`: /// 1. **MANDATORY**: Drop this BTreeMap object first (let it go out of scope) - /// 2. Call `release_virtual_memory_buckets()` on the memory manager + /// 2. Call `reclaim_memory()` on the memory manager /// 3. Create new structures as needed /// - /// Using this BTreeMap after bucket release causes data corruption. - /// Note: You can still call `clear_new()` if you need to clear data without bucket release. + /// Using this BTreeMap after memory reclamation causes data corruption. + /// Note: You can still call `clear_new()` if you need to clear data without memory reclamation. pub fn clear_new(&mut self) { self.root_addr = NULL; self.length = 0; diff --git a/src/btreeset.rs b/src/btreeset.rs index f5b3edad..781c5531 100644 --- a/src/btreeset.rs +++ b/src/btreeset.rs @@ -353,14 +353,14 @@ where /// assert!(set.is_empty()); /// ``` /// - /// # Safety Note for Bucket Release - /// If using manual bucket release via `MemoryManager::release_virtual_memory_buckets()`: + /// # Safety Note for Memory Reclamation + /// If using manual memory reclamation via `MemoryManager::reclaim_memory()`: /// 1. **MANDATORY**: Drop this BTreeSet object first (let it go out of scope) - /// 2. Call `release_virtual_memory_buckets()` on the memory manager + /// 2. Call `reclaim_memory()` on the memory manager /// 3. Create new structures as needed /// - /// Using this BTreeSet after bucket release causes data corruption. - /// Note: You can still call `clear()` if you need to clear data without bucket release. + /// Using this BTreeSet after memory reclamation causes data corruption. + /// Note: You can still call `clear()` if you need to clear data without memory reclamation. pub fn clear(&mut self) { self.map.clear_new(); } diff --git a/src/memory_manager.rs b/src/memory_manager.rs index aa64f287..5d76705e 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -130,11 +130,11 @@ const HEADER_RESERVED_BYTES: usize = 32; /// /// # Current Limitations & Safety Requirements /// -/// - **Manual bucket release** - call `release_virtual_memory_buckets()` after dropping structures -/// - **Structure invalidation** - original structures become invalid after bucket release -/// - **Mandatory drop** - you MUST drop original structures BEFORE releasing buckets +/// - **Manual memory reclamation** - call `reclaim_memory()` after dropping structures +/// - **Structure invalidation** - original structures become invalid after memory reclamation +/// - **Mandatory drop** - you MUST drop original structures BEFORE reclaiming memory /// - **No safety verification** - user discipline required to prevent data corruption -/// - **Incorrect usage** - using structures after bucket release causes data corruption +/// - **Incorrect usage** - using structures after memory reclamation causes data corruption pub struct MemoryManager { inner: Rc>>, } @@ -164,27 +164,22 @@ impl MemoryManager { } } - /// Releases buckets allocated to the specified virtual memory for reuse. + /// Reclaims memory allocated to the specified virtual memory for reuse. /// - /// **CRITICAL SAFETY REQUIREMENT**: Before calling this method: - /// 1. You MUST drop the original structure object first + /// **CRITICAL SAFETY REQUIREMENT**: Drop the original structure object first! /// - /// **After calling this method**: - /// 2. The data structure using this memory ID becomes INVALID - /// 3. You MUST create a new structure if you want to reuse the memory ID - /// - /// **Correct Usage Pattern:** + /// **Usage Pattern:** /// ```rust,ignore - /// drop(map); // 1. MANDATORY: Drop the structure object first - /// let count = memory_manager.release_virtual_memory_buckets(memory_id); // 2. Release buckets - /// let new_map = BTreeMap::new(memory_manager.get(memory_id)); // 3. Create fresh structure + /// drop(map); // 1. Drop the structure object first + /// let pages = memory_manager.reclaim_memory(memory_id); // 2. Reclaim memory + /// let new_map = BTreeMap::new(memory_manager.get(memory_id)); // 3. Create new structure /// ``` /// - /// **DANGER**: Using the original structure after bucket release causes data corruption. + /// **DANGER**: Using the original structure after reclamation causes data corruption. /// - /// Returns the number of buckets released and added to the free pool (0 if none allocated). - pub fn release_virtual_memory_buckets(&self, id: MemoryId) -> usize { - self.inner.borrow_mut().release_virtual_memory_buckets(id) + /// Returns the number of pages reclaimed (0 if none allocated). + pub fn reclaim_memory(&self, id: MemoryId) -> u64 { + self.inner.borrow_mut().reclaim_memory(id) } /// Returns the underlying memory. @@ -439,16 +434,14 @@ impl MemoryManagerInner { old_size as i64 } - /// Releases buckets for the specified memory, marking them unallocated in stable storage + /// Reclaims memory for the specified virtual memory, marking buckets unallocated /// and adding them to the free pool. Resets memory size to 0. /// /// **CRITICAL**: This invalidates any data structures using this memory ID. - /// Caller must ensure: - /// 1. Original structure object is dropped BEFORE calling this - /// 2. New structure is created if memory ID needs to be reused + /// Drop the original structure object BEFORE calling this method. /// - /// Returns the number of buckets released and added to the free pool (0 if none allocated). - fn release_virtual_memory_buckets(&mut self, id: MemoryId) -> usize { + /// Returns the number of pages reclaimed (0 if none allocated). + fn reclaim_memory(&mut self, id: MemoryId) -> u64 { let memory_buckets = &mut self.memory_buckets[id.0 as usize]; let bucket_count = memory_buckets.len(); @@ -457,18 +450,18 @@ impl MemoryManagerInner { } // Mark all buckets as unallocated in stable storage and collect them - let mut released_buckets = Vec::new(); + let mut reclaimed_buckets = Vec::new(); for &bucket_id in memory_buckets.iter() { write( &self.memory, bucket_allocations_address(bucket_id).get(), &[UNALLOCATED_BUCKET_MARKER], ); - released_buckets.push(bucket_id); + reclaimed_buckets.push(bucket_id); } - // Add released buckets to free pool and sort to maintain increasing order - self.free_buckets.extend(released_buckets); + // Add reclaimed buckets to free pool and sort to maintain increasing order + self.free_buckets.extend(reclaimed_buckets); let mut sorted_buckets: Vec<_> = self.free_buckets.drain(..).collect(); sorted_buckets.sort_by_key(|bucket| bucket.0); self.free_buckets = VecDeque::from(sorted_buckets); @@ -481,7 +474,8 @@ impl MemoryManagerInner { self.save_header(); - bucket_count + // Return pages reclaimed (bucket count * pages per bucket) + bucket_count as u64 * self.bucket_size_in_pages as u64 } fn write(&self, id: MemoryId, offset: u64, src: &[u8], bucket_cache: &BucketCache) { @@ -1209,7 +1203,7 @@ mod test { } #[test] - fn memory_grows_without_manual_release() { + fn memory_grows_without_manual_reclaim() { let mem = make_memory(); let mem_mgr = MemoryManager::init(mem.clone()); let initial_size = mem.size(); @@ -1223,7 +1217,7 @@ mod test { // Verify first allocation grew memory assert_eq!(size_after_first, initial_size + BUCKET_SIZE_IN_PAGES); - // Create second memory WITHOUT releasing first - should allocate bucket 1 + // Create second memory WITHOUT reclaiming first - should allocate bucket 1 let memory_1 = mem_mgr.get(MemoryId::new(1)); memory_1.grow(BUCKET_SIZE_IN_PAGES); memory_1.write(0, b"bucket_id_1"); @@ -1246,7 +1240,7 @@ mod test { } #[test] - fn memory_reuses_buckets_with_manual_release() { + fn memory_reuses_buckets_with_manual_reclaim() { let mem = make_memory(); let mem_mgr = MemoryManager::init(mem.clone()); @@ -1256,17 +1250,21 @@ mod test { memory_0.write(0, b"bucket_id_0"); let size_after_allocation = mem.size(); - // Manually release first memory's buckets - let released_count = mem_mgr.release_virtual_memory_buckets(MemoryId::new(0)); - assert_eq!(released_count, 1, "Should release exactly 1 bucket"); + // Manually reclaim first memory + let pages_reclaimed = mem_mgr.reclaim_memory(MemoryId::new(0)); + assert_eq!( + pages_reclaimed, BUCKET_SIZE_IN_PAGES, + "Should reclaim exactly {} pages", + BUCKET_SIZE_IN_PAGES + ); - // CRITICAL: Drop the memory_0 after bucket release as it's now invalid + // CRITICAL: Drop the memory_0 after memory reclamation as it's now invalid drop(memory_0); // Verify memory size didn't change (buckets marked free, not deallocated) assert_eq!(mem.size(), size_after_allocation); - // Create second memory AFTER manual release - should reuse bucket 0 + // Create second memory AFTER manual memory reclamation - should reuse bucket 0 let memory_1 = mem_mgr.get(MemoryId::new(1)); memory_1.grow(BUCKET_SIZE_IN_PAGES); @@ -1304,11 +1302,21 @@ mod test { memory_1.write(0, b"bucket_id_3"); memory_1.write(BUCKET_SIZE_IN_PAGES * WASM_PAGE_SIZE, b"bucket_id_4"); - // Release all buckets in reverse order to test sorting - let released_1 = mem_mgr.release_virtual_memory_buckets(MemoryId::new(1)); // releases 3, 4 - let released_0 = mem_mgr.release_virtual_memory_buckets(MemoryId::new(0)); // releases 0, 1, 2 - assert_eq!(released_1, 2, "Should release 2 buckets from memory 1"); - assert_eq!(released_0, 3, "Should release 3 buckets from memory 0"); + // Reclaim all buckets in reverse order to test sorting + let pages_reclaimed_1 = mem_mgr.reclaim_memory(MemoryId::new(1)); // reclaims 3, 4 + let pages_reclaimed_0 = mem_mgr.reclaim_memory(MemoryId::new(0)); // reclaims 0, 1, 2 + assert_eq!( + pages_reclaimed_1, + BUCKET_SIZE_IN_PAGES * 2, + "Should reclaim {} pages from memory 1", + BUCKET_SIZE_IN_PAGES * 2 + ); + assert_eq!( + pages_reclaimed_0, + BUCKET_SIZE_IN_PAGES * 3, + "Should reclaim {} pages from memory 0", + BUCKET_SIZE_IN_PAGES * 3 + ); // Allocate new memories - should reuse buckets in increasing order (0, 1, then 2) let memory_2 = mem_mgr.get(MemoryId::new(2)); @@ -1342,10 +1350,10 @@ mod test { } #[cfg(test)] -mod bucket_release_core_tests; +mod memory_reclaim_core_tests; #[cfg(test)] -mod bucket_release_btreemap_tests; +mod memory_reclaim_btreemap_tests; #[cfg(test)] -mod bucket_release_vec_tests; +mod memory_reclaim_vec_tests; diff --git a/src/memory_manager/bucket_release_btreemap_tests.rs b/src/memory_manager/memory_reclaim_btreemap_tests.rs similarity index 78% rename from src/memory_manager/bucket_release_btreemap_tests.rs rename to src/memory_manager/memory_reclaim_btreemap_tests.rs index db965d19..ff28d4d2 100644 --- a/src/memory_manager/bucket_release_btreemap_tests.rs +++ b/src/memory_manager/memory_reclaim_btreemap_tests.rs @@ -1,13 +1,13 @@ -//! Migration scenario tests for BTreeMap with bucket release. +//! Migration scenario tests for BTreeMap with memory reclamation. //! //! These tests demonstrate real-world migration patterns where users move data -//! from one structure to another. They show how bucket release prevents memory +//! from one structure to another. They show how memory reclamation prevents memory //! waste during common migration scenarios, and most importantly, demonstrate //! the data corruption bug and its safe usage solution. //! //! **CRITICAL SAFETY REQUIREMENTS**: -//! All bucket release operations require mandatory Rust object drop BEFORE release. -//! Using original data structures after bucket release causes data corruption. +//! All memory reclamation operations require mandatory Rust object drop BEFORE reclamation. +//! Using original data structures after memory reclamation causes data corruption. //! See MemoryManager documentation for proper usage patterns. use super::{MemoryId, MemoryManager}; @@ -24,8 +24,8 @@ fn large_value(id: u64) -> Vec { } #[test] -fn migration_without_release_wastes_buckets() { - // Scenario: Populate A → Drop A without bucket release → Populate B +fn migration_without_reclaim_wastes_buckets() { + // Scenario: Populate A → Drop A without memory reclamation → Populate B // Result: B cannot reuse A's buckets, causing memory waste (growth) let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); @@ -42,7 +42,7 @@ fn migration_without_release_wastes_buckets() { drop(a_map); let stable_before = mock_stable_memory.size(); - // Allocate in B → should need new buckets since A's aren't released + // Allocate in B → should need new buckets since A's aren't reclaimed let mut b_map = BTreeMap::init(mm.get(b)); for i in 0u64..50 { b_map.insert(i, large_value(i + 100)); @@ -55,9 +55,9 @@ fn migration_without_release_wastes_buckets() { } #[test] -fn migration_with_release_reuses_buckets() { - // Scenario: Populate A → Drop A with bucket release → Populate B - // Result: B reuses A's released buckets, preventing memory waste (no growth) +fn migration_with_reclaim_reuses_buckets() { + // Scenario: Populate A → Drop A with memory reclamation → Populate B + // Result: B reuses A's reclaimed buckets, preventing memory waste (no growth) let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory.clone()); @@ -69,15 +69,15 @@ fn migration_with_release_reuses_buckets() { } assert_eq!(a_map.len(), 50); - // MANDATORY: Drop the Rust object before releasing buckets + // MANDATORY: Drop the Rust object before reclaiming memory drop(a_map); - // Release the buckets after dropping the object - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0); + // Reclaim the memory after dropping the object + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0); let stable_before = mock_stable_memory.size(); - // Allocate in B → should reuse A's released buckets + // Allocate in B → should reuse A's reclaimed buckets let mut b_map = BTreeMap::init(mm.get(b)); for i in 0u64..50 { b_map.insert(i, large_value(i + 100)); @@ -92,8 +92,8 @@ fn migration_with_release_reuses_buckets() { ); } -/// **DANGER**: This test demonstrates data corruption from unsafe bucket release usage. -/// This shows what happens when you DON'T drop the original object before bucket release. +/// **DANGER**: This test demonstrates data corruption from unsafe memory reclamation usage. +/// This shows what happens when you DON'T drop the original object before memory reclamation. #[test] fn data_corruption_without_mandatory_drop() { let (a, b) = (MemoryId::new(0), MemoryId::new(1)); @@ -104,10 +104,10 @@ fn data_corruption_without_mandatory_drop() { map_a.insert(1u64, 100u64); assert_eq!(map_a.get(&1u64).unwrap(), 100u64); - // DANGEROUS: Release buckets but keep map_a alive - mm.release_virtual_memory_buckets(a); + // DANGEROUS: Reclaim memory but keep map_a alive + mm.reclaim_memory(a); - // Create BTreeMap B - reuses A's released buckets + // Create BTreeMap B - reuses A's reclaimed buckets let mut map_b = BTreeMap::init(mm.get(b)); map_b.insert(2u64, 200u64); assert_eq!(map_b.get(&2u64).unwrap(), 200u64); @@ -135,7 +135,7 @@ fn data_corruption_without_mandatory_drop() { Ok(_) => { // If it succeeds, check if map_b can see the new data (shared allocation) if map_b.get(&3u64).is_some() { - println!("CORRUPTION: Both maps operating on the same released memory space"); + println!("CORRUPTION: Both maps operating on the same reclaimed memory space"); } } Err(_) => { @@ -144,10 +144,10 @@ fn data_corruption_without_mandatory_drop() { } } - // This test proves why objects MUST be dropped before bucket release + // This test proves why objects MUST be dropped before memory reclamation } -/// **SAFE**: This test demonstrates the correct way to use bucket release. +/// **SAFE**: This test demonstrates the correct way to use memory reclamation. /// This shows how mandatory drop prevents data corruption. #[test] fn safe_usage_with_mandatory_drop() { @@ -159,14 +159,14 @@ fn safe_usage_with_mandatory_drop() { map_a.insert(1u64, 100u64); assert_eq!(map_a.get(&1u64).unwrap(), 100u64); - // MANDATORY: Drop the Rust object before releasing buckets + // MANDATORY: Drop the Rust object before reclaiming memory drop(map_a); - // Release the buckets after dropping the object - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0); + // Reclaim the memory after dropping the object + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0); - // Create BTreeMap B - safely reuses A's released buckets + // Create BTreeMap B - safely reuses A's reclaimed buckets let mut map_b = BTreeMap::init(mm.get(b)); map_b.insert(2u64, 200u64); assert_eq!(map_b.get(&2u64).unwrap(), 200u64); diff --git a/src/memory_manager/bucket_release_core_tests.rs b/src/memory_manager/memory_reclaim_core_tests.rs similarity index 51% rename from src/memory_manager/bucket_release_core_tests.rs rename to src/memory_manager/memory_reclaim_core_tests.rs index 8e436cf0..c5088e28 100644 --- a/src/memory_manager/bucket_release_core_tests.rs +++ b/src/memory_manager/memory_reclaim_core_tests.rs @@ -1,12 +1,12 @@ -//! Core bucket release functionality tests for MemoryManager. +//! Core memory reclamation functionality tests for MemoryManager. //! //! These tests verify the basic memory management operations without dependency //! on specific data structures. They test the fundamental bucket allocation, -//! release, and reuse mechanisms. +//! reclamation, and reuse mechanisms. //! //! **CRITICAL SAFETY REQUIREMENTS**: -//! All bucket release operations require mandatory Rust object drop BEFORE release. -//! Using original data structures after bucket release causes data corruption. +//! All memory reclamation operations require mandatory Rust object drop BEFORE reclamation. +//! Using original data structures after memory reclamation causes data corruption. //! See MemoryManager documentation for proper usage patterns. use super::{MemoryId, MemoryManager}; @@ -22,26 +22,26 @@ fn allocate_buckets_via_btreemap( for i in 0..count { map.insert(i, vec![42u8; 2000]); // 2KB blob to trigger bucket allocation } - drop(map); // Drop the structure before bucket release + drop(map); // Drop the structure before memory reclamation } #[test] -fn release_empty_memory_returns_zero() { - // Test: Releasing buckets from unused memory should return 0 +fn reclaim_empty_memory_returns_zero() { + // Test: Reclaiming memory from unused memory should return 0 let memory_id = MemoryId::new(0); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory); - // Try to release buckets from empty memory - let released_buckets = mm.release_virtual_memory_buckets(memory_id); + // Try to reclaim memory from empty memory + let pages_reclaimed = mm.reclaim_memory(memory_id); - // Should return 0 since no buckets were allocated - assert_eq!(released_buckets, 0, "Empty memory should release 0 buckets"); + // Should return 0 since no pages were allocated + assert_eq!(pages_reclaimed, 0, "Empty memory should reclaim 0 pages"); } #[test] -fn double_release_returns_zero_second_time() { - // Test: Releasing the same memory twice should return 0 on second attempt +fn double_reclaim_returns_zero_second_time() { + // Test: Reclaiming the same memory twice should return 0 on second attempt let memory_id = MemoryId::new(0); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory); @@ -49,18 +49,18 @@ fn double_release_returns_zero_second_time() { // Allocate buckets and clear structure allocate_buckets_via_btreemap(&mm, memory_id, 50); - // Release buckets - let first_release = mm.release_virtual_memory_buckets(memory_id); - assert!(first_release > 0, "First release should return >0 buckets"); + // First reclamation should return non-zero + let first_reclaim = mm.reclaim_memory(memory_id); + assert!(first_reclaim > 0, "First reclaim should return pages > 0"); - // Try to release again - let second_release = mm.release_virtual_memory_buckets(memory_id); - assert_eq!(second_release, 0, "Second release should return 0 buckets"); + // Second reclamation should return 0 (nothing left to reclaim) + let second_reclaim = mm.reclaim_memory(memory_id); + assert_eq!(second_reclaim, 0, "Second reclaim should return 0 pages"); } #[test] -fn release_only_affects_target_memory() { - // Test: Releasing memory A should not affect memory B's buckets +fn reclaim_only_affects_target_memory() { + // Test: Reclaiming memory A should not affect memory B's buckets let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory); @@ -69,45 +69,45 @@ fn release_only_affects_target_memory() { allocate_buckets_via_btreemap(&mm, a, 30); allocate_buckets_via_btreemap(&mm, b, 30); - // Release A's buckets - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0, "Should release A's buckets"); + // Reclaim A's memory + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0, "Should reclaim A's pages"); - // Verify B still has its buckets (can't be released again) - let b_release_attempt = mm.release_virtual_memory_buckets(b); + // Verify B still has its memory (can't be reclaimed again) + let b_reclaim_attempt = mm.reclaim_memory(b); assert!( - b_release_attempt > 0, - "B should still have releasable buckets" + b_reclaim_attempt > 0, + "B should still have reclaimable pages" ); } #[test] -fn multiple_release_cycles() { - // Test: Multiple allocate→release cycles should work consistently +fn multiple_reclaim_cycles() { + // Test: Multiple allocate→reclaim cycles should work consistently let memory_id = MemoryId::new(0); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory); - // Perform several cycles of allocate→release + // Perform several cycles of allocate→reclaim for cycle in 0..5 { // Allocate buckets allocate_buckets_via_btreemap(&mm, memory_id, 40); - // Release buckets - let released_buckets = mm.release_virtual_memory_buckets(memory_id); + // Reclaim memory + let pages_reclaimed = mm.reclaim_memory(memory_id); assert!( - released_buckets > 0, - "Cycle {} should release >0 buckets, got {}", + pages_reclaimed > 0, + "Cycle {} should reclaim >0 pages, got {}", cycle, - released_buckets + pages_reclaimed ); } } #[test] fn bucket_reuse_prevents_memory_growth() { - // Test: Released buckets are actually reused by subsequent allocations + // Test: Reclaimed buckets are actually reused by subsequent allocations let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory.clone()); @@ -115,18 +115,18 @@ fn bucket_reuse_prevents_memory_growth() { // Allocate buckets in A allocate_buckets_via_btreemap(&mm, a, 50); - // Release A's buckets - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0); + // Reclaim A's memory + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0); let stable_before = mock_stable_memory.size(); - // Allocate same amount in B → should reuse A's released buckets + // Allocate same amount in B → should reuse A's reclaimed buckets allocate_buckets_via_btreemap(&mm, b, 50); let stable_after = mock_stable_memory.size(); // Verify: stable memory should not grow significantly (reuse occurred) assert!( stable_after <= stable_before + 1, // Allow minimal growth for structure overhead - "Stable memory should not grow when reusing released buckets" + "Stable memory should not grow when reusing reclaimed buckets" ); } diff --git a/src/memory_manager/bucket_release_vec_tests.rs b/src/memory_manager/memory_reclaim_vec_tests.rs similarity index 79% rename from src/memory_manager/bucket_release_vec_tests.rs rename to src/memory_manager/memory_reclaim_vec_tests.rs index f90daa0f..d5bd00b4 100644 --- a/src/memory_manager/bucket_release_vec_tests.rs +++ b/src/memory_manager/memory_reclaim_vec_tests.rs @@ -1,17 +1,17 @@ -//! Migration scenario tests for Vec with bucket release. +//! Migration scenario tests for Vec with memory reclamation. //! //! These tests demonstrate real-world migration patterns where users move data -//! from one structure to another. They show how bucket release prevents memory +//! from one structure to another. They show how memory reclamation prevents memory //! waste during common migration scenarios, and most importantly, demonstrate //! the data corruption bug and its safe usage solution for Vec structures. //! //! **CRITICAL SAFETY REQUIREMENTS**: -//! All bucket release operations require mandatory Rust object drop BEFORE release. -//! Using original data structures after bucket release causes data corruption. +//! All memory reclamation operations require mandatory Rust object drop BEFORE reclamation. +//! Using original data structures after memory reclamation causes data corruption. //! See MemoryManager documentation for proper usage patterns. //! //! **Vec-Specific Notes**: -//! Vec doesn't have a clear() method. Since we drop the object before bucket release, +//! Vec doesn't have a clear() method. Since we drop the object before memory reclamation, //! there's no need to clear the data first. use super::{MemoryId, MemoryManager}; @@ -28,8 +28,8 @@ fn large_data(id: u64) -> [u8; 1024] { } #[test] -fn migration_without_release_wastes_buckets() { - // Scenario: Populate A → Clear A without bucket release → Populate B +fn migration_without_reclaim_wastes_buckets() { + // Scenario: Populate A → Clear A without memory reclamation → Populate B // Result: B cannot reuse A's buckets, causing memory waste (growth) let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); @@ -47,7 +47,7 @@ fn migration_without_release_wastes_buckets() { assert_eq!(vec_a.len(), 0); let stable_before = mock_stable_memory.size(); - // Allocate in B → should need new buckets since A's aren't released + // Allocate in B → should need new buckets since A's aren't reclaimed let vec_b = StableVec::init(mm.get(b)); for i in 0u64..50 { vec_b.push(&large_data(i + 100)); @@ -61,9 +61,9 @@ fn migration_without_release_wastes_buckets() { } #[test] -fn migration_with_release_reuses_buckets() { - // Scenario: Populate A → Clear A with bucket release → Populate B - // Result: B reuses A's released buckets, preventing memory waste (no growth) +fn migration_with_reclaim_reuses_buckets() { + // Scenario: Populate A → Clear A with memory reclamation → Populate B + // Result: B reuses A's reclaimed buckets, preventing memory waste (no growth) let (a, b) = (MemoryId::new(0), MemoryId::new(1)); let mock_stable_memory = VectorMemory::default(); let mm = MemoryManager::init(mock_stable_memory.clone()); @@ -75,15 +75,15 @@ fn migration_with_release_reuses_buckets() { } assert_eq!(vec_a_original.len(), 50); - // MANDATORY: Drop the Rust object before releasing buckets + // MANDATORY: Drop the Rust object before reclaiming memory drop(vec_a_original); - // Release the buckets after dropping the object - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0); + // Reclaim the memory after dropping the object + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0); let stable_before = mock_stable_memory.size(); - // Allocate in B → should reuse A's released buckets + // Allocate in B → should reuse A's reclaimed buckets let vec_b = StableVec::init(mm.get(b)); for i in 0u64..50 { vec_b.push(&large_data(i + 100)); @@ -98,8 +98,8 @@ fn migration_with_release_reuses_buckets() { ); } -/// **DANGER**: This test demonstrates data corruption from unsafe bucket release usage. -/// This shows what happens when you DON'T drop the original object after bucket release. +/// **DANGER**: This test demonstrates data corruption from unsafe memory reclamation usage. +/// This shows what happens when you DON'T drop the original object after memory reclamation. #[test] fn data_corruption_without_mandatory_drop() { let (a, b) = (MemoryId::new(0), MemoryId::new(1)); @@ -113,9 +113,9 @@ fn data_corruption_without_mandatory_drop() { // "Clear" by creating new instance, but keep vec_a alive (DANGEROUS!) let vec_a: StableVec = StableVec::new(mm.get(a)); assert_eq!(vec_a.len(), 0); - mm.release_virtual_memory_buckets(a); + mm.reclaim_memory(a); - // Create Vec B - reuses A's released buckets + // Create Vec B - reuses A's reclaimed buckets let vec_b = StableVec::init(mm.get(b)); vec_b.push(&2u64); assert_eq!(vec_b.get(0).unwrap(), 2u64); @@ -143,7 +143,7 @@ fn data_corruption_without_mandatory_drop() { // If it succeeds, check if vec_b can see the new data (shared allocation) if !vec_a.is_empty() && vec_b.len() > vec_a.len() { // Check if data appears in unexpected places - println!("CORRUPTION: Both vecs operating on the same released memory space"); + println!("CORRUPTION: Both vecs operating on the same reclaimed memory space"); } } Err(_) => { @@ -152,10 +152,10 @@ fn data_corruption_without_mandatory_drop() { } } - // This test proves why objects MUST be dropped after bucket release + // This test proves why objects MUST be dropped after memory reclamation } -/// **SAFE**: This test demonstrates the correct way to use bucket release. +/// **SAFE**: This test demonstrates the correct way to use memory reclamation. /// This shows how mandatory drop prevents data corruption. #[test] fn safe_usage_with_mandatory_drop() { @@ -167,14 +167,14 @@ fn safe_usage_with_mandatory_drop() { vec_a.push(&1u64); assert_eq!(vec_a.get(0).unwrap(), 1u64); - // MANDATORY: Drop the Rust object before releasing buckets + // MANDATORY: Drop the Rust object before reclaiming memory drop(vec_a); - // Release the buckets after dropping the object - let released_buckets = mm.release_virtual_memory_buckets(a); - assert!(released_buckets > 0); + // Reclaim the memory after dropping the object + let pages_reclaimed = mm.reclaim_memory(a); + assert!(pages_reclaimed > 0); - // Create Vec B - safely reuses A's released buckets + // Create Vec B - safely reuses A's reclaimed buckets let vec_b = StableVec::init(mm.get(b)); vec_b.push(&2u64); assert_eq!(vec_b.get(0).unwrap(), 2u64);