Merge branch 'pakhomov/lsmt-merge-overlays' into 'master'

feat: merge sharded overlays Merge sharded overlay files. See merge request dfinity-lab/public/ic!18023
dfinity · Mar 13, 2024 · f9372c7 · f9372c7
2 parents 5d9f14b + 57bd876
commit f9372c7
Show file tree

Hide file tree

Showing 8 changed files with 536 additions and 356 deletions.
diff --git a/rs/config/src/state_manager.rs b/rs/config/src/state_manager.rs
@@ -48,6 +48,9 @@ fn file_backed_memory_allocator_default() -> FlagStatus {
 pub fn lsmt_config_default() -> LsmtConfig {
     LsmtConfig {
         lsmt_status: FlagStatus::Disabled,
-        shard_num_pages: u64::MAX,
+        // 40GiB
+        // DO NOT CHANGE after LSMT is enabled, as it would crash the new replica trying to merge
+        // old data.
+        shard_num_pages: 10 * 1024 * 1024,
     }
 }
diff --git a/rs/replicated_state/src/page_map.rs b/rs/replicated_state/src/page_map.rs
@@ -18,7 +18,7 @@ pub use page_allocator::{
 };
 pub use storage::{
     BaseFileSerialization, MergeCandidate, OverlayFileSerialization, Shard, StorageLayout,
-    StorageSerialization, MAX_NUMBER_OF_FILES,
+    StorageResult, StorageSerialization, MAX_NUMBER_OF_FILES,
 };
 use storage::{OverlayFile, OverlayVersion, Storage};