Incremental GC Barriers

default.nix

@@ -382,7 +382,6 @@ rec {
           EXTRA_MOC_ARGS = "--sanity-checks --generational-gc";
       });
 
-
     perf_subdir = dir: deps:
       (test_subdir dir deps).overrideAttrs (args: {
         checkPhase = ''

rts/motoko-rts-tests/src/continuation_table.rs

@@ -1,9 +1,12 @@
+use std::array::from_fn;
+
 use crate::memory::TestMemory;
 
 use motoko_rts::continuation_table::{
     continuation_count, recall_continuation, remember_continuation,
 };
-use motoko_rts::types::{size_of, Array, Value, Words};
+use motoko_rts::memory::alloc_blob;
+use motoko_rts::types::{size_of, Array, Blob, Bytes, Value, Words};
 
 pub unsafe fn test() {
     println!("Testing continuation table ...");
@@ -13,36 +16,35 @@ pub unsafe fn test() {
     const N: usize = 2000; // >256, to exercise `double_continuation_table`
 
     // Array will be doubled 3 times, so 256 + 512 + 1024 + 2048 = 3840 words, plus each array will
-    // have an array header.
-    let mut heap = TestMemory::new(Words(3840 + 4 * size_of::<Array>().to_bytes().as_u32()));
+    // have an array header. Also add the set of N pointers to empty blobs.
+    let mut heap = TestMemory::new(Words(
+        3840 + 4 * size_of::<Array>().to_bytes().as_u32()
+            + N as u32 * size_of::<Blob>().to_bytes().as_u32(),
+    ));
+
+    let pointers: [Value; N] = from_fn(|_| alloc_blob(&mut heap, Bytes(0)));
 
     let mut references: [u32; N] = [0; N];
     for i in 0..N {
-        references[i] = remember_continuation(
-            &mut heap,
-            Value::from_raw(((i as u32) << 2).wrapping_sub(1)),
-        );
+        references[i] = remember_continuation(&mut heap, pointers[i]);
         assert_eq!(continuation_count(), (i + 1) as u32);
     }
 
     for i in 0..N / 2 {
         let c = recall_continuation(references[i]);
-        assert_eq!(c.get_raw(), (i << 2).wrapping_sub(1) as u32);
+        assert_eq!(c.get_ptr(), pointers[i].get_ptr());
         assert_eq!(continuation_count(), (N - i - 1) as u32);
     }
 
     for i in 0..N / 2 {
-        references[i] = remember_continuation(
-            &mut heap,
-            Value::from_raw(((i as u32) << 2).wrapping_sub(1)),
-        );
+        references[i] = remember_continuation(&mut heap, pointers[i]);
         assert_eq!(continuation_count(), (N / 2 + i + 1) as u32);
     }
 
     for i in (0..N).rev() {
         assert_eq!(
-            recall_continuation(references[i]).get_raw(),
-            (i << 2).wrapping_sub(1) as u32,
+            recall_continuation(references[i]).get_ptr(),
+            pointers[i].get_ptr(),
         );
         assert_eq!(continuation_count(), i as u32);
     }

rts/motoko-rts-tests/src/gc/generational.rs

@@ -1,8 +1,10 @@
 mod mark_stack;
+mod remembered_set;
 
 pub fn test() {
-    println!("Testing generational GC components ...");
+    println!("Testing generational GC ...");
     unsafe {
         mark_stack::test();
+        remembered_set::test();
     }
 }

rts/motoko-rts-tests/src/gc/generational/mark_stack.rs

@@ -10,7 +10,7 @@ use motoko_rts::types::*;
 use proptest::test_runner::{Config, TestCaseError, TestCaseResult, TestRunner};
 
 pub unsafe fn test() {
-    println!("  Testing generational GC mark stack ...");
+    println!("  Testing mark stack ...");
 
     test_push_pop();
     test_grow_stack();

rts/motoko-rts-tests/src/remembered_set.rs → rts/motoko-rts-tests/src/gc/generational/remembered_set.rs

@@ -9,7 +9,7 @@ use motoko_rts::types::{Value, Words};
 const GROW_LIMIT: u32 = INITIAL_TABLE_LENGTH * OCCUPATION_THRESHOLD_PERCENT / 100;
 
 pub unsafe fn test() {
-    println!("Testing remembered set ...");
+    println!("  Testing remembered set ...");
 
     test_remembered_set(0);
     test_remembered_set(1);
@@ -31,8 +31,6 @@ unsafe fn test_remembered_set(amount: u32) {
 }
 
 unsafe fn test_insert_iterate(amount: u32) {
-    println!("  Testing insert/iterate {amount}");
-
     let mut mem = TestMemory::new(Words(2 * amount + 1024 * 1024));
 
     let mut remembered_set = RememberedSet::new(&mut mem);
@@ -54,8 +52,6 @@ unsafe fn test_insert_iterate(amount: u32) {
 }
 
 unsafe fn test_duplicates(amount: u32) {
-    println!("  Testing duplicates {amount}");
-
     let mut mem = TestMemory::new(Words(2 * amount + 1024 * 1024));
 
     let mut remembered_set = RememberedSet::new(&mut mem);
@@ -72,8 +68,6 @@ unsafe fn test_duplicates(amount: u32) {
 }
 
 unsafe fn test_collisions(amount: u32) {
-    println!("  Testing collisions {amount}");
-
     let mut mem = TestMemory::new(Words(2 * amount + 1024 * 1024));
 
     let mut remembered_set = RememberedSet::new(&mut mem);

rts/motoko-rts-tests/src/main.rs

@@ -7,7 +7,6 @@ mod gc;
 mod leb128;
 mod memory;
 mod principal_id;
-mod remembered_set;
 mod stream;
 mod text;
 mod utf8;
@@ -32,7 +31,6 @@ fn main() {
         stream::test();
         text::test();
         utf8::test();
-        remembered_set::test();
     }
 }
 

rts/motoko-rts/Cargo.toml

@@ -14,10 +14,17 @@ crate-type = ["staticlib"]
 # is used in RTS tests.
 default = ["ic"]
 
+# To enable extensive memory sanity checks in the incremental GC, use the
+# following default configuration instead:
+# default = ["ic", "memory_check"]
+
 # This feature is used to enable stuff needed for the RTS linked with
 # moc-generated code, but not when testing the RTS
 ic = []
 
+# This feature enables extensive memory sanity checks in the incremental GC.
+memory_check = []
+
 [dependencies]
 libc = { version = "0.2.139", default_features = false }
 motoko-rts-macros = { path = "../motoko-rts-macros" }

rts/motoko-rts/src/bigint.rs

@@ -32,6 +32,7 @@ This scheme makes the following assumptions:
 */
 
 use crate::buf::{read_byte, Buf};
+use crate::gc::incremental::barriers::allocation_barrier;
 use crate::mem_utils::memcpy_bytes;
 use crate::memory::Memory;
 use crate::tommath_bindings::*;
@@ -50,6 +51,7 @@ unsafe fn mp_alloc<M: Memory>(mem: &mut M, size: Bytes<u32>) -> *mut u8 {
     let size = size.as_usize();
     debug_assert_eq!((size % core::mem::size_of::<mp_digit>()), 0);
     (*blob).mp_int.alloc = (size / core::mem::size_of::<mp_digit>()) as i32;
+    allocation_barrier(ptr);
     blob.payload_addr() as *mut u8
 }
 

rts/motoko-rts/src/blob_iter.rs

@@ -1,4 +1,7 @@
-use crate::types::{size_of, Array, Bytes, Value, Words, TAG_ARRAY};
+use crate::{
+    gc::incremental::barriers::allocation_barrier,
+    types::{size_of, Array, Bytes, Value, Words, TAG_ARRAY},
+};
 
 use motoko_rts_macros::ic_mem_fn;
 
@@ -17,10 +20,10 @@ unsafe fn blob_iter<M: crate::memory::Memory>(mem: &mut M, blob: Value) -> Value
     (*iter_array).header.forward = iter_ptr;
     (*iter_array).len = 2;
 
-    iter_array.set_pointer(ITER_BLOB_IDX, blob, mem);
+    iter_array.initialize(ITER_BLOB_IDX, blob, mem);
     iter_array.set_scalar(ITER_POS_IDX, Value::from_scalar(0));
 
-    iter_ptr
+    allocation_barrier(iter_ptr)
 }
 
 /// Returns whether the iterator is finished

rts/motoko-rts/src/continuation_table.rs

@@ -22,6 +22,7 @@
 //! the free list. Since all indices are relative to the payload begin, they stay valid. We never
 //! shrink the table.
 
+use crate::gc::incremental::barriers::allocation_barrier;
 use crate::memory::{alloc_array, Memory};
 use crate::rts_trap_with;
 use crate::types::Value;
@@ -49,6 +50,7 @@ unsafe fn create_continuation_table<M: Memory>(mem: &mut M) {
     for i in 0..INITIAL_SIZE {
         table.set_scalar(i, Value::from_scalar(i + 1));
     }
+    allocation_barrier(TABLE);
 }
 
 unsafe fn double_continuation_table<M: Memory>(mem: &mut M) {
@@ -64,16 +66,13 @@ unsafe fn double_continuation_table<M: Memory>(mem: &mut M) {
 
     for i in 0..old_size {
         let old_value = old_array.get(i);
-        if old_value.is_ptr() {
-            new_array.set_pointer(i, old_value, mem);
-        } else {
-            new_array.set_scalar(i, old_value);
-        }
+        new_array.initialize(i, old_value, mem);
     }
 
     for i in old_size..new_size {
         new_array.set_scalar(i, Value::from_scalar(i + 1));
     }
+    allocation_barrier(TABLE);
 }
 
 pub unsafe fn table_initialized() -> bool {

rts/motoko-rts/src/gc.rs

@@ -1,5 +1,6 @@
 pub mod copying;
 pub mod generational;
+pub mod incremental;
 pub mod mark_compact;
 
 #[cfg(feature = "ic")]

rts/motoko-rts/src/gc/copying.rs

@@ -5,6 +5,12 @@ use crate::types::*;
 
 use motoko_rts_macros::ic_mem_fn;
 
+#[no_mangle]
+#[cfg(feature = "ic")]
+pub unsafe extern "C" fn initialize_copying_gc() {
+    crate::memory::ic::initialize_memory();
+}
+
 #[ic_mem_fn(ic_only)]
 unsafe fn schedule_copying_gc<M: Memory>(mem: &mut M) {
     // Half of the heap.

rts/motoko-rts/src/gc/generational.rs

@@ -7,7 +7,7 @@
 
 pub mod mark_stack;
 pub mod remembered_set;
-#[cfg(debug_assertions)]
+#[cfg(feature = "memory_check")]
 mod sanity_checks;
 pub mod write_barrier;
 
@@ -27,6 +27,12 @@ use motoko_rts_macros::ic_mem_fn;
 use self::mark_stack::{free_mark_stack, pop_mark_stack};
 use self::write_barrier::REMEMBERED_SET;
 
+#[ic_mem_fn(ic_only)]
+unsafe fn initialize_generational_gc<M: Memory>(mem: &mut M) {
+    crate::memory::ic::initialize_memory();
+    write_barrier::init_generational_write_barrier(mem);
+}
+
 #[ic_mem_fn(ic_only)]
 unsafe fn schedule_generational_gc<M: Memory>(mem: &mut M) {
     let limits = get_limits();
@@ -51,13 +57,10 @@ unsafe fn generational_gc<M: Memory>(mem: &mut M) {
     };
     let strategy = decide_strategy(&heap.limits);
 
-    #[cfg(debug_assertions)]
-    let forced_gc = strategy.is_none();
-
     let strategy = strategy.unwrap_or(Strategy::Young);
     let mut gc = GenerationalGC::new(heap, strategy);
 
-    #[cfg(debug_assertions)]
+    #[cfg(feature = "memory_check")]
     sanity_checks::verify_snapshot(&gc.heap, false);
 
     gc.run();
@@ -67,13 +70,12 @@ unsafe fn generational_gc<M: Memory>(mem: &mut M) {
     update_statistics(&old_limits, new_limits);
     update_strategy(strategy, new_limits);
 
-    #[cfg(debug_assertions)]
-    if !forced_gc {
-        sanity_checks::check_memory(&gc.heap.limits, &gc.heap.roots);
-        sanity_checks::take_snapshot(&mut gc.heap);
-    }
+    #[cfg(feature = "memory_check")]
+    sanity_checks::check_memory(&gc.heap.limits, &gc.heap.roots);
+    #[cfg(feature = "memory_check")]
+    sanity_checks::take_snapshot(&mut gc.heap);
 
-    write_barrier::init_write_barrier(gc.heap.mem);
+    write_barrier::init_generational_write_barrier(gc.heap.mem);
 }
 
 #[cfg(feature = "ic")]

rts/motoko-rts/src/gc/generational/mark_stack.rs

@@ -26,6 +26,7 @@ pub unsafe fn alloc_mark_stack<M: Memory>(mem: &mut M) {
     assert!(STACK_BLOB_PTR.is_null());
 
     // Allocating an actual object here to not break dump_heap
+    // No post allocation barrier as this RTS-internal blob will be collected by the GC.
     STACK_BLOB_PTR = alloc_blob(mem, INIT_STACK_SIZE.to_bytes()).get_ptr() as *mut Blob;
     STACK_BASE = STACK_BLOB_PTR.payload_addr() as *mut usize;
     STACK_PTR = STACK_BASE;

rts/motoko-rts/src/gc/generational/remembered_set.rs

@@ -223,6 +223,7 @@ impl RememberedSetIterator {
 }
 
 unsafe fn new_table<M: Memory>(mem: &mut M, size: u32) -> *mut Blob {
+    // No post allocation barrier as this RTS-internal blob will be collected by the GC.
     let table = alloc_blob(mem, Bytes(size * size_of::<HashEntry>() as u32)).as_blob_mut();
     for index in 0..size {
         table_set(table, index, null_ptr_value());
@@ -232,6 +233,7 @@ unsafe fn new_table<M: Memory>(mem: &mut M, size: u32) -> *mut Blob {
 
 unsafe fn new_collision_node<M: Memory>(mem: &mut M, value: Value) -> *mut CollisionNode {
     debug_assert!(!is_null_ptr_value(value));
+    // No post allocation barrier as this RTS-internal blob will be collected by the GC.
     let node =
         alloc_blob(mem, Bytes(size_of::<HashEntry>() as u32)).as_blob_mut() as *mut CollisionNode;
     (*node).entry = HashEntry {

rts/motoko-rts/src/gc/generational/sanity_checks.rs

@@ -17,6 +17,7 @@ static mut SNAPSHOT: *mut Blob = null_mut();
 /// Take a memory snapshot. To be initiated after GC run.
 pub unsafe fn take_snapshot<M: Memory>(heap: &mut Heap<M>) {
     let length = Bytes(heap.limits.free as u32);
+    // No post allocation barrier as this RTS-internal blob will be collected by the GC.
     let blob = alloc_blob(heap.mem, length).get_ptr() as *mut Blob;
     memcpy_bytes(blob.payload_addr() as usize, 0, length);
     SNAPSHOT = blob;

rts/motoko-rts/src/gc/generational/write_barrier.rs

@@ -9,22 +9,22 @@ pub static mut REMEMBERED_SET: Option<RememberedSet> = None;
 pub static mut HEAP_BASE: u32 = 0;
 pub static mut LAST_HP: u32 = 0;
 
+#[cfg(feature = "ic")]
 /// (Re-)initialize the write barrier for generational GC.
-#[ic_mem_fn(ic_only)]
-pub unsafe fn init_write_barrier<M: Memory>(mem: &mut M) {
+pub(crate) unsafe fn init_generational_write_barrier<M: Memory>(mem: &mut M) {
     use crate::memory::ic;
     REMEMBERED_SET = Some(RememberedSet::new(mem));
     HEAP_BASE = ic::get_aligned_heap_base();
     LAST_HP = ic::LAST_HP;
 }
 
-/// Write barrier to be called AFTER the pointer store, used for generational GC.
+/// Write barrier to be called AFTER the pointer store, used for the generational GC.
 /// `location`: location of modified pointer (address of object field or array element).
 ///
 /// As the barrier is called after the write, `*location` refers to the NEW value.
-/// No effect is the write barrier is deactivated.
+/// No effect if the write barrier is deactivated.
 #[ic_mem_fn]
-pub unsafe fn write_barrier<M: Memory>(mem: &mut M, location: u32) {
+pub unsafe fn post_write_barrier<M: Memory>(mem: &mut M, location: u32) {
     // Must be an unskewed address.
     debug_assert_eq!(location & 0b1, 0);
     // Checks have been optimized according to the frequency of occurrence.
@@ -33,6 +33,7 @@ pub unsafe fn write_barrier<M: Memory>(mem: &mut M, location: u32) {
         // Nested ifs are more efficient when counting instructions on IC (explicit return counts as an instruction).
         let value = *(location as *mut Value);
         if value.points_to_or_beyond(LAST_HP as usize) {
+            #[allow(clippy::collapsible_if)]
             if location >= HEAP_BASE {
                 // Trap pointers that lead from old generation (or static roots) to young generation.
                 REMEMBERED_SET

rts/motoko-rts/src/gc/incremental.rs

@@ -0,0 +1,18 @@
+use motoko_rts_macros::ic_mem_fn;
+
+#[cfg(feature = "ic")]
+use crate::memory::Memory;
+
+pub mod barriers;
+
+#[ic_mem_fn(ic_only)]
+unsafe fn initialize_incremental_gc<M: Memory>(_mem: &mut M) {
+    use crate::memory::ic;
+    ic::initialize_memory();
+}
+
+#[ic_mem_fn(ic_only)]
+unsafe fn schedule_incremental_gc<M: Memory>(_mem: &mut M) {}
+
+#[ic_mem_fn(ic_only)]
+unsafe fn incremental_gc<M: Memory>(_mem: &mut M) {}