Incremental GC (Intermediate Version)

default.nix

@@ -367,6 +367,11 @@ rec {
           EXTRA_MOC_ARGS = "--sanity-checks --compacting-gc";
       });
 
+    incremental_gc_subdir = dir: deps:
+      (test_subdir dir deps).overrideAttrs (args: {
+          EXTRA_MOC_ARGS = "--sanity-checks --incremental-gc";
+      });
+
     generational_gc_subdir = dir: deps:
       (test_subdir dir deps).overrideAttrs (args: {
           EXTRA_MOC_ARGS = "--sanity-checks --generational-gc";
@@ -484,11 +489,12 @@ rec {
       run-dbg    = snty_subdir "run"        [ moc ] ;
       ic-ref-run = test_subdir "run-drun"   [ moc ic-ref-run ];
       ic-ref-run-compacting-gc = compacting_gc_subdir "run-drun" [ moc ic-ref-run ] ;
-      ic-ref-run-generational-gc = generational_gc_subdir "run-drun" [ moc ic-ref-run ] ;
+      ic-ref-run-incremental-gc = incremental_gc_subdir "run-drun" [ moc ic-ref-run ] ;
       drun       = test_subdir "run-drun"   [ moc nixpkgs.drun ];
       drun-dbg   = snty_subdir "run-drun"   [ moc nixpkgs.drun ];
       drun-compacting-gc = compacting_gc_subdir "run-drun" [ moc nixpkgs.drun ] ;
       drun-generational-gc = generational_gc_subdir "run-drun" [ moc nixpkgs.drun ] ;
+      drun-incremental-gc = incremental_gc_subdir "run-drun" [ moc nixpkgs.drun ] ;
       fail       = test_subdir "fail"       [ moc ];
       repl       = test_subdir "repl"       [ moc ];
       ld         = test_subdir "ld"         ([ mo-ld ] ++ ldTestDeps);

doc/md/compiler-ref.md

@@ -34,8 +34,9 @@ You can use the following options with the `moc` command.
 | `-help`,`--help`                          | Displays usage information.                                                                                                                           |
 | `--hide-warnings`                         | Hides compiler warnings.                                                                                                                              |
 | `-Werror`                                 | Treat warnings as errors.                                                                                                                             |
-| `--idl`                                   | Compile binary and emit Candid IDL specification to `.did` file.                                                                                      |
 | `-i`                                      | Runs the compiler in an interactive read–eval–print loop (REPL) shell so you can evaluate program execution (implies -r).                             |
+| `--incremental-gc`                        | Use incremental GC                                                                                                                                    |
+| `--idl`                                   | Compile binary and emit Candid IDL specification to `.did` file.                                                                                      |
 | `--map`                                   | Outputs a JavaScript source map.                                                                                                                      |
 | `--max-stable-pages <n>`                  | Set maximum number of pages available for library `ExperimentStableMemory.mo` (default 65536).                                                        |
 | `-no-system-api`                          | Disables system API imports.                                                                                                                          |

rts/motoko-rts-tests/src/gc.rs

@@ -8,6 +8,7 @@
 mod compacting;
 mod generational;
 mod heap;
+mod incremental;
 mod random;
 mod utils;
 
@@ -17,7 +18,8 @@ use motoko_rts::gc::generational::write_barrier::{LAST_HP, REMEMBERED_SET};
 use utils::{get_scalar_value, read_word, unskew_pointer, ObjectIdx, GC, GC_IMPLS, WORD_SIZE};
 
 use motoko_rts::gc::copying::copying_gc_internal;
-use motoko_rts::gc::generational::{GenerationalGC, Limits, Roots, Strategy};
+use motoko_rts::gc::generational::{GenerationalGC, Strategy};
+use motoko_rts::gc::incremental::IncrementalGC;
 use motoko_rts::gc::mark_compact::compacting_gc_internal;
 use motoko_rts::types::*;
 
@@ -26,6 +28,10 @@ use std::fmt::Write;
 use fxhash::{FxHashMap, FxHashSet};
 
 pub fn test() {
+    unsafe {
+        incremental::test();
+    }
+
     println!("Testing garbage collection ...");
 
     println!("  Testing pre-defined heaps...");
@@ -113,6 +119,11 @@ fn test_gc(
     roots: &[ObjectIdx],
     continuation_table: &[ObjectIdx],
 ) {
+    if gc == GC::Incremental {
+        unsafe {
+            IncrementalGC::<MotokoHeap>::initialize();
+        }
+    }
     let mut heap = MotokoHeap::new(refs, roots, continuation_table, gc);
 
     // Check `create_dynamic_heap` sanity
@@ -125,6 +136,7 @@ fn test_gc(
         heap.heap_base_offset(),
         heap.heap_ptr_offset(),
         heap.continuation_table_ptr_offset(),
+        false,
     );
 
     for round in 0..3 {
@@ -134,16 +146,22 @@ fn test_gc(
         let heap_ptr_offset = heap.heap_ptr_offset();
         let continuation_table_ptr_offset = heap.continuation_table_ptr_offset();
         check_dynamic_heap(
-            check_all_reclaimed, // after gc
+            check_all_reclaimed, // check for unreachable objects
             refs,
             roots,
             continuation_table,
             &**heap.heap(),
             heap_base_offset,
             heap_ptr_offset,
             continuation_table_ptr_offset,
+            gc == GC::Incremental,
         );
     }
+    if gc == GC::Incremental {
+        unsafe {
+            IncrementalGC::<MotokoHeap>::initialize();
+        }
+    }
 }
 
 /// Check the dynamic heap:
@@ -164,6 +182,7 @@ fn check_dynamic_heap(
     heap_base_offset: usize,
     heap_ptr_offset: usize,
     continuation_table_ptr_offset: usize,
+    incremental: bool,
 ) {
     let objects_map: FxHashMap<ObjectIdx, &[ObjectIdx]> = objects
         .iter()
@@ -193,49 +212,65 @@ fn check_dynamic_heap(
             continue;
         }
 
-        let tag = read_word(heap, offset);
-        offset += WORD_SIZE;
-
-        assert_eq!(tag, TAG_ARRAY);
-
-        let n_fields = read_word(heap, offset);
-        offset += WORD_SIZE;
+        let raw_tag = read_word(heap, offset);
+        if incremental && !is_marked(raw_tag) && raw_tag > TAG_FREE_BLOCK_MIN {
+            // skip free block
+            let size = Words(raw_tag - TAG_FREE_BLOCK_MIN);
+            offset += size.to_bytes().as_usize();
+        } else {
+            let tag = unmark(raw_tag);
+            offset += WORD_SIZE;
+            if incremental && tag == TAG_BLOB {
+                // in-heap mark stack blobs
+                let length = read_word(heap, offset);
+                offset += WORD_SIZE + length as usize;
+            } else {
+                if incremental {
+                    assert!(tag == TAG_ARRAY || tag >= TAG_ARRAY_SLICE_MIN);
+                } else {
+                    assert_eq!(tag, TAG_ARRAY);
+                }
 
-        // There should be at least one field for the index
-        assert!(n_fields >= 1);
+                let n_fields = read_word(heap, offset);
+                offset += WORD_SIZE;
 
-        let object_idx = get_scalar_value(read_word(heap, offset));
-        offset += WORD_SIZE;
-        let old = seen.insert(object_idx, address);
-        if let Some(old) = old {
-            panic!(
-                "Object with index {} seen multiple times: {:#x}, {:#x}",
-                object_idx, old, address
-            );
-        }
+                // There should be at least one field for the index
+                assert!(n_fields >= 1);
 
-        let object_expected_pointees = objects_map.get(&object_idx).unwrap_or_else(|| {
-            panic!("Object with index {} is not in the objects map", object_idx)
-        });
+                let object_idx = get_scalar_value(read_word(heap, offset));
+                offset += WORD_SIZE;
+                let old = seen.insert(object_idx, address);
+                if let Some(old) = old {
+                    panic!(
+                        "Object with index {} seen multiple times: {:#x}, {:#x}",
+                        object_idx, old, address
+                    );
+                }
 
-        for field_idx in 1..n_fields {
-            let field = read_word(heap, offset);
-            offset += WORD_SIZE;
-            // Get index of the object pointed by the field
-            let pointee_address = field.wrapping_add(1); // unskew
-            let pointee_offset = (pointee_address as usize) - (heap.as_ptr() as usize);
-            let pointee_idx_offset = pointee_offset as usize + 2 * WORD_SIZE; // skip header + length
-            let pointee_idx = get_scalar_value(read_word(heap, pointee_idx_offset));
-            let expected_pointee_idx = object_expected_pointees[(field_idx - 1) as usize];
-            assert_eq!(
-                pointee_idx,
-                expected_pointee_idx,
-                "Object with index {} points to {} in field {}, but expected to point to {}",
-                object_idx,
-                pointee_idx,
-                field_idx - 1,
-                expected_pointee_idx,
-            );
+                let object_expected_pointees = objects_map.get(&object_idx).unwrap_or_else(|| {
+                    panic!("Object with index {} is not in the objects map", object_idx)
+                });
+
+                for field_idx in 1..n_fields {
+                    let field = read_word(heap, offset);
+                    offset += WORD_SIZE;
+                    // Get index of the object pointed by the field
+                    let pointee_address = field.wrapping_add(1); // unskew
+                    let pointee_offset = (pointee_address as usize) - (heap.as_ptr() as usize);
+                    let pointee_idx_offset = pointee_offset as usize + 2 * WORD_SIZE; // skip header + length
+                    let pointee_idx = get_scalar_value(read_word(heap, pointee_idx_offset));
+                    let expected_pointee_idx = object_expected_pointees[(field_idx - 1) as usize];
+                    assert_eq!(
+                        pointee_idx,
+                        expected_pointee_idx,
+                        "Object with index {} points to {} in field {}, but expected to point to {}",
+                        object_idx,
+                        pointee_idx,
+                        field_idx - 1,
+                        expected_pointee_idx,
+                    );
+                }
+            }
         }
     }
 
@@ -317,7 +352,7 @@ fn compute_reachable_objects(
 }
 
 fn check_continuation_table(mut offset: usize, continuation_table: &[ObjectIdx], heap: &[u8]) {
-    assert_eq!(read_word(heap, offset), TAG_ARRAY);
+    assert_eq!(unmark(read_word(heap, offset)), TAG_ARRAY);
     offset += WORD_SIZE;
 
     assert_eq!(read_word(heap, offset), continuation_table.len() as u32);
@@ -394,12 +429,12 @@ impl GC {
                     REMEMBERED_SET = Some(RememberedSet::new(heap));
                     LAST_HP = heap_1.last_ptr_address() as u32;
 
-                    let limits = Limits {
+                    let limits = motoko_rts::gc::generational::Limits {
                         base: heap_base as usize,
                         last_free: heap_1.last_ptr_address(),
                         free: heap_1.heap_ptr_address(),
                     };
-                    let roots = Roots {
+                    let roots = motoko_rts::gc::generational::Roots {
                         static_roots,
                         continuation_table_ptr_loc: continuation_table_ptr_address,
                     };
@@ -416,6 +451,22 @@ impl GC {
                 }
                 round >= 2
             }
+
+            GC::Incremental => unsafe {
+                const INCREMENTS_UNTIL_COMPLETION: usize = 16;
+                for _ in 0..INCREMENTS_UNTIL_COMPLETION {
+                    let limits = motoko_rts::gc::incremental::Limits {
+                        base: heap_base as usize,
+                        free: heap_1.heap_ptr_address(),
+                    };
+                    let roots = motoko_rts::gc::incremental::Roots {
+                        static_roots,
+                        continuation_table: *continuation_table_ptr_address,
+                    };
+                    IncrementalGC::instance(heap).empty_call_stack_increment(limits, roots);
+                }
+                false
+            },
         }
     }
 }

rts/motoko-rts-tests/src/gc/generational.rs

@@ -1,8 +1,10 @@
 mod mark_stack;
+mod remembered_set;
 
 pub fn test() {
-    println!("Testing generational GC components ...");
+    println!("Testing generational GC ...");
     unsafe {
         mark_stack::test();
+        remembered_set::test();
     }
 }

rts/motoko-rts-tests/src/gc/generational/mark_stack.rs

@@ -10,7 +10,7 @@ use motoko_rts::types::*;
 use proptest::test_runner::{Config, TestCaseError, TestCaseResult, TestRunner};
 
 pub unsafe fn test() {
-    println!("  Testing generational GC mark stack ...");
+    println!("  Testing mark stack ...");
 
     test_push_pop();
     test_grow_stack();

rts/motoko-rts-tests/src/remembered_set.rs → rts/motoko-rts-tests/src/gc/generational/remembered_set.rs

@@ -9,7 +9,7 @@ use motoko_rts::types::{Value, Words};
 const GROW_LIMIT: u32 = INITIAL_TABLE_LENGTH * OCCUPATION_THRESHOLD_PERCENT / 100;
 
 pub unsafe fn test() {
-    println!("Testing remembered set ...");
+    println!("  Testing remembered set ...");
 
     test_remembered_set(0);
     test_remembered_set(1);
@@ -31,8 +31,6 @@ unsafe fn test_remembered_set(amount: u32) {
 }
 
 unsafe fn test_insert_iterate(amount: u32) {
-    println!("  Testing insert/iterate {amount}");
-
     let mut mem = TestMemory::new(Words(2 * amount + 1024 * 1024));
 
     let mut remembered_set = RememberedSet::new(&mut mem);
@@ -54,8 +52,6 @@ unsafe fn test_insert_iterate(amount: u32) {
 }
 
 unsafe fn test_duplicates(amount: u32) {
-    println!("  Testing duplicates {amount}");
-
     let mut mem = TestMemory::new(Words(2 * amount + 1024 * 1024));
 
     let mut remembered_set = RememberedSet::new(&mut mem);
@@ -72,8 +68,6 @@ unsafe fn test_duplicates(amount: u32) {
 }
 
 unsafe fn test_collisions(amount: u32) {
-    println!("  Testing collisions {amount}");
-
     let mut mem = TestMemory::new(Words(2 * amount + 1024 * 1024));
 
     let mut remembered_set = RememberedSet::new(&mut mem);

rts/motoko-rts-tests/src/gc/heap.rs

@@ -20,7 +20,11 @@ pub struct MotokoHeap {
 }
 
 impl Memory for MotokoHeap {
-    unsafe fn alloc_words(&mut self, n: Words<u32>) -> Value {
+    unsafe fn allocate(&mut self, n: Words<u32>) -> Value {
+        self.grow_heap(n)
+    }
+
+    unsafe fn grow_heap(&mut self, n: Words<u32>) -> Value {
         self.inner.borrow_mut().alloc_words(n)
     }
 }
@@ -235,7 +239,7 @@ impl MotokoHeapInner {
         // MarkCompact assumes that the dynamic heap starts at a 32-byte multiple
         let realign = match gc {
             GC::Copying => 0,
-            GC::MarkCompact | GC::Generational => {
+            GC::MarkCompact | GC::Generational | GC::Incremental => {
                 (32 - (heap.as_ptr() as usize + static_heap_size_bytes) % 32) % 32
             }
         };
@@ -338,6 +342,10 @@ fn heap_size_for_gc(
             );
             size + ROUNDS * REMEMBERED_SET_MAXIMUM_SIZE
         }
+        GC::Incremental => {
+            const MARK_STACK_SIZE: usize = 1024 * 1024;
+            total_heap_size_bytes + MARK_STACK_SIZE
+        }
     }
 }
 

rts/motoko-rts-tests/src/gc/incremental.rs

@@ -0,0 +1,10 @@
+mod free_list;
+mod mark_bit;
+mod mark_stack;
+
+pub unsafe fn test() {
+    println!("Testing incremental GC ...");
+    free_list::test();
+    mark_bit::test();
+    mark_stack::test();
+}