Incremental GC

Changelog.md

@@ -1,5 +1,34 @@
 # Motoko compiler changelog
 
+* motoko (`moc`)
+
+  * **For beta testing:** Add a new _incremental_ GC, enabled with new moc flag `--incremental-gc` (#3837).
+    The incremental garbage collector is designed to scale for large program heap sizes.
+
+    The GC distributes its workload across multiple steps, called increments, that each pause the mutator 
+    (user's program) for only a limited amount of time. As a result, the GC work can fit within the instruction-limited 
+    IC messages, regardless of the heap size and the object structures.
+
+    According to GC benchmark measurements, the incremental GC is more efficient than the existing copying, compacting, 
+    and generational GC in the following regards:
+    * Scalability: Able to use the full heap space, 3x more object allocations on average.
+    * Shorter interruptions: The GC pause has a maximum limit that is up to 10x shorter.
+    * Lower runtimes: The number of executed instructions is reduced by 10% on average (compared to the copying GC).
+    * Less GC overhead: The amount of GC work in proportion to the user's program work drops by 10-16%.
+
+    The GC incurs a moderate memory overhead: The allocated WASM memory has been measured to be 9% higher 
+    on average compared to the copying GC, which is the current default GC.
+
+    To activate the incremental GC under `dfx`, the following command-line argument needs to be specified in `dfx.json`:
+
+    ```
+    ...
+      "type" : "motoko"
+      ...
+      "args" : "--incremental-gc"
+    ...
+    ```
+
 ## 0.8.8 (2023-05-02)
 
 * motoko (`moc`)

default.nix

@@ -361,12 +361,7 @@ rec {
       (test_subdir dir deps).overrideAttrs (args: {
           EXTRA_MOC_ARGS = "--sanity-checks";
       });
-
-    compacting_gc_subdir = dir: deps:
-      (test_subdir dir deps).overrideAttrs (args: {
-          EXTRA_MOC_ARGS = "--compacting-gc";
-      });
-
+
     generational_gc_subdir = dir: deps:
       (test_subdir dir deps).overrideAttrs (args: {
           EXTRA_MOC_ARGS = "--generational-gc";
@@ -382,6 +377,11 @@ rec {
           EXTRA_MOC_ARGS = "--sanity-checks --generational-gc";
       });
 
+    snty_incremental_gc_subdir = dir: deps:
+      (test_subdir dir deps).overrideAttrs (args: {
+          EXTRA_MOC_ARGS = "--sanity-checks --incremental-gc";
+      });
+
     perf_subdir = dir: deps:
       (test_subdir dir deps).overrideAttrs (args: {
         checkPhase = ''
@@ -497,6 +497,7 @@ rec {
       drun-dbg   = snty_subdir "run-drun"   [ moc nixpkgs.drun ];
       drun-compacting-gc = snty_compacting_gc_subdir "run-drun" [ moc nixpkgs.drun ] ;
       drun-generational-gc = snty_generational_gc_subdir "run-drun" [ moc nixpkgs.drun ] ;
+      drun-incremental-gc = snty_incremental_gc_subdir "run-drun" [ moc nixpkgs.drun ] ;
       fail       = test_subdir "fail"       [ moc ];
       repl       = test_subdir "repl"       [ moc ];
       ld         = test_subdir "ld"         ([ mo-ld ] ++ ldTestDeps);

doc/md/compiler-ref.md

@@ -34,6 +34,7 @@ You can use the following options with the `moc` command.
 | `-help`,`--help`                          | Displays usage information.                                                                                                                           |
 | `--hide-warnings`                         | Hides compiler warnings.                                                                                                                              |
 | `-Werror`                                 | Treat warnings as errors.                                                                                                                             |
+| `--incremental-gc`                        | Use incremental GC (only for beta-testing)                                                                                                            |
 | `--idl`                                   | Compile binary and emit Candid IDL specification to `.did` file.                                                                                      |
 | `-i`                                      | Runs the compiler in an interactive read–eval–print loop (REPL) shell so you can evaluate program execution (implies -r).                             |
 | `--map`                                   | Outputs a JavaScript source map.                                                                                                                      |

rts/motoko-rts-tests/src/gc.rs

@@ -8,24 +8,32 @@
 mod compacting;
 mod generational;
 mod heap;
+mod incremental;
 mod random;
 mod utils;
 
 use heap::MotokoHeap;
 use motoko_rts::gc::generational::remembered_set::RememberedSet;
 use motoko_rts::gc::generational::write_barrier::{LAST_HP, REMEMBERED_SET};
-use utils::{get_scalar_value, read_word, unskew_pointer, ObjectIdx, GC, GC_IMPLS, WORD_SIZE};
+use motoko_rts::gc::incremental::partitioned_heap::PARTITION_SIZE;
+use motoko_rts::gc::incremental::{
+    get_partitioned_heap, incremental_gc_state, reset_partitioned_heap,
+};
+use utils::{
+    get_scalar_value, make_pointer, read_word, unskew_pointer, ObjectIdx, GC, GC_IMPLS, WORD_SIZE,
+};
 
 use motoko_rts::gc::copying::copying_gc_internal;
-use motoko_rts::gc::generational::{GenerationalGC, Limits, Roots, Strategy};
+use motoko_rts::gc::generational::{GenerationalGC, Strategy};
+use motoko_rts::gc::incremental::IncrementalGC;
 use motoko_rts::gc::mark_compact::compacting_gc_internal;
 use motoko_rts::types::*;
 
 use std::fmt::Write;
 
 use fxhash::{FxHashMap, FxHashSet};
 
-use crate::gc::utils::make_pointer;
+use crate::memory::TestMemory;
 
 pub fn test() {
     println!("Testing garbage collection ...");
@@ -46,6 +54,7 @@ pub fn test() {
 
     compacting::test();
     generational::test();
+    incremental::test();
 }
 
 fn test_heaps() -> Vec<TestHeap> {
@@ -107,6 +116,7 @@ fn test_gcs(heap_descr: &TestHeap) {
             &heap_descr.continuation_table,
         );
     }
+    reset_incremental_gc();
 }
 
 fn test_gc(
@@ -116,7 +126,7 @@ fn test_gc(
     continuation_table: &[ObjectIdx],
 ) {
     let mut heap = MotokoHeap::new(refs, roots, continuation_table, gc);
-
+    initialize_gc_state(&mut heap, gc);
     // Check `create_dynamic_heap` sanity
     check_dynamic_heap(
         false, // before gc
@@ -127,6 +137,7 @@ fn test_gc(
         heap.heap_base_offset(),
         heap.heap_ptr_offset(),
         heap.continuation_table_ptr_offset(),
+        false,
     );
 
     for round in 0..3 {
@@ -136,18 +147,46 @@ fn test_gc(
         let heap_ptr_offset = heap.heap_ptr_offset();
         let continuation_table_ptr_offset = heap.continuation_table_ptr_offset();
         check_dynamic_heap(
-            check_all_reclaimed, // after gc
+            check_all_reclaimed, // check for unreachable objects
             refs,
             roots,
             continuation_table,
             &**heap.heap(),
             heap_base_offset,
             heap_ptr_offset,
             continuation_table_ptr_offset,
+            gc == GC::Incremental,
         );
     }
 }
 
+fn initialize_gc_state(heap: &mut MotokoHeap, gc: GC) {
+    unsafe {
+        match gc {
+            GC::Incremental => initialize_incremental_gc(heap),
+            _ => reset_partitioned_heap(),
+        }
+    }
+}
+
+unsafe fn initialize_incremental_gc(heap: &mut MotokoHeap) {
+    IncrementalGC::initialize(heap, heap.heap_base_address());
+    let allocation_size = heap.heap_ptr_address() - heap.heap_base_address();
+
+    // Synchronize the partitioned heap with one big combined allocation by starting from the base pointer as the heap pointer.
+    let result = get_partitioned_heap().allocate(heap, Bytes(allocation_size as u32).to_words());
+    // Check that the heap pointer (here equals base pointer) is unchanged, i.e. no partition switch has happened.
+    // This is a restriction in the unit test where `MotokoHeap` only supports contiguous bump allocation during initialization.
+    assert_eq!(result.get_ptr(), heap.heap_base_address());
+}
+
+fn reset_incremental_gc() {
+    let mut memory = TestMemory::new(Words(PARTITION_SIZE as u32));
+    unsafe {
+        IncrementalGC::initialize(&mut memory, 0);
+    }
+}
+
 /// Check the dynamic heap:
 ///
 /// - All (and in post-gc mode, only) reachable objects should be in the heap. Reachable objects
@@ -166,6 +205,7 @@ fn check_dynamic_heap(
     heap_base_offset: usize,
     heap_ptr_offset: usize,
     continuation_table_ptr_offset: usize,
+    incremental: bool,
 ) {
     let objects_map: FxHashMap<ObjectIdx, &[ObjectIdx]> = objects
         .iter()
@@ -198,52 +238,88 @@ fn check_dynamic_heap(
         let tag = read_word(heap, offset);
         offset += WORD_SIZE;
 
-        assert_eq!(tag, TAG_ARRAY);
-
-        let forward = read_word(heap, offset);
-        offset += WORD_SIZE;
-
-        assert_eq!(forward, make_pointer(address as u32));
-
-        let n_fields = read_word(heap, offset);
-        offset += WORD_SIZE;
+        if tag == TAG_ONE_WORD_FILLER {
+            assert!(incremental);
+        } else if tag == TAG_FREE_SPACE {
+            assert!(incremental);
+            let words = read_word(heap, offset) as usize;
+            offset += WORD_SIZE;
+            offset += words * WORD_SIZE;
+        } else {
+            let forward = read_word(heap, offset);
+            offset += WORD_SIZE;
 
-        // There should be at least one field for the index
-        assert!(n_fields >= 1);
+            let is_forwarded = forward != make_pointer(address as u32);
+
+            if incremental && tag == TAG_BLOB {
+                assert!(!is_forwarded);
+                // in-heap mark stack blobs
+                let length = read_word(heap, offset);
+                offset += WORD_SIZE + length as usize;
+            } else {
+                if incremental {
+                    assert!(tag == TAG_ARRAY || tag >= TAG_ARRAY_SLICE_MIN);
+                } else {
+                    assert_eq!(tag, TAG_ARRAY);
+                }
 
-        let object_idx = get_scalar_value(read_word(heap, offset));
-        offset += WORD_SIZE;
-        let old = seen.insert(object_idx, address);
-        if let Some(old) = old {
-            panic!(
-                "Object with index {} seen multiple times: {:#x}, {:#x}",
-                object_idx, old, address
-            );
-        }
+                if is_forwarded {
+                    assert!(incremental);
 
-        let object_expected_pointees = objects_map.get(&object_idx).unwrap_or_else(|| {
-            panic!("Object with index {} is not in the objects map", object_idx)
-        });
+                    let forward_offset = forward as usize - heap.as_ptr() as usize;
+                    let length = read_word(
+                        heap,
+                        forward_offset + size_of::<Obj>().to_bytes().as_usize(),
+                    );
 
-        for field_idx in 1..n_fields {
-            let field = read_word(heap, offset);
-            offset += WORD_SIZE;
-            // Get index of the object pointed by the field
-            let pointee_address = field.wrapping_add(1); // unskew
-            let pointee_offset = (pointee_address as usize) - (heap.as_ptr() as usize);
-            let pointee_idx_offset =
-                pointee_offset as usize + size_of::<Array>().to_bytes().as_usize(); // skip array header (incl. length)
-            let pointee_idx = get_scalar_value(read_word(heap, pointee_idx_offset));
-            let expected_pointee_idx = object_expected_pointees[(field_idx - 1) as usize];
-            assert_eq!(
-                pointee_idx,
-                expected_pointee_idx,
-                "Object with index {} points to {} in field {}, but expected to point to {}",
-                object_idx,
-                pointee_idx,
-                field_idx - 1,
-                expected_pointee_idx,
-            );
+                    // Skip stale object version that has been relocated during incremental GC.
+                    offset += length as usize * WORD_SIZE;
+                } else {
+                    let n_fields = read_word(heap, offset);
+                    offset += WORD_SIZE;
+
+                    // There should be at least one field for the index
+                    assert!(n_fields >= 1);
+
+                    let object_idx = get_scalar_value(read_word(heap, offset));
+                    offset += WORD_SIZE;
+
+                    let old = seen.insert(object_idx, address);
+                    if let Some(old) = old {
+                        panic!(
+                            "Object with index {} seen multiple times: {:#x}, {:#x}",
+                            object_idx, old, address
+                        );
+                    }
+
+                    let object_expected_pointees =
+                        objects_map.get(&object_idx).unwrap_or_else(|| {
+                            panic!("Object with index {} is not in the objects map", object_idx)
+                        });
+
+                    for field_idx in 1..n_fields {
+                        let field = read_word(heap, offset);
+                        offset += WORD_SIZE;
+                        // Get index of the object pointed by the field
+                        let pointee_address = field.wrapping_add(1); // unskew
+                        let pointee_offset = (pointee_address as usize) - (heap.as_ptr() as usize);
+                        let pointee_idx_offset =
+                            pointee_offset as usize + size_of::<Array>().to_bytes().as_usize(); // skip array header (incl. length)
+                        let pointee_idx = get_scalar_value(read_word(heap, pointee_idx_offset));
+                        let expected_pointee_idx =
+                            object_expected_pointees[(field_idx - 1) as usize];
+                        assert_eq!(
+                            pointee_idx,
+                            expected_pointee_idx,
+                            "Object with index {} points to {} in field {}, but expected to point to {}",
+                            object_idx,
+                            pointee_idx,
+                            field_idx - 1,
+                            expected_pointee_idx,
+                        );
+                    }
+                }
+            }
         }
     }
 
@@ -406,12 +482,12 @@ impl GC {
                     REMEMBERED_SET = Some(RememberedSet::new(heap));
                     LAST_HP = heap_1.last_ptr_address() as u32;
 
-                    let limits = Limits {
+                    let limits = motoko_rts::gc::generational::Limits {
                         base: heap_base as usize,
                         last_free: heap_1.last_ptr_address(),
                         free: heap_1.heap_ptr_address(),
                     };
-                    let roots = Roots {
+                    let roots = motoko_rts::gc::generational::Roots {
                         static_roots,
                         continuation_table_ptr_loc: continuation_table_ptr_address,
                     };
@@ -428,6 +504,19 @@ impl GC {
                 }
                 round >= 2
             }
+
+            GC::Incremental => unsafe {
+                const INCREMENTS_UNTIL_COMPLETION: usize = 16;
+                for _ in 0..INCREMENTS_UNTIL_COMPLETION {
+                    let roots = motoko_rts::gc::incremental::roots::Roots {
+                        static_roots,
+                        continuation_table_location: continuation_table_ptr_address,
+                    };
+                    IncrementalGC::instance(heap, incremental_gc_state())
+                        .empty_call_stack_increment(roots);
+                }
+                false
+            },
         }
     }
 }