diff --git a/rts/motoko-rts/src/continuation_table.rs b/rts/motoko-rts/src/continuation_table.rs index 225d9aacbe1..0d4dcbbda54 100644 --- a/rts/motoko-rts/src/continuation_table.rs +++ b/rts/motoko-rts/src/continuation_table.rs @@ -24,7 +24,8 @@ use crate::memory::{alloc_array, Memory}; use crate::rts_trap_with; -use crate::types::Value; +use crate::types::SkewedPtr; +use crate::write_barrier::write_barrier; use motoko_rts_macros::ic_mem_fn; @@ -92,8 +93,13 @@ pub unsafe fn remember_continuation(mem: &mut M, ptr: Value) -> u32 { let idx = FREE_SLOT; - FREE_SLOT = TABLE.as_array().get(idx).get_scalar(); - TABLE.as_array().set(idx, ptr); + let table = TABLE.as_array(); + + FREE_SLOT = (table.get(idx).0 >> 2) as u32; + + write_barrier(table.payload_addr().add(idx as usize) as usize); + table.set(idx, ptr); + N_CONTINUATIONS += 1; idx @@ -132,9 +138,13 @@ pub unsafe extern "C" fn recall_continuation(idx: u32) -> Value { rts_trap_with("recall_continuation: Continuation index out of range"); } - let ptr = TABLE.as_array().get(idx); + let table = TABLE.as_array(); + + let ptr = table.get(idx); + + write_barrier(table.payload_addr().add(idx as usize) as usize); + table.set(idx, SkewedPtr((FREE_SLOT << 2) as usize)); - TABLE.as_array().set(idx, Value::from_scalar(FREE_SLOT)); FREE_SLOT = idx; N_CONTINUATIONS -= 1; diff --git a/rts/motoko-rts/src/gc/copying.rs b/rts/motoko-rts/src/gc/copying.rs index 95672901f49..703d9a6678d 100644 --- a/rts/motoko-rts/src/gc/copying.rs +++ b/rts/motoko-rts/src/gc/copying.rs @@ -21,9 +21,13 @@ unsafe fn schedule_copying_gc(mem: &mut M) { unsafe fn copying_gc(mem: &mut M) { use crate::memory::ic; + let heap_base = ic::get_heap_base(); + + crate::write_barrier::check_heap_copy(heap_base, ic::HP); + copying_gc_internal( mem, - ic::get_heap_base(), + heap_base, // get_hp || ic::HP as usize, // set_hp @@ -36,6 +40,8 @@ unsafe fn copying_gc(mem: &mut M) { |reclaimed| ic::RECLAIMED += Bytes(u64::from(reclaimed.as_u32())), ); + crate::write_barrier::copy_heap(mem, heap_base, ic::HP); + ic::LAST_HP = ic::HP; } diff --git a/rts/motoko-rts/src/gc/mark_compact.rs b/rts/motoko-rts/src/gc/mark_compact.rs index 654908f4484..c908e6cf2ff 100644 --- a/rts/motoko-rts/src/gc/mark_compact.rs +++ b/rts/motoko-rts/src/gc/mark_compact.rs @@ -36,6 +36,10 @@ unsafe fn schedule_compacting_gc(mem: &mut M) { unsafe fn compacting_gc(mem: &mut M) { use crate::memory::ic; + let heap_base = ic::get_heap_base(); + + crate::write_barrier::check_heap_copy(heap_base, ic::HP); + compacting_gc_internal( mem, ic::get_aligned_heap_base(), @@ -51,6 +55,8 @@ unsafe fn compacting_gc(mem: &mut M) { |reclaimed| ic::RECLAIMED += Bytes(u64::from(reclaimed.as_u32())), ); + crate::write_barrier::copy_heap(mem, heap_base, ic::HP); + ic::LAST_HP = ic::HP; } diff --git a/rts/motoko-rts/src/lib.rs b/rts/motoko-rts/src/lib.rs index 9896d23b67f..899a5b17707 100644 --- a/rts/motoko-rts/src/lib.rs +++ b/rts/motoko-rts/src/lib.rs @@ -33,6 +33,7 @@ mod tommath_bindings; pub mod types; pub mod utf8; mod visitor; +mod write_barrier; use types::Bytes; diff --git a/rts/motoko-rts/src/text_iter.rs b/rts/motoko-rts/src/text_iter.rs index a8ccee05879..28550f5e049 100644 --- a/rts/motoko-rts/src/text_iter.rs +++ b/rts/motoko-rts/src/text_iter.rs @@ -13,7 +13,8 @@ use crate::memory::{alloc_array, Memory}; use crate::rts_trap_with; use crate::text::decode_code_point; -use crate::types::{Value, TAG_BLOB, TAG_CONCAT}; +use crate::types::{SkewedPtr, TAG_BLOB, TAG_CONCAT}; +use crate::write_barrier::write_barrier; use motoko_rts_macros::ic_mem_fn; @@ -104,17 +105,25 @@ pub unsafe fn text_iter_next(mem: &mut M, iter: Value) -> u32 { // If next one is a concat node re-use both the iterator and the todo objects (avoids // allocation) let concat = text.as_concat(); + + write_barrier(todo_array.payload_addr().add(TODO_TEXT_IDX as usize) as usize); todo_array.set(TODO_TEXT_IDX, (*concat).text2); iter_array.set(ITER_POS_IDX, Value::from_scalar(0)); let todo_addr = iter_array.payload_addr().add(ITER_TODO_IDX as usize); + + write_barrier(iter_array.payload_addr().add(ITER_BLOB_IDX as usize) as usize); iter_array.set(ITER_BLOB_IDX, find_leaf(mem, (*concat).text1, todo_addr)); + text_iter_next(mem, iter) } else { // Otherwise remove the entry from the chain debug_assert_eq!(text.tag(), TAG_BLOB); + + write_barrier(iter_array.payload_addr().add(ITER_BLOB_IDX as usize) as usize); iter_array.set(ITER_BLOB_IDX, text); iter_array.set(ITER_POS_IDX, Value::from_scalar(0)); iter_array.set(ITER_TODO_IDX, todo_array.get(TODO_LINK_IDX)); + text_iter_next(mem, iter) } } else { diff --git a/rts/motoko-rts/src/types.rs b/rts/motoko-rts/src/types.rs index b7d03bd9565..37fe04cafbb 100644 --- a/rts/motoko-rts/src/types.rs +++ b/rts/motoko-rts/src/types.rs @@ -716,3 +716,39 @@ pub(crate) unsafe fn object_size(obj: usize) -> Words { } } } + +pub(crate) unsafe fn tag_str(tag: Tag) -> &'static str { + match tag { + TAG_OBJECT => "Object", + + TAG_OBJ_IND => "Object indirection", + + TAG_ARRAY => "Array", + + TAG_BITS64 => "Bits64", + + TAG_MUTBOX => "MutBox", + + TAG_CLOSURE => "Closure", + + TAG_SOME => "Some", + + TAG_VARIANT => "Variant", + + TAG_BLOB => "Blob", + + TAG_FWD_PTR => "Forwarding pointer", + + TAG_BITS32 => "Bits32", + + TAG_BIGINT => "BitInt", + + TAG_CONCAT => "Concat", + + TAG_NULL => "Null", + + _ => { + rts_trap_with("tag_str: invalid object tag"); + } + } +} diff --git a/rts/motoko-rts/src/write_barrier.rs b/rts/motoko-rts/src/write_barrier.rs new file mode 100644 index 00000000000..b236d9fc5b6 --- /dev/null +++ b/rts/motoko-rts/src/write_barrier.rs @@ -0,0 +1,103 @@ +//! Write barrier implementation and testing utilities + +use crate::memory::{alloc_blob, Memory}; +use crate::types::*; +use crate::visitor::visit_pointer_fields; + +static mut UPDATED_FIELDS: [usize; 1024] = [0usize; 1024]; + +static mut N_UPDATED_FIELDS: usize = 0; + +static mut HEAP_COPY: *mut Blob = core::ptr::null_mut(); + +// loc: updated location (object field) +// new: value being written +// +// Called before writing the value, so `*loc` gives the old (current) value. +#[no_mangle] +pub unsafe extern "C" fn write_barrier(loc: usize) { + // Make sure we unskewed the object when calculating the field + assert_eq!(loc & 0b1, 0); + + assert!(N_UPDATED_FIELDS < 1024); + + // println!(100, "Write barrier recording {:#x}", loc); + + UPDATED_FIELDS[N_UPDATED_FIELDS] = loc; + N_UPDATED_FIELDS += 1; +} + +/// Copy the current heap, store it in heap as a blob. Update `HEAP_COPY_BLOB` to the location of +/// the blob. This copy will then be used to check write barrier correctness, by `check_heap_copy`. +pub unsafe fn copy_heap(mem: &mut M, heap_base: u32, hp: u32) { + let blob_size = Bytes(hp - heap_base); + let blob = alloc_blob(mem, blob_size).unskew() as *mut Blob; + let mut payload = blob.payload_addr() as *mut u32; + + let mut p = heap_base; + while p < hp { + *payload = *(p as *const u32); + payload = payload.add(1); + p += core::mem::size_of::() as u32; + } + + HEAP_COPY = blob; + N_UPDATED_FIELDS = 0; +} + +pub unsafe fn check_heap_copy(heap_base: u32, hp: u32) { + if HEAP_COPY.is_null() { + return; + } + + assert!((HEAP_COPY as u32) < hp); + + let mut copy_ptr = HEAP_COPY.payload_addr() as *mut u32; + + let mut p = heap_base; + while p < HEAP_COPY as u32 { + let obj_heap = p as *mut Obj; + let obj_copy = copy_ptr as *mut Obj; + let diff = obj_copy as usize - obj_heap as usize; + + let tag = obj_heap.tag(); + + assert_eq!(tag, obj_copy.tag()); + + visit_pointer_fields(obj_heap, tag, heap_base as usize, |obj_field_ptr| { + let copy_field_ptr = (obj_field_ptr as usize + diff) as *mut SkewedPtr; + + let obj_field_value = *obj_field_ptr; + let copy_field_value = *copy_field_ptr; + + if obj_field_value != copy_field_value { + // Field was updated, check that we called the write barrier + let mut found = false; + for updated_field_idx in 0..N_UPDATED_FIELDS { + if UPDATED_FIELDS[updated_field_idx] == obj_field_ptr as usize { + found = true; + break; + } + } + + if !found { + panic!( + "Updated field {:#x} of object {:#x} ({}) not found in \ + updated fields recorded by write barrier (heap base={:#x}, hp={:#x}, static={})", + obj_field_ptr as usize, + obj_heap as usize, + tag_str(tag), + heap_base, + hp, + (obj_heap as u32) < heap_base, + ); + } + } + }); + + let size: Words = object_size(obj_heap as usize); + + p += size.to_bytes().0; + copy_ptr = copy_ptr.add(size.0 as usize); + } +} diff --git a/src/codegen/compile.ml b/src/codegen/compile.ml index 32e2eeff241..df984e8a32e 100644 --- a/src/codegen/compile.ml +++ b/src/codegen/compile.ml @@ -925,14 +925,7 @@ module RTS = struct E.add_func_import env "rts" "init" [I32Type] []; E.add_func_import env "rts" "alloc_blob" [I32Type] [I32Type]; E.add_func_import env "rts" "alloc_array" [I32Type] [I32Type]; - E.add_func_import env "rts" "alloc_stream" [I32Type] [I32Type]; - E.add_func_import env "rts" "stream_write" [I32Type; I32Type; I32Type] []; - E.add_func_import env "rts" "stream_write_byte" [I32Type; I32Type] []; - E.add_func_import env "rts" "stream_write_text" [I32Type; I32Type] []; - E.add_func_import env "rts" "stream_split" [I32Type] [I32Type]; - E.add_func_import env "rts" "stream_shutdown" [I32Type] []; - E.add_func_import env "rts" "stream_reserve" [I32Type; I32Type] [I32Type]; - E.add_func_import env "rts" "stream_stable_dest" [I32Type; I64Type; I64Type] []; + E.add_func_import env "rts" "write_barrier" [I32Type] []; () end (* RTS *) @@ -6758,16 +6751,34 @@ module Var = struct G.nop, sr, G.i (LocalSet (nr i)) + | Some (HeapInd i) -> + G.i (LocalGet (nr i)) ^^ + compile_add_const ptr_unskew ^^ + compile_add_const (Int32.mul MutBox.field Heap.word_size) ^^ + E.call_import env "rts" "write_barrier" ^^ + G.i (LocalGet (nr i)), SR.Vanilla, Heap.store_field MutBox.field + | Some (HeapStatic ptr) -> - compile_unboxed_const ptr, - SR.Vanilla, + let (set_new_val, get_new_val) = new_local env "new_val" in + set_new_val ^^ + + compile_unboxed_const ptr ^^ + compile_add_const ptr_unskew ^^ + compile_add_const (Int32.mul MutBox.field Heap.word_size) ^^ + E.call_import env "rts" "write_barrier" ^^ + + compile_unboxed_const ptr ^^ + get_new_val ^^ Heap.store_field MutBox.field + | Some (Const _) -> fatal "set_val: %s is const" var + | Some (PublicMethod _) -> fatal "set_val: %s is PublicMethod" var + | None -> fatal "set_val: %s missing" var (* Stores the payload. Returns stack preparation code, and code that consumes the values from the stack *) @@ -8165,17 +8176,47 @@ let compile_load_field env typ name = let rec compile_lexp (env : E.t) ae lexp = (fun (code, sr, fill_code) -> G.(with_region lexp.at code, sr, with_region lexp.at fill_code)) @@ match lexp.it with - | VarLE var -> Var.set_val env ae var + | VarLE var -> + G.nop, + Var.set_val env ae var | IdxLE (e1, e2) -> - compile_exp_vanilla env ae e1 ^^ (* offset to array *) + let (set_field, get_field) = new_local env "field" in + let (set_new_value, get_new_value) = new_local env "new_value" in + + ( compile_exp_vanilla env ae e1 ^^ (* offset to array *) compile_exp_vanilla env ae e2 ^^ (* idx *) - Arr.idx_bigint env, + Arr.idx_bigint env ^^ + set_field + + , + set_new_value ^^ + + get_field ^^ + compile_add_const ptr_unskew ^^ + E.call_import env "rts" "write_barrier" ^^ + + get_field ^^ + get_new_value ^^ SR.Vanilla, store_ptr + ) | DotLE (e, n) -> - compile_exp_vanilla env ae e ^^ + let (set_field, get_field) = new_local env "field" in + let (set_new_value, get_new_value) = new_local env "new_value" in + + ( compile_exp_vanilla env ae e ^^ (* Only real objects have mutable fields, no need to branch on the tag *) - Object.idx env e.note.Note.typ n, + Object.idx env e.note.Note.typ n ^^ + set_field + , + set_new_value ^^ + + get_field ^^ + compile_add_const ptr_unskew ^^ + E.call_import env "rts" "write_barrier" ^^ + + get_field ^^ + get_new_value ^^ SR.Vanilla, store_ptr