Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/osa1/write_barriers' into luc/ge…
Browse files Browse the repository at this point in the history
…nerational_gc
  • Loading branch information
luc-blaeser committed Oct 20, 2022
1 parent 48957c7 commit f8dd690
Show file tree
Hide file tree
Showing 8 changed files with 234 additions and 22 deletions.
20 changes: 15 additions & 5 deletions rts/motoko-rts/src/continuation_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@

use crate::memory::{alloc_array, Memory};
use crate::rts_trap_with;
use crate::types::Value;
use crate::types::SkewedPtr;
use crate::write_barrier::write_barrier;

use motoko_rts_macros::ic_mem_fn;

Expand Down Expand Up @@ -92,8 +93,13 @@ pub unsafe fn remember_continuation<M: Memory>(mem: &mut M, ptr: Value) -> u32 {

let idx = FREE_SLOT;

FREE_SLOT = TABLE.as_array().get(idx).get_scalar();
TABLE.as_array().set(idx, ptr);
let table = TABLE.as_array();

FREE_SLOT = (table.get(idx).0 >> 2) as u32;

write_barrier(table.payload_addr().add(idx as usize) as usize);
table.set(idx, ptr);

N_CONTINUATIONS += 1;

idx
Expand Down Expand Up @@ -132,9 +138,13 @@ pub unsafe extern "C" fn recall_continuation(idx: u32) -> Value {
rts_trap_with("recall_continuation: Continuation index out of range");
}

let ptr = TABLE.as_array().get(idx);
let table = TABLE.as_array();

let ptr = table.get(idx);

write_barrier(table.payload_addr().add(idx as usize) as usize);
table.set(idx, SkewedPtr((FREE_SLOT << 2) as usize));

TABLE.as_array().set(idx, Value::from_scalar(FREE_SLOT));
FREE_SLOT = idx;

N_CONTINUATIONS -= 1;
Expand Down
8 changes: 7 additions & 1 deletion rts/motoko-rts/src/gc/copying.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,13 @@ unsafe fn schedule_copying_gc<M: Memory>(mem: &mut M) {
unsafe fn copying_gc<M: Memory>(mem: &mut M) {
use crate::memory::ic;

let heap_base = ic::get_heap_base();

crate::write_barrier::check_heap_copy(heap_base, ic::HP);

copying_gc_internal(
mem,
ic::get_heap_base(),
heap_base,
// get_hp
|| ic::HP as usize,
// set_hp
Expand All @@ -36,6 +40,8 @@ unsafe fn copying_gc<M: Memory>(mem: &mut M) {
|reclaimed| ic::RECLAIMED += Bytes(u64::from(reclaimed.as_u32())),
);

crate::write_barrier::copy_heap(mem, heap_base, ic::HP);

ic::LAST_HP = ic::HP;
}

Expand Down
6 changes: 6 additions & 0 deletions rts/motoko-rts/src/gc/mark_compact.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ unsafe fn schedule_compacting_gc<M: Memory>(mem: &mut M) {
unsafe fn compacting_gc<M: Memory>(mem: &mut M) {
use crate::memory::ic;

let heap_base = ic::get_heap_base();

crate::write_barrier::check_heap_copy(heap_base, ic::HP);

compacting_gc_internal(
mem,
ic::get_aligned_heap_base(),
Expand All @@ -51,6 +55,8 @@ unsafe fn compacting_gc<M: Memory>(mem: &mut M) {
|reclaimed| ic::RECLAIMED += Bytes(u64::from(reclaimed.as_u32())),
);

crate::write_barrier::copy_heap(mem, heap_base, ic::HP);

ic::LAST_HP = ic::HP;
}

Expand Down
1 change: 1 addition & 0 deletions rts/motoko-rts/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ mod tommath_bindings;
pub mod types;
pub mod utf8;
mod visitor;
mod write_barrier;

use types::Bytes;

Expand Down
11 changes: 10 additions & 1 deletion rts/motoko-rts/src/text_iter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
use crate::memory::{alloc_array, Memory};
use crate::rts_trap_with;
use crate::text::decode_code_point;
use crate::types::{Value, TAG_BLOB, TAG_CONCAT};
use crate::types::{SkewedPtr, TAG_BLOB, TAG_CONCAT};
use crate::write_barrier::write_barrier;

use motoko_rts_macros::ic_mem_fn;

Expand Down Expand Up @@ -104,17 +105,25 @@ pub unsafe fn text_iter_next<M: Memory>(mem: &mut M, iter: Value) -> u32 {
// If next one is a concat node re-use both the iterator and the todo objects (avoids
// allocation)
let concat = text.as_concat();

write_barrier(todo_array.payload_addr().add(TODO_TEXT_IDX as usize) as usize);
todo_array.set(TODO_TEXT_IDX, (*concat).text2);
iter_array.set(ITER_POS_IDX, Value::from_scalar(0));
let todo_addr = iter_array.payload_addr().add(ITER_TODO_IDX as usize);

write_barrier(iter_array.payload_addr().add(ITER_BLOB_IDX as usize) as usize);
iter_array.set(ITER_BLOB_IDX, find_leaf(mem, (*concat).text1, todo_addr));

text_iter_next(mem, iter)
} else {
// Otherwise remove the entry from the chain
debug_assert_eq!(text.tag(), TAG_BLOB);

write_barrier(iter_array.payload_addr().add(ITER_BLOB_IDX as usize) as usize);
iter_array.set(ITER_BLOB_IDX, text);
iter_array.set(ITER_POS_IDX, Value::from_scalar(0));
iter_array.set(ITER_TODO_IDX, todo_array.get(TODO_LINK_IDX));

text_iter_next(mem, iter)
}
} else {
Expand Down
36 changes: 36 additions & 0 deletions rts/motoko-rts/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -716,3 +716,39 @@ pub(crate) unsafe fn object_size(obj: usize) -> Words<u32> {
}
}
}

pub(crate) unsafe fn tag_str(tag: Tag) -> &'static str {
match tag {
TAG_OBJECT => "Object",

TAG_OBJ_IND => "Object indirection",

TAG_ARRAY => "Array",

TAG_BITS64 => "Bits64",

TAG_MUTBOX => "MutBox",

TAG_CLOSURE => "Closure",

TAG_SOME => "Some",

TAG_VARIANT => "Variant",

TAG_BLOB => "Blob",

TAG_FWD_PTR => "Forwarding pointer",

TAG_BITS32 => "Bits32",

TAG_BIGINT => "BitInt",

TAG_CONCAT => "Concat",

TAG_NULL => "Null",

_ => {
rts_trap_with("tag_str: invalid object tag");
}
}
}
103 changes: 103 additions & 0 deletions rts/motoko-rts/src/write_barrier.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
//! Write barrier implementation and testing utilities

use crate::memory::{alloc_blob, Memory};
use crate::types::*;
use crate::visitor::visit_pointer_fields;

static mut UPDATED_FIELDS: [usize; 1024] = [0usize; 1024];

static mut N_UPDATED_FIELDS: usize = 0;

static mut HEAP_COPY: *mut Blob = core::ptr::null_mut();

// loc: updated location (object field)
// new: value being written
//
// Called before writing the value, so `*loc` gives the old (current) value.
#[no_mangle]
pub unsafe extern "C" fn write_barrier(loc: usize) {
// Make sure we unskewed the object when calculating the field
assert_eq!(loc & 0b1, 0);

assert!(N_UPDATED_FIELDS < 1024);

// println!(100, "Write barrier recording {:#x}", loc);

UPDATED_FIELDS[N_UPDATED_FIELDS] = loc;
N_UPDATED_FIELDS += 1;
}

/// Copy the current heap, store it in heap as a blob. Update `HEAP_COPY_BLOB` to the location of
/// the blob. This copy will then be used to check write barrier correctness, by `check_heap_copy`.
pub unsafe fn copy_heap<M: Memory>(mem: &mut M, heap_base: u32, hp: u32) {
let blob_size = Bytes(hp - heap_base);
let blob = alloc_blob(mem, blob_size).unskew() as *mut Blob;
let mut payload = blob.payload_addr() as *mut u32;

let mut p = heap_base;
while p < hp {
*payload = *(p as *const u32);
payload = payload.add(1);
p += core::mem::size_of::<u32>() as u32;
}

HEAP_COPY = blob;
N_UPDATED_FIELDS = 0;
}

pub unsafe fn check_heap_copy(heap_base: u32, hp: u32) {
if HEAP_COPY.is_null() {
return;
}

assert!((HEAP_COPY as u32) < hp);

let mut copy_ptr = HEAP_COPY.payload_addr() as *mut u32;

let mut p = heap_base;
while p < HEAP_COPY as u32 {
let obj_heap = p as *mut Obj;
let obj_copy = copy_ptr as *mut Obj;
let diff = obj_copy as usize - obj_heap as usize;

let tag = obj_heap.tag();

assert_eq!(tag, obj_copy.tag());

visit_pointer_fields(obj_heap, tag, heap_base as usize, |obj_field_ptr| {
let copy_field_ptr = (obj_field_ptr as usize + diff) as *mut SkewedPtr;

let obj_field_value = *obj_field_ptr;
let copy_field_value = *copy_field_ptr;

if obj_field_value != copy_field_value {
// Field was updated, check that we called the write barrier
let mut found = false;
for updated_field_idx in 0..N_UPDATED_FIELDS {
if UPDATED_FIELDS[updated_field_idx] == obj_field_ptr as usize {
found = true;
break;
}
}

if !found {
panic!(
"Updated field {:#x} of object {:#x} ({}) not found in \
updated fields recorded by write barrier (heap base={:#x}, hp={:#x}, static={})",
obj_field_ptr as usize,
obj_heap as usize,
tag_str(tag),
heap_base,
hp,
(obj_heap as u32) < heap_base,
);
}
}
});

let size: Words<u32> = object_size(obj_heap as usize);

p += size.to_bytes().0;
copy_ptr = copy_ptr.add(size.0 as usize);
}
}
71 changes: 56 additions & 15 deletions src/codegen/compile.ml
Original file line number Diff line number Diff line change
Expand Up @@ -925,14 +925,7 @@ module RTS = struct
E.add_func_import env "rts" "init" [I32Type] [];
E.add_func_import env "rts" "alloc_blob" [I32Type] [I32Type];
E.add_func_import env "rts" "alloc_array" [I32Type] [I32Type];
E.add_func_import env "rts" "alloc_stream" [I32Type] [I32Type];
E.add_func_import env "rts" "stream_write" [I32Type; I32Type; I32Type] [];
E.add_func_import env "rts" "stream_write_byte" [I32Type; I32Type] [];
E.add_func_import env "rts" "stream_write_text" [I32Type; I32Type] [];
E.add_func_import env "rts" "stream_split" [I32Type] [I32Type];
E.add_func_import env "rts" "stream_shutdown" [I32Type] [];
E.add_func_import env "rts" "stream_reserve" [I32Type; I32Type] [I32Type];
E.add_func_import env "rts" "stream_stable_dest" [I32Type; I64Type; I64Type] [];
E.add_func_import env "rts" "write_barrier" [I32Type] [];
()

end (* RTS *)
Expand Down Expand Up @@ -6758,16 +6751,34 @@ module Var = struct
G.nop,
sr,
G.i (LocalSet (nr i))

| Some (HeapInd i) ->
G.i (LocalGet (nr i)) ^^
compile_add_const ptr_unskew ^^
compile_add_const (Int32.mul MutBox.field Heap.word_size) ^^
E.call_import env "rts" "write_barrier" ^^

G.i (LocalGet (nr i)),
SR.Vanilla,
Heap.store_field MutBox.field

| Some (HeapStatic ptr) ->
compile_unboxed_const ptr,
SR.Vanilla,
let (set_new_val, get_new_val) = new_local env "new_val" in
set_new_val ^^

compile_unboxed_const ptr ^^
compile_add_const ptr_unskew ^^
compile_add_const (Int32.mul MutBox.field Heap.word_size) ^^
E.call_import env "rts" "write_barrier" ^^

compile_unboxed_const ptr ^^
get_new_val ^^
Heap.store_field MutBox.field

| Some (Const _) -> fatal "set_val: %s is const" var

| Some (PublicMethod _) -> fatal "set_val: %s is PublicMethod" var

| None -> fatal "set_val: %s missing" var

(* Stores the payload. Returns stack preparation code, and code that consumes the values from the stack *)
Expand Down Expand Up @@ -8165,17 +8176,47 @@ let compile_load_field env typ name =
let rec compile_lexp (env : E.t) ae lexp =
(fun (code, sr, fill_code) -> G.(with_region lexp.at code, sr, with_region lexp.at fill_code)) @@
match lexp.it with
| VarLE var -> Var.set_val env ae var
| VarLE var ->
G.nop,
Var.set_val env ae var
| IdxLE (e1, e2) ->
compile_exp_vanilla env ae e1 ^^ (* offset to array *)
let (set_field, get_field) = new_local env "field" in
let (set_new_value, get_new_value) = new_local env "new_value" in

( compile_exp_vanilla env ae e1 ^^ (* offset to array *)
compile_exp_vanilla env ae e2 ^^ (* idx *)
Arr.idx_bigint env,
Arr.idx_bigint env ^^
set_field

,
set_new_value ^^

get_field ^^
compile_add_const ptr_unskew ^^
E.call_import env "rts" "write_barrier" ^^

get_field ^^
get_new_value ^^
SR.Vanilla,
store_ptr
)
| DotLE (e, n) ->
compile_exp_vanilla env ae e ^^
let (set_field, get_field) = new_local env "field" in
let (set_new_value, get_new_value) = new_local env "new_value" in

( compile_exp_vanilla env ae e ^^
(* Only real objects have mutable fields, no need to branch on the tag *)
Object.idx env e.note.Note.typ n,
Object.idx env e.note.Note.typ n ^^
set_field
,
set_new_value ^^

get_field ^^
compile_add_const ptr_unskew ^^
E.call_import env "rts" "write_barrier" ^^

get_field ^^
get_new_value ^^
SR.Vanilla,
store_ptr

Expand Down

0 comments on commit f8dd690

Please sign in to comment.