diff --git a/README.md b/README.md index 1336a9f1a..b7795da1b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ website](https://inko-lang.org/about/). Inko officially supports Linux, Mac OS, and Windows (when compiled with a MingW toolchain such as [MSYS2](http://www.msys2.org/)). Other Unix-like platforms such as the various BSDs should also work, but are not officially supported at -this time. +this time. Inko only supports 64-bits architectures. ## Requirements diff --git a/vm/Cargo.lock b/vm/Cargo.lock index 6024216ac..01bf10480 100644 --- a/vm/Cargo.lock +++ b/vm/Cargo.lock @@ -49,7 +49,7 @@ dependencies = [ [[package]] name = "autocfg" -version = "0.1.5" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -176,15 +176,6 @@ dependencies = [ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "crossbeam-deque" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "crossbeam-deque" version = "0.7.1" @@ -234,11 +225,6 @@ dependencies = [ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "either" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "env_logger" version = "0.6.2" @@ -346,7 +332,6 @@ dependencies = [ "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "siphasher 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "socket2 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", @@ -488,7 +473,7 @@ name = "num-integer" version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -497,7 +482,7 @@ name = "num-traits" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -585,7 +570,7 @@ name = "rand" version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -603,7 +588,7 @@ name = "rand_chacha" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -664,7 +649,7 @@ name = "rand_pcg" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -676,28 +661,6 @@ dependencies = [ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "rayon" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", - "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon-core 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rayon-core" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.10.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "rdrand" version = "0.4.0" @@ -979,7 +942,7 @@ dependencies = [ "checksum argon2rs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3f67b0b6a86dae6e67ff4ca2b6201396074996379fba2b92ff649126f37cb392" "checksum arrayvec 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b8d73f9beda665eaa98ab9e4f7442bd4e7de6652587de55b2525e52e29c1b0ba" "checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" -"checksum autocfg 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "22130e92352b948e7e82a49cdb0aa94f2211761117f29e052dd397c1ac33542b" +"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" "checksum backtrace 0.3.33 (registry+https://github.com/rust-lang/crates.io-index)" = "88fb679bc9af8fa639198790a77f52d345fe13656c08b43afa9424c206b731c6" "checksum backtrace-sys 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "82a830b4ef2d1124a711c71d263c5abdc710ef8e907bd508c88be475cebc422b" "checksum bindgen 0.49.2 (registry+https://github.com/rust-lang/crates.io-index)" = "846a1fba6535362a01487ef6b10f0275faa12e5c5d835c5c1c627aabc46ccbd6" @@ -994,13 +957,11 @@ dependencies = [ "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" "checksum crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa" -"checksum crossbeam-deque 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13" "checksum crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b18cd2e169ad86297e6bc0ad9aa679aee9daa4f19e8163860faf7c164e4f5a71" "checksum crossbeam-epoch 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fedcd6772e37f3da2a9af9bf12ebe046c0dfe657992377b4df982a2b54cd37a9" "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" "checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" -"checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" "checksum env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" @@ -1050,8 +1011,6 @@ dependencies = [ "checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" "checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" "checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -"checksum rayon 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a4b0186e22767d5b9738a05eab7c6ac90b15db17e5b5f9bd87976dd7d89a10a4" -"checksum rayon-core 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ebbe0df8435ac0c397d467b6cad6d25543d06e8a019ef3f6af3c384597515bd2" "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index f04baf883..b884b2c3a 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -3,6 +3,7 @@ name = "inko" version = "0.5.0" # VERSION authors = ["Yorick Peterse "] edition = "2018" +build = "build.rs" [features] default = [] @@ -19,7 +20,6 @@ test = false [dependencies] getopts = "^0.2" num_cpus = "^1.10" -rayon = "^1.0" parking_lot = "^0.7" fnv = "^1.0" time = "^0.1" diff --git a/vm/build.rs b/vm/build.rs new file mode 100644 index 000000000..95c7606e7 --- /dev/null +++ b/vm/build.rs @@ -0,0 +1,5 @@ +fn main() { + if !cfg!(target_arch = "x86_64") { + panic!("The Inko virtual machine requires a 64-bits architecture"); + } +} diff --git a/vm/src/binding.rs b/vm/src/binding.rs index 9cc0f8746..5c4487021 100644 --- a/vm/src/binding.rs +++ b/vm/src/binding.rs @@ -4,9 +4,8 @@ use crate::arc_without_weak::ArcWithoutWeak; use crate::block::Block; use crate::chunk::Chunk; -use crate::gc::work_list::WorkList; use crate::immix::copy_object::CopyObject; -use crate::object_pointer::ObjectPointer; +use crate::object_pointer::{ObjectPointer, ObjectPointerPointer}; use std::cell::UnsafeCell; pub struct Binding { @@ -92,18 +91,20 @@ impl Binding { unsafe { &mut *self.locals.get() } } - /// Pushes all pointers in this binding into the supplied vector. - pub fn push_pointers(&self, pointers: &mut WorkList) { + pub fn each_pointer(&self, mut callback: F) + where + F: FnMut(ObjectPointerPointer), + { let mut current = Some(self); while let Some(binding) = current { - pointers.push(binding.receiver.pointer()); + callback(binding.receiver.pointer()); for index in 0..binding.locals().len() { let local = &binding.locals()[index]; if !local.is_null() { - pointers.push(local.pointer()); + callback(local.pointer()); } } @@ -139,34 +140,12 @@ impl Binding { parent, }) } - - // Moves all pointers in this binding to the given heap. - pub fn move_pointers_to(&mut self, heap: &mut H) { - if let Some(ref mut bind) = self.parent { - bind.move_pointers_to(heap); - } - - { - let locals = self.locals_mut(); - - for index in 0..locals.len() { - let pointer = locals[index]; - - if !pointer.is_null() { - locals[index] = heap.move_object(pointer); - } - } - } - - self.receiver = heap.move_object(self.receiver); - } } #[cfg(test)] mod tests { use super::*; use crate::config::Config; - use crate::gc::work_list::WorkList; use crate::immix::global_allocator::GlobalAllocator; use crate::immix::local_allocator::LocalAllocator; use crate::object_pointer::ObjectPointer; @@ -295,7 +274,7 @@ mod tests { } #[test] - fn test_push_pointers() { + fn test_each_pointer() { let mut alloc = LocalAllocator::new(GlobalAllocator::with_rc(), &Config::new()); @@ -311,27 +290,29 @@ mod tests { binding2.parent = Some(binding1.clone()); binding2.set_local(0, local2); - let mut pointers = WorkList::new(); + let mut pointer_pointers = Vec::new(); - binding2.push_pointers(&mut pointers); + binding2.each_pointer(|ptr| pointer_pointers.push(ptr)); - assert!(*pointers.pop().unwrap().get() == receiver); - assert!(*pointers.pop().unwrap().get() == local2); + let pointers: Vec<_> = + pointer_pointers.into_iter().map(|x| *x.get()).collect(); - assert!(*pointers.pop().unwrap().get() == receiver); - assert!(*pointers.pop().unwrap().get() == local1); + assert_eq!(pointers.iter().filter(|x| **x == receiver).count(), 2); + assert!(pointers.contains(&local2)); + assert!(pointers.contains(&local1)); } #[test] - fn test_push_pointers_and_update() { + fn test_each_pointer_and_update() { let mut alloc = LocalAllocator::new(GlobalAllocator::with_rc(), &Config::new()); let mut binding = Binding::with_rc(1, alloc.allocate_empty()); - let mut pointers = WorkList::new(); + let mut pointers = Vec::new(); binding.set_local(0, alloc.allocate_empty()); - binding.push_pointers(&mut pointers); + + binding.each_pointer(|ptr| pointers.push(ptr)); while let Some(pointer_pointer) = pointers.pop() { let pointer = pointer_pointer.get_mut(); @@ -377,34 +358,6 @@ mod tests { assert_eq!(bind_copy_parent.receiver.float_value().unwrap(), 8.0); } - #[test] - fn test_move_pointers_to() { - let galloc = GlobalAllocator::with_rc(); - let mut alloc1 = LocalAllocator::new(galloc.clone(), &Config::new()); - let mut alloc2 = LocalAllocator::new(galloc, &Config::new()); - - let ptr1 = alloc1.allocate_without_prototype(object_value::float(5.0)); - let ptr2 = alloc1.allocate_without_prototype(object_value::float(2.0)); - let ptr3 = alloc1.allocate_without_prototype(object_value::float(8.0)); - - let mut src_bind1 = Binding::with_rc(1, ptr3); - let mut src_bind2 = Binding::with_rc(1, ptr3); - - src_bind2.parent = Some(src_bind1.clone()); - src_bind1.set_local(0, ptr1); - src_bind2.set_local(0, ptr2); - src_bind2.move_pointers_to(&mut alloc2); - - // The original pointers now point to empty objects. - assert!(ptr1.get().value.is_none()); - assert!(ptr2.get().value.is_none()); - assert!(ptr3.get().value.is_none()); - - assert_eq!(src_bind2.get_local(0).float_value().unwrap(), 2.0); - assert_eq!(src_bind1.get_local(0).float_value().unwrap(), 5.0); - assert_eq!(src_bind1.receiver.float_value().unwrap(), 8.0); - } - #[test] fn test_receiver_with_receiver() { let pointer = ObjectPointer::integer(5); diff --git a/vm/src/config.rs b/vm/src/config.rs index fc3f4feee..7e78ce508 100644 --- a/vm/src/config.rs +++ b/vm/src/config.rs @@ -8,6 +8,7 @@ //! of the VM to easily access these configuration details. use crate::immix::block::BLOCK_SIZE; use num_cpus; +use std::cmp::max; use std::env; use std::path::PathBuf; @@ -24,7 +25,6 @@ macro_rules! set_from_env { const DEFAULT_YOUNG_THRESHOLD: u32 = (8 * 1024 * 1024) / (BLOCK_SIZE as u32); const DEFAULT_MATURE_THRESHOLD: u32 = (16 * 1024 * 1024) / (BLOCK_SIZE as u32); -const DEFAULT_MAILBOX_THRESHOLD: u32 = 1; const DEFAULT_GROWTH_FACTOR: f64 = 1.5; const DEFAULT_GROWTH_THRESHOLD: f64 = 0.9; const DEFAULT_REDUCTIONS: usize = 1000; @@ -35,17 +35,25 @@ pub struct Config { pub directories: Vec, /// The number of primary process threads to run. + /// + /// This defaults to the number of CPU cores. pub primary_threads: usize, /// The number of blocking process threads to run. + /// + /// This defaults to the number of CPU cores. pub blocking_threads: usize, /// The number of garbage collector threads to run. + /// + /// This defaults to half the number of CPU cores, with a minimum of two. pub gc_threads: usize, - /// The number of threads to use for various generic parallel tasks such as - /// scanning stack frames during garbage collection. - pub generic_parallel_threads: usize, + /// The number of threads to run for tracing objects during garbage + /// collection. + /// + /// This defaults to half the number of CPU cores, with a minimum of two. + pub tracer_threads: usize, /// The number of reductions a process can perform before being suspended. /// Defaults to 1000. @@ -66,36 +74,34 @@ pub struct Config { /// should be used before increasing the heap size. pub heap_growth_threshold: f64, - /// The number of memory blocks that can be allocated before triggering a - /// mailbox collection. - pub mailbox_threshold: u32, - - /// The block allocation growth factor for the mailbox heap. - pub mailbox_growth_factor: f64, - - /// The percentage of memory in the mailbox heap that should be used before - /// increasing the size. - pub mailbox_growth_threshold: f64, + /// When enabled, GC timings will be printed to STDERR. + pub print_gc_timings: bool, } impl Config { pub fn new() -> Config { let cpu_count = num_cpus::get(); + // GC threads may consume a lot of CPU as they trace through objects. If + // we have N cores and spawn N threads, we would end up with many + // threads competing over CPU time. + // + // To keep this somewhat under control we limit both GC control and + // tracer threads to half the number of CPU cores. + let cpu_half = max(cpu_count / 2, 1); + Config { directories: Vec::new(), primary_threads: cpu_count, - gc_threads: cpu_count, blocking_threads: cpu_count, - generic_parallel_threads: cpu_count, + gc_threads: cpu_half, + tracer_threads: cpu_half, reductions: DEFAULT_REDUCTIONS, young_threshold: DEFAULT_YOUNG_THRESHOLD, mature_threshold: DEFAULT_MATURE_THRESHOLD, heap_growth_factor: DEFAULT_GROWTH_FACTOR, heap_growth_threshold: DEFAULT_GROWTH_THRESHOLD, - mailbox_threshold: DEFAULT_MAILBOX_THRESHOLD, - mailbox_growth_factor: DEFAULT_GROWTH_FACTOR, - mailbox_growth_threshold: DEFAULT_GROWTH_THRESHOLD, + print_gc_timings: false, } } @@ -105,10 +111,10 @@ impl Config { allow(cyclomatic_complexity, cognitive_complexity) )] pub fn populate_from_env(&mut self) { - set_from_env!(self, primary_threads, "CONCURRENCY", usize); - set_from_env!(self, blocking_threads, "CONCURRENCY", usize); - set_from_env!(self, gc_threads, "CONCURRENCY", usize); - set_from_env!(self, generic_parallel_threads, "CONCURRENCY", usize); + set_from_env!(self, primary_threads, "PRIMARY_THREADS", usize); + set_from_env!(self, blocking_threads, "BLOCKING_THREADS", usize); + set_from_env!(self, gc_threads, "GC_THREADS", usize); + set_from_env!(self, tracer_threads, "TRACER_THREADS", usize); set_from_env!(self, reductions, "REDUCTIONS", usize); @@ -123,21 +129,7 @@ impl Config { f64 ); - set_from_env!(self, mailbox_threshold, "MAILBOX_THRESHOLD", u32); - - set_from_env!( - self, - mailbox_growth_factor, - "MAILBOX_GROWTH_FACTOR", - f64 - ); - - set_from_env!( - self, - mailbox_growth_threshold, - "MAILBOX_GROWTH_THRESHOLD", - f64 - ); + set_from_env!(self, print_gc_timings, "PRINT_GC_TIMINGS", bool); } pub fn add_directory(&mut self, path: String) { @@ -162,7 +154,7 @@ mod tests { #[test] fn test_populate_from_env() { - env::set_var("INKO_CONCURRENCY", "42"); + env::set_var("INKO_PRIMARY_THREADS", "42"); env::set_var("INKO_HEAP_GROWTH_FACTOR", "4.2"); let mut config = Config::new(); @@ -170,7 +162,6 @@ mod tests { config.populate_from_env(); // Unset before any assertions may fail. - env::remove_var("INKO_CONCURRENCY"); env::remove_var("INKO_HEAP_GROWTH_FACTOR"); assert_eq!(config.primary_threads, 42); diff --git a/vm/src/execution_context.rs b/vm/src/execution_context.rs index 8a465ae46..336f2c7c1 100644 --- a/vm/src/execution_context.rs +++ b/vm/src/execution_context.rs @@ -5,9 +5,8 @@ use crate::binding::{Binding, RcBinding}; use crate::block::Block; use crate::compiled_code::CompiledCodePointer; -use crate::gc::work_list::WorkList; use crate::global_scope::GlobalScopePointer; -use crate::object_pointer::ObjectPointer; +use crate::object_pointer::{ObjectPointer, ObjectPointerPointer}; use crate::process::RcProcess; use crate::register::Register; @@ -168,24 +167,16 @@ impl ExecutionContext { } } - /// Returns pointers to all pointers stored in this context. - pub fn pointers(&self) -> WorkList { - let mut pointers = WorkList::new(); - - // We don't use chain() here since it may perform worse than separate - // for loops, and we want the garbage collector (which calls this - // method) to be as fast as possible. - // - // See https://github.com/rust-lang/rust/issues/38038 for more - // information. - self.binding.push_pointers(&mut pointers); - self.register.push_pointers(&mut pointers); + pub fn each_pointer(&self, mut callback: F) + where + F: FnMut(ObjectPointerPointer), + { + self.binding.each_pointer(|ptr| callback(ptr)); + self.register.each_pointer(|ptr| callback(ptr)); for pointer in &self.deferred_blocks { - pointers.push(pointer.pointer()); + callback(pointer.pointer()); } - - pointers } /// Returns the top-most parent binding of the current binding. @@ -376,7 +367,7 @@ mod tests { } #[test] - fn test_pointers() { + fn test_each_pointer() { let (_machine, block, _) = setup(); let mut context = ExecutionContext::from_block(&block, None); let pointer = ObjectPointer::new(0x1 as RawObjectPointer); @@ -386,11 +377,18 @@ mod tests { context.binding.set_local(0, pointer); context.add_defer(deferred); - let mut pointers = context.pointers(); + let mut pointer_pointers = Vec::new(); + + context.each_pointer(|ptr| pointer_pointers.push(ptr)); + + let pointers: Vec<_> = + pointer_pointers.into_iter().map(|x| *x.get()).collect(); + + assert_eq!(pointers.len(), 4); + assert_eq!(pointers.iter().filter(|x| **x == pointer).count(), 2); - assert!(pointers.pop().is_some()); - assert!(pointers.pop().is_some()); - assert!(pointers.pop().is_some()); + assert!(pointers.contains(&context.binding.receiver)); + assert!(pointers.contains(&deferred)); } #[test] diff --git a/vm/src/gc/collection.rs b/vm/src/gc/collection.rs new file mode 100644 index 000000000..daca385eb --- /dev/null +++ b/vm/src/gc/collection.rs @@ -0,0 +1,515 @@ +//! Types and methods for scheduling garbage collection of a process. +use crate::gc::statistics::{CollectionStatistics, TraceStatistics}; +use crate::gc::tracer::Pool; +use crate::mailbox::Mailbox; +use crate::process::RcProcess; +use crate::vm::state::State; +use std::thread; +use std::time::Instant; + +/// A garbage collection to perform. +pub struct Collection { + /// The process that is being garbage collected. + process: RcProcess, + + /// The time at which garbage collection started. + start_time: Instant, +} + +impl Collection { + pub fn new(process: RcProcess) -> Self { + Collection { + process, + start_time: Instant::now(), + } + } + + /// Starts garbage collecting the process. + pub fn perform(&self, vm_state: &State) -> CollectionStatistics { + // We must lock the mailbox before performing any work, as otherwise new + // objects may be allocated during garbage collection. + let local_data = self.process.local_data_mut(); + let mut mailbox = local_data.mailbox.lock(); + let collect_mature = self.process.should_collect_mature_generation(); + let move_objects = self.process.prepare_for_collection(collect_mature); + + let trace_stats = self.trace( + &mut mailbox, + move_objects, + collect_mature, + vm_state.config.tracer_threads, + ); + + self.process.reclaim_blocks(vm_state, collect_mature); + + // We drop the mutex guard before rescheduling so the process can + // immediately start receiving messages again, and so it can send itself + // messages. + drop(mailbox); + + vm_state.scheduler.schedule(self.process.clone()); + + let stats = CollectionStatistics { + duration: self.start_time.elapsed(), + trace: trace_stats, + }; + + if vm_state.config.print_gc_timings { + eprintln!( + "[{:#x}] GC (mature: {}) in {:?}, {} marked, {} promoted, {} evacuated", + self.process.identifier(), + collect_mature, + stats.duration, + stats.trace.marked, + stats.trace.promoted, + stats.trace.evacuated + ); + } + + stats + } + + /// Traces through and marks all reachable objects. + fn trace( + &self, + mailbox: &mut Mailbox, + move_objects: bool, + mature: bool, + concurrency: usize, + ) -> TraceStatistics { + let (pool, tracers) = Pool::new(self.process.clone(), concurrency); + + self.process.each_global_pointer(|ptr| pool.schedule(ptr)); + + mailbox.each_pointer(|ptr| pool.schedule(ptr)); + + if !mature { + self.process + .each_remembered_pointer(|ptr| pool.schedule(ptr)); + } + + for context in self.process.contexts() { + context.each_pointer(|ptr| pool.schedule(ptr)); + } + + let handles: Vec<_> = tracers + .into_iter() + .map(|tracer| { + thread::spawn(move || { + if move_objects { + tracer.trace_with_moving() + } else { + tracer.trace_without_moving() + } + }) + }) + .collect(); + + let stats = handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .fold(TraceStatistics::new(), |acc, curr| acc + curr); + + if mature { + self.process.prune_remembered_set(); + } + + stats + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::binding::Binding; + use crate::block::Block; + use crate::config::Config; + use crate::object::Object; + use crate::object_pointer::ObjectPointer; + use crate::object_value; + use crate::vm::state::State; + use crate::vm::test::setup; + + #[test] + fn test_perform() { + let (_machine, _block, process) = setup(); + let state = State::with_rc(Config::new(), &[]); + let pointer = process.allocate_empty(); + let collection = Collection::new(process.clone()); + + process.set_register(0, pointer); + + let stats = collection.perform(&state); + + assert_eq!(stats.trace.marked, 1); + assert_eq!(stats.trace.evacuated, 0); + assert_eq!(stats.trace.promoted, 0); + + assert!(pointer.is_marked()); + } + + #[test] + fn test_trace_trace_without_moving_without_mature() { + let (_machine, _block, process) = setup(); + let collection = Collection::new(process.clone()); + let young = process.allocate_empty(); + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + mature.mark(); + + process.set_register(0, young); + process.set_register(1, mature); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + false, + false, + 1, + ); + + assert_eq!(stats.marked, 1); + assert_eq!(stats.evacuated, 0); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_trace_trace_without_moving_with_mature() { + let (_machine, _block, process) = setup(); + let collection = Collection::new(process.clone()); + let young = process.allocate_empty(); + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + process.set_register(0, young); + process.set_register(1, mature); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + false, + true, + 1, + ); + + assert_eq!(stats.marked, 2); + assert_eq!(stats.evacuated, 0); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_trace_trace_with_moving_without_mature() { + let (_machine, _block, process) = setup(); + let collection = Collection::new(process.clone()); + let young = process.allocate_empty(); + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + mature.mark(); + + young.block_mut().set_fragmented(); + + process.set_register(0, young); + process.set_register(1, mature); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + true, + false, + 1, + ); + + assert_eq!(stats.marked, 1); + assert_eq!(stats.evacuated, 1); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_trace_trace_with_moving_with_mature() { + let (_machine, _block, process) = setup(); + let collection = Collection::new(process.clone()); + let young = process.allocate_empty(); + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + young.block_mut().set_fragmented(); + mature.block_mut().set_fragmented(); + + process.set_register(0, young); + process.set_register(1, mature); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + true, + true, + 1, + ); + + assert_eq!(stats.marked, 2); + assert_eq!(stats.evacuated, 2); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_trace_remembered_set_without_moving() { + let (_machine, _block, process) = setup(); + let collection = Collection::new(process.clone()); + let local_data = process.local_data_mut(); + let pointer1 = local_data + .allocator + .allocate_mature(Object::new(object_value::none())); + + local_data.allocator.remember_object(pointer1); + + process.prepare_for_collection(false); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + false, + false, + 1, + ); + + let remembered = + local_data.allocator.remembered_set.iter().next().unwrap(); + + assert!(remembered.is_marked()); + + assert_eq!(stats.marked, 1); + assert_eq!(stats.evacuated, 0); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_trace_remembered_set_with_moving() { + let (_machine, _block, process) = setup(); + let collection = Collection::new(process.clone()); + let local_data = process.local_data_mut(); + let pointer1 = local_data + .allocator + .allocate_mature(Object::new(object_value::float(4.5))); + + pointer1.block_mut().set_fragmented(); + + local_data.allocator.remember_object(pointer1); + + process.prepare_for_collection(false); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + true, + false, + 1, + ); + + let remembered = + local_data.allocator.remembered_set.iter().next().unwrap(); + + assert_eq!(remembered.block().is_fragmented(), false); + assert!(remembered.is_marked()); + assert!(remembered.float_value().is_ok()); + + assert_eq!(stats.marked, 1); + assert_eq!(stats.evacuated, 1); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_prune_remembered_set() { + let (_machine, _block, process) = setup(); + let collection = Collection::new(process.clone()); + let local_data = process.local_data_mut(); + + let pointer1 = local_data + .allocator + .allocate_mature(Object::new(object_value::none())); + + let pointer2 = local_data + .allocator + .allocate_mature(Object::new(object_value::none())); + + process.set_register(0, pointer2); + + local_data.allocator.remember_object(pointer1); + local_data.allocator.remember_object(pointer2); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + false, + true, + 1, + ); + + let mut iter = local_data.allocator.remembered_set.iter(); + + assert!(iter.next() == Some(&pointer2)); + assert!(iter.next().is_none()); + + assert_eq!(stats.marked, 1); + assert_eq!(stats.evacuated, 0); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_trace_mailbox_with_moving_without_mature() { + let (_machine, _block, process) = setup(); + let young = process.allocate_empty(); + let collection = Collection::new(process.clone()); + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + mature.mark(); + + young.block_mut().set_fragmented(); + + process.send_message_from_self(young); + process.send_message_from_self(mature); + process.prepare_for_collection(false); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + true, + false, + 1, + ); + + assert_eq!(stats.marked, 1); + assert_eq!(stats.evacuated, 1); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_trace_mailbox_with_moving_with_mature() { + let (_machine, _block, process) = setup(); + let young = process.allocate_empty(); + let collection = Collection::new(process.clone()); + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + young.block_mut().set_fragmented(); + + process.send_message_from_self(young); + process.send_message_from_self(mature); + process.prepare_for_collection(true); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + true, + true, + 1, + ); + + assert_eq!(stats.marked, 2); + assert_eq!(stats.evacuated, 1); + assert_eq!(stats.promoted, 0); + + assert!(mature.is_marked()); + } + + #[test] + fn test_trace_mailbox_without_moving_without_mature() { + let (_machine, _block, process) = setup(); + let young = process.allocate_empty(); + let collection = Collection::new(process.clone()); + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + mature.mark(); + + process.send_message_from_self(young); + process.send_message_from_self(mature); + process.prepare_for_collection(false); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + false, + false, + 1, + ); + + assert_eq!(stats.marked, 1); + assert_eq!(stats.evacuated, 0); + assert_eq!(stats.promoted, 0); + + assert!(young.is_marked()); + } + + #[test] + fn test_trace_mailbox_without_moving_with_mature() { + let (_machine, _block, process) = setup(); + let young = process.allocate_empty(); + let collection = Collection::new(process.clone()); + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + process.send_message_from_self(young); + process.send_message_from_self(mature); + process.prepare_for_collection(true); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + false, + true, + 1, + ); + + assert_eq!(stats.marked, 2); + assert_eq!(stats.evacuated, 0); + assert_eq!(stats.promoted, 0); + + assert!(young.is_marked()); + assert!(mature.is_marked()); + } + + #[test] + fn test_trace_with_panic_handler() { + let (_machine, block, process) = setup(); + let collection = Collection::new(process.clone()); + let local = process.allocate_empty(); + let receiver = process.allocate_empty(); + + let code = process.context().code.clone(); + let mut binding = Binding::with_rc(1, ObjectPointer::integer(1)); + + binding.set_local(0, local); + + let new_block = + Block::new(code, Some(binding), receiver, block.global_scope); + + let panic_handler = + process.allocate_without_prototype(object_value::block(new_block)); + + process.set_panic_handler(panic_handler); + process.prepare_for_collection(false); + + let stats = collection.trace( + &mut process.local_data_mut().mailbox.lock(), + false, + false, + 1, + ); + + assert_eq!(stats.marked, 3); + assert_eq!(stats.evacuated, 0); + assert_eq!(stats.promoted, 0); + + assert!(panic_handler.is_marked()); + assert!(receiver.is_marked()); + assert!(local.is_marked()); + } +} diff --git a/vm/src/gc/collector.rs b/vm/src/gc/collector.rs deleted file mode 100644 index 90562f636..000000000 --- a/vm/src/gc/collector.rs +++ /dev/null @@ -1,285 +0,0 @@ -//! Functions and macros for performing garbage collection. -use crate::gc::trace_result::TraceResult; -use crate::gc::work_list::WorkList; -use crate::object::ObjectStatus; -use crate::object_pointer::ObjectPointer; -use crate::process::RcProcess; - -/// Macro that returns true if the pointer can be skipped during tracing. -macro_rules! can_skip_pointer { - ($pointer:expr, $mature:expr) => { - $pointer.is_marked() || !$mature && $pointer.is_mature() - }; -} - -/// Promotes an object to the mature generation. -/// -/// The pointer to promote is updated to point to the new location. -pub fn promote_mature(process: &RcProcess, pointer: &mut ObjectPointer) { - { - let local_data = process.local_data_mut(); - let old_obj = pointer.get_mut(); - let new_pointer = local_data.allocator.allocate_mature(old_obj.take()); - - old_obj.forward_to(new_pointer); - } - - pointer.resolve_forwarding_pointer(); -} - -// Evacuates a pointer. -// -// The pointer to evacuate is updated to point to the new location. -pub fn evacuate(process: &RcProcess, pointer: &mut ObjectPointer) { - { - // When evacuating an object we must ensure we evacuate the object into - // the same bucket. - let local_data = process.local_data_mut(); - let bucket = pointer.block_mut().bucket_mut().unwrap(); - - let old_obj = pointer.get_mut(); - let new_obj = old_obj.take(); - - let (_, new_pointer) = - bucket.allocate(&local_data.allocator.global_allocator, new_obj); - - old_obj.forward_to(new_pointer); - } - - pointer.resolve_forwarding_pointer(); -} - -/// Traces through the given pointers, and potentially moves objects around. -pub fn trace_pointers_with_moving( - process: &RcProcess, - mut objects: WorkList, - mature: bool, -) -> TraceResult { - let mut marked = 0; - let mut evacuated = 0; - let mut promoted = 0; - - while let Some(pointer_pointer) = objects.pop() { - let pointer = pointer_pointer.get_mut(); - - if can_skip_pointer!(pointer, mature) { - continue; - } - - match pointer.status() { - ObjectStatus::Resolve => pointer.resolve_forwarding_pointer(), - ObjectStatus::Promote => { - promote_mature(process, pointer); - promoted += 1; - } - ObjectStatus::Evacuate => { - evacuate(process, pointer); - evacuated += 1; - } - ObjectStatus::PendingMove => { - objects.push(pointer_pointer.clone()); - continue; - } - _ => {} - } - - pointer.mark(); - - marked += 1; - - pointer.get().push_pointers(&mut objects); - } - - TraceResult::with(marked, evacuated, promoted) -} - -/// Traces through the roots and all their child pointers, without moving -/// objects around. -pub fn trace_pointers_without_moving( - mut objects: WorkList, - mature: bool, -) -> TraceResult { - let mut marked = 0; - - while let Some(pointer_pointer) = objects.pop() { - let pointer = pointer_pointer.get(); - - if can_skip_pointer!(pointer, mature) { - continue; - } - - pointer.mark(); - - marked += 1; - - pointer.get().push_pointers(&mut objects); - } - - TraceResult::with(marked, 0, 0) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::object::Object; - use crate::object_value; - use crate::vm::test::setup; - - #[test] - fn test_promote_mature() { - let (_machine, _block, process) = setup(); - - let mut pointer = - process.allocate_without_prototype(object_value::float(15.0)); - - let old_address = pointer.raw.raw as usize; - - promote_mature(&process, &mut pointer); - - let new_address = pointer.raw.raw as usize; - - assert!(old_address != new_address); - assert!(pointer.is_mature()); - assert_eq!(pointer.float_value().unwrap(), 15.0); - } - - #[test] - fn test_evacuate() { - let (_machine, _block, process) = setup(); - - let mut pointer = - process.allocate_without_prototype(object_value::float(15.0)); - - let old_address = pointer.raw.raw as usize; - - evacuate(&process, &mut pointer); - - let new_address = pointer.raw.raw as usize; - - assert!(old_address != new_address); - assert_eq!(pointer.float_value().unwrap(), 15.0); - } - - #[test] - fn test_trace_pointers_with_moving_without_mature() { - let (_machine, _block, process) = setup(); - - let young_parent = process.allocate_empty(); - let young_child = process.allocate_empty(); - - young_parent.add_attribute(&process, young_child, young_child); - - young_parent.block_mut().set_fragmented(); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - mature.block_mut().set_fragmented(); - - let mut pointers = WorkList::new(); - - pointers.push(young_parent.pointer()); - pointers.push(mature.pointer()); - - let result = trace_pointers_with_moving(&process, pointers, false); - - assert_eq!(mature.is_marked(), false); - - assert_eq!(result.marked, 2); - assert_eq!(result.evacuated, 2); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_trace_pointers_with_moving_with_mature() { - let (_machine, _block, process) = setup(); - - let young_parent = process.allocate_empty(); - let young_child = process.allocate_empty(); - - young_parent.add_attribute(&process, young_child, young_child); - - young_parent.block_mut().set_fragmented(); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - mature.block_mut().set_fragmented(); - - let mut pointers = WorkList::new(); - - pointers.push(young_parent.pointer()); - pointers.push(mature.pointer()); - - let result = trace_pointers_with_moving(&process, pointers, true); - - assert_eq!(result.marked, 3); - assert_eq!(result.evacuated, 3); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_trace_pointers_without_moving_without_mature() { - let (_machine, _block, process) = setup(); - - let young_parent = process.allocate_empty(); - let young_child = process.allocate_empty(); - - young_parent.add_attribute(&process, young_child, young_child); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - let mut pointers = WorkList::new(); - - pointers.push(young_parent.pointer()); - pointers.push(mature.pointer()); - - let result = trace_pointers_without_moving(pointers, false); - - assert!(young_parent.is_marked()); - assert!(young_child.is_marked()); - - assert_eq!(mature.is_marked(), false); - - assert_eq!(result.marked, 2); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_trace_pointers_without_moving_with_mature() { - let (_machine, _block, process) = setup(); - - let young_parent = process.allocate_empty(); - let young_child = process.allocate_empty(); - - young_parent.add_attribute(&process, young_child, young_child); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - let mut pointers = WorkList::new(); - - pointers.push(young_parent.pointer()); - pointers.push(mature.pointer()); - - let result = trace_pointers_without_moving(pointers, true); - - assert!(young_parent.is_marked()); - assert!(young_child.is_marked()); - assert!(mature.is_marked()); - - assert_eq!(result.marked, 3); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - } -} diff --git a/vm/src/gc/coordinator.rs b/vm/src/gc/coordinator.rs new file mode 100644 index 000000000..d4fcd2a1b --- /dev/null +++ b/vm/src/gc/coordinator.rs @@ -0,0 +1,180 @@ +//! Types for starting and coordinating garbage collecting of a process. +use crate::arc_without_weak::ArcWithoutWeak; +use crate::gc::collection::Collection; +use crate::scheduler::join_list::JoinList; +use crate::scheduler::pool_state::PoolState; +use crate::scheduler::queue::RcQueue; +use crate::scheduler::worker::Worker as WorkerTrait; +use crate::vm::state::RcState; +use std::thread; + +/// A worker used for coordinating the garbage collecting a process. +pub struct Worker { + /// The queue owned by this worker. + queue: RcQueue, + + /// The state of the pool this worker belongs to. + state: ArcWithoutWeak>, + + /// The VM state this worker belongs to. + vm_state: RcState, +} + +impl Worker { + pub fn new( + queue: RcQueue, + state: ArcWithoutWeak>, + vm_state: RcState, + ) -> Self { + Worker { + queue, + state, + vm_state, + } + } +} + +impl WorkerTrait for Worker { + fn state(&self) -> &PoolState { + &self.state + } + + fn queue(&self) -> &RcQueue { + &self.queue + } + + fn process_job(&mut self, job: Collection) { + job.perform(&self.vm_state); + } +} + +/// A pool of threads for coordinating the garbage collecting of processes. +pub struct Pool { + state: ArcWithoutWeak>, +} + +impl Pool { + pub fn new(threads: usize) -> Self { + assert!(threads > 0, "GC pools require at least a single thread"); + + Self { + state: ArcWithoutWeak::new(PoolState::new(threads)), + } + } + + /// Schedules a job onto the global queue. + pub fn schedule(&self, job: Collection) { + self.state.push_global(job); + } + + /// Informs this pool it should terminate as soon as possible. + pub fn terminate(&self) { + self.state.terminate(); + } + + /// Starts the pool, without blocking the calling thread. + pub fn start(&self, vm_state: RcState) -> JoinList<()> { + let handles = self + .state + .queues + .iter() + .enumerate() + .map(|(index, queue)| { + self.spawn_thread(index, queue.clone(), vm_state.clone()) + }) + .collect(); + + JoinList::new(handles) + } + + fn spawn_thread( + &self, + id: usize, + queue: RcQueue, + vm_state: RcState, + ) -> thread::JoinHandle<()> { + let state = self.state.clone(); + + thread::Builder::new() + .name(format!("GC {}", id)) + .spawn(move || { + Worker::new(queue, state, vm_state).run(); + }) + .unwrap() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + use crate::gc::collection::Collection; + use crate::vm::state::State; + use crate::vm::test::setup; + + fn worker() -> Worker { + let state = ArcWithoutWeak::new(PoolState::new(2)); + let vm_state = State::with_rc(Config::new(), &[]); + + Worker::new(state.queues[0].clone(), state, vm_state) + } + + #[test] + fn test_worker_run_global_jobs() { + let (_machine, _block, process) = setup(); + let mut worker = worker(); + + worker.state.push_global(Collection::new(process)); + worker.run(); + + assert_eq!(worker.state.has_global_jobs(), false); + } + + #[test] + fn test_worker_run_steal_then_terminate() { + let (_machine, _block, process) = setup(); + let mut worker = worker(); + + worker.state.queues[1].push_internal(Collection::new(process)); + worker.run(); + + assert_eq!(worker.state.queues[1].has_local_jobs(), false); + } + + #[test] + fn test_worker_run_steal_then_work() { + let (_machine, _block, process) = setup(); + let mut worker = worker(); + + worker.state.queues[1].push_internal(Collection::new(process.clone())); + worker.queue.push_internal(Collection::new(process)); + worker.run(); + + assert_eq!(worker.state.queues[1].has_local_jobs(), false); + assert_eq!(worker.queue.has_local_jobs(), false); + } + + #[test] + #[should_panic] + fn test_pool_new_with_zero_threads() { + Pool::new(0); + } + + #[test] + fn test_pool_spawn_thread() { + let (machine, _block, process) = setup(); + let pool = Pool::new(1); + + pool.schedule(Collection::new(process)); + + let thread = pool.spawn_thread( + 0, + pool.state.queues[0].clone(), + machine.state.clone(), + ); + + thread.join().unwrap(); + + assert_eq!(pool.state.has_global_jobs(), false); + } +} diff --git a/vm/src/gc/finished_collector.rs b/vm/src/gc/finished_collector.rs deleted file mode 100644 index f8ba9e5ea..000000000 --- a/vm/src/gc/finished_collector.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Functions for performing garbage collection of a finished process. - -use crate::gc::profile::Profile; -use crate::process::RcProcess; -use crate::vm::state::RcState; - -pub fn collect(vm_state: &RcState, process: &RcProcess, profile: &mut Profile) { - process.reclaim_and_finalize(vm_state); - profile.total.stop(); -} diff --git a/vm/src/gc/heap_collector.rs b/vm/src/gc/heap_collector.rs deleted file mode 100644 index 22f4900a3..000000000 --- a/vm/src/gc/heap_collector.rs +++ /dev/null @@ -1,683 +0,0 @@ -//! Functions for performing garbage collection of a process heap. - -use rayon::prelude::*; - -use crate::gc::collector; -use crate::gc::profile::Profile; -use crate::gc::trace_result::TraceResult; -use crate::process::RcProcess; -use crate::vm::state::RcState; - -pub fn collect(vm_state: &RcState, process: &RcProcess, profile: &mut Profile) { - let collect_mature = process.should_collect_mature_generation(); - - profile.prepare.start(); - - let move_objects = process.prepare_for_collection(collect_mature); - - profile.prepare.stop(); - profile.trace.start(); - - let trace_result = trace(process, move_objects, collect_mature); - - profile.trace.stop(); - profile.reclaim.start(); - - process.reclaim_blocks(vm_state, collect_mature); - process.update_collection_statistics(&vm_state.config, collect_mature); - - profile.reclaim.stop(); - - vm_state.scheduler.schedule(process.clone()); - - profile.suspended.stop(); - - profile.total.stop(); - profile.populate_tracing_statistics(&trace_result); -} - -/// Traces through and marks all reachable objects. -pub fn trace( - process: &RcProcess, - move_objects: bool, - mature: bool, -) -> TraceResult { - let mut result = if move_objects { - trace_mailbox_locals_with_moving(process, mature) - + trace_with_moving(process, mature) - } else { - trace_mailbox_locals_without_moving(process, mature) - + trace_without_moving(process, mature) - }; - - if mature { - prune_remembered_set(process); - } else if process.has_remembered_objects() { - result = result + trace_remembered_set(process, move_objects); - } - - result -} - -/// Traces through all pointers in the remembered set. -/// -/// Any young pointers found are promoted to the mature generation -/// immediately. This removes the need for keeping track of pointers in the -/// remembered set for a potential long amount of time. -/// -/// Returns true if any objects were promoted. -pub fn trace_remembered_set( - process: &RcProcess, - move_objects: bool, -) -> TraceResult { - let pointers = process.local_data().allocator.remembered_pointers(); - - if move_objects { - collector::trace_pointers_with_moving(process, pointers, true) - } else { - collector::trace_pointers_without_moving(pointers, true) - } -} - -/// Removes unmarked objects from the remembered set. -/// -/// During a mature collection we don't examine the remembered set since we -/// already traverse all mature objects. This allows us to remove any -/// unmarked mature objects from the remembered set. -pub fn prune_remembered_set(process: &RcProcess) { - process - .local_data_mut() - .allocator - .prune_remembered_objects(); -} - -/// Traces through all local pointers in a mailbox, without moving objects. -pub fn trace_mailbox_locals_without_moving( - process: &RcProcess, - mature: bool, -) -> TraceResult { - let local_data = process.local_data_mut(); - let objects = local_data.mailbox.local_pointers(); - - collector::trace_pointers_without_moving(objects, mature) -} - -/// Traces through all local pointers in a mailbox, potentially moving -/// objects. -pub fn trace_mailbox_locals_with_moving( - process: &RcProcess, - mature: bool, -) -> TraceResult { - let local_data = process.local_data_mut(); - let objects = local_data.mailbox.local_pointers(); - - collector::trace_pointers_with_moving(process, objects, mature) -} - -/// Traces through all objects without moving any. -pub fn trace_without_moving(process: &RcProcess, mature: bool) -> TraceResult { - let result = process - .contexts() - .par_iter() - .map(|context| { - collector::trace_pointers_without_moving(context.pointers(), mature) - }) - .reduce(TraceResult::new, |acc, curr| acc + curr); - - result - + collector::trace_pointers_without_moving( - process.global_pointers_to_trace(), - mature, - ) -} - -/// Traces through the roots and all their child pointers, potentially -/// moving objects around. -pub fn trace_with_moving(process: &RcProcess, mature: bool) -> TraceResult { - let result = process - .contexts() - .par_iter() - .map(|context| { - collector::trace_pointers_with_moving( - process, - context.pointers(), - mature, - ) - }) - .reduce(TraceResult::new, |acc, curr| acc + curr); - - result - + collector::trace_pointers_with_moving( - process, - process.global_pointers_to_trace(), - mature, - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::binding::Binding; - use crate::block::Block; - use crate::config::Config; - use crate::execution_context::ExecutionContext; - use crate::object::Object; - use crate::object_pointer::ObjectPointer; - use crate::object_value; - use crate::vm::state::State; - use crate::vm::test::setup; - - #[test] - fn test_collect() { - let (_machine, _block, process) = setup(); - let state = State::with_rc(Config::new(), &[]); - let pointer = process.allocate_empty(); - let mut profile = Profile::young(); - - process.set_register(0, pointer); - - collect(&state, &process, &mut profile); - - assert_eq!(profile.marked, 1); - assert_eq!(profile.evacuated, 0); - assert_eq!(profile.promoted, 0); - - assert!(pointer.is_marked()); - } - - #[test] - fn test_trace_trace_without_moving_without_mature() { - let (_machine, _block, process) = setup(); - - let young = process.allocate_empty(); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - process.set_register(0, young); - process.set_register(1, mature); - - let result = trace(&process, false, false); - - assert_eq!(result.marked, 1); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_trace_trace_without_moving_with_mature() { - let (_machine, _block, process) = setup(); - - let young = process.allocate_empty(); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - process.set_register(0, young); - process.set_register(1, mature); - - let result = trace(&process, false, true); - - assert_eq!(result.marked, 2); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_trace_trace_with_moving_without_mature() { - let (_machine, _block, process) = setup(); - - let young = process.allocate_empty(); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - young.block_mut().set_fragmented(); - - process.set_register(0, young); - process.set_register(1, mature); - - let result = trace(&process, true, false); - - assert_eq!(result.marked, 1); - assert_eq!(result.evacuated, 1); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_trace_trace_with_moving_with_mature() { - let (_machine, _block, process) = setup(); - - let young = process.allocate_empty(); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - young.block_mut().set_fragmented(); - mature.block_mut().set_fragmented(); - - process.set_register(0, young); - process.set_register(1, mature); - - let result = trace(&process, true, true); - - assert_eq!(result.marked, 2); - assert_eq!(result.evacuated, 2); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_trace_remembered_set_without_moving() { - let (_machine, _block, process) = setup(); - - let local_data = process.local_data_mut(); - - let pointer1 = local_data - .allocator - .allocate_mature(Object::new(object_value::none())); - - local_data.allocator.remember_object(pointer1); - - process.prepare_for_collection(false); - - let result = trace_remembered_set(&process, false); - - assert_eq!(result.marked, 1); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_trace_remembered_set_with_moving() { - let (_machine, _block, process) = setup(); - - let local_data = process.local_data_mut(); - - let pointer1 = local_data - .allocator - .allocate_mature(Object::new(object_value::none())); - - pointer1.block_mut().set_fragmented(); - - local_data.allocator.remember_object(pointer1); - - process.prepare_for_collection(false); - - let result = trace_remembered_set(&process, true); - - assert_eq!(result.marked, 1); - assert_eq!(result.evacuated, 1); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_prune_remembered_set() { - let (_machine, _block, process) = setup(); - - let local_data = process.local_data_mut(); - - let pointer1 = local_data - .allocator - .allocate_mature(Object::new(object_value::none())); - - let pointer2 = local_data - .allocator - .allocate_mature(Object::new(object_value::none())); - - pointer2.mark(); - - local_data.allocator.remember_object(pointer1); - local_data.allocator.remember_object(pointer2); - - prune_remembered_set(&process); - - assert_eq!( - local_data.allocator.remembered_set.contains(&pointer1), - false - ); - - assert!(local_data.allocator.remembered_set.contains(&pointer2)); - } - - #[test] - fn test_trace_mailbox_locals_with_moving_without_mature() { - let (_machine, _block, process) = setup(); - let young = process.allocate_empty(); - let local_data = process.local_data_mut(); - - let mature = local_data - .allocator - .allocate_mature(Object::new(object_value::none())); - - young.block_mut().set_fragmented(); - - local_data.mailbox.send_from_self(young); - local_data.mailbox.send_from_self(mature); - - process.prepare_for_collection(false); - - let result = trace_mailbox_locals_with_moving(&process, false); - - assert_eq!(result.marked, 1); - assert_eq!(result.evacuated, 1); - assert_eq!(result.promoted, 0); - - assert_eq!(mature.is_marked(), false); - } - - #[test] - fn test_trace_mailbox_locals_with_moving_with_mature() { - let (_machine, _block, process) = setup(); - let young = process.allocate_empty(); - let local_data = process.local_data_mut(); - - let mature = local_data - .allocator - .allocate_mature(Object::new(object_value::none())); - - young.block_mut().set_fragmented(); - - local_data.mailbox.send_from_self(young); - local_data.mailbox.send_from_self(mature); - - process.prepare_for_collection(true); - - let result = trace_mailbox_locals_with_moving(&process, true); - - assert_eq!(result.marked, 2); - assert_eq!(result.evacuated, 1); - assert_eq!(result.promoted, 0); - - assert!(mature.is_marked()); - } - - #[test] - fn test_trace_mailbox_locals_without_moving_without_mature() { - let (_machine, _block, process) = setup(); - let young = process.allocate_empty(); - let local_data = process.local_data_mut(); - - let mature = local_data - .allocator - .allocate_mature(Object::new(object_value::none())); - - local_data.mailbox.send_from_self(young); - local_data.mailbox.send_from_self(mature); - - process.prepare_for_collection(false); - - let result = trace_mailbox_locals_without_moving(&process, false); - - assert_eq!(result.marked, 1); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - - assert!(young.is_marked()); - assert_eq!(mature.is_marked(), false); - } - - #[test] - fn test_trace_mailbox_locals_without_moving_with_mature() { - let (_machine, _block, process) = setup(); - let young = process.allocate_empty(); - let local_data = process.local_data_mut(); - - let mature = local_data - .allocator - .allocate_mature(Object::new(object_value::none())); - - local_data.mailbox.send_from_self(young); - local_data.mailbox.send_from_self(mature); - - process.prepare_for_collection(true); - - let result = trace_mailbox_locals_without_moving(&process, true); - - assert_eq!(result.marked, 2); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - - assert!(young.is_marked()); - assert!(mature.is_marked()); - } - - #[test] - fn test_trace_without_moving_without_mature() { - let (_machine, block, process) = setup(); - let pointer1 = process.allocate_empty(); - let pointer2 = process.allocate_empty(); - let pointer3 = process.allocate_empty(); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - let receiver = process.allocate_empty(); - let code = process.context().code.clone(); - let new_block = Block::new(code, None, receiver, block.global_scope); - let mut context = ExecutionContext::from_block(&new_block, None); - - context.add_defer(pointer3); - process.set_register(0, pointer1); - process.push_context(context); - - process.set_register(0, pointer2); - process.set_register(1, mature); - - pointer1.block_mut().set_fragmented(); - - process.prepare_for_collection(false); - - let result = trace_without_moving(&process, false); - - assert_eq!(result.marked, 4); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - - assert_eq!(mature.is_marked(), false); - assert!(receiver.is_marked()); - assert!(pointer3.is_marked()); - } - - #[test] - fn test_trace_without_moving_with_panic_handler() { - let (_machine, block, process) = setup(); - let local = process.allocate_empty(); - let receiver = process.allocate_empty(); - - let code = process.context().code.clone(); - let mut binding = Binding::with_rc(1, ObjectPointer::integer(1)); - - binding.set_local(0, local); - - let new_block = - Block::new(code, Some(binding), receiver, block.global_scope); - - let panic_handler = - process.allocate_without_prototype(object_value::block(new_block)); - - process.set_panic_handler(panic_handler); - process.prepare_for_collection(false); - - let result = trace_without_moving(&process, false); - - assert_eq!(result.marked, 3); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - - assert!(panic_handler.is_marked()); - assert!(receiver.is_marked()); - assert!(local.is_marked()); - } - - #[test] - fn test_trace_with_moving_with_panic_handler() { - let (_machine, block, process) = setup(); - let local = process.allocate_empty(); - let receiver = process.allocate_empty(); - - let code = process.context().code.clone(); - let mut binding = Binding::with_rc(1, ObjectPointer::integer(1)); - - binding.set_local(0, local); - - let new_block = - Block::new(code, Some(binding), receiver, block.global_scope); - - let panic_handler = - process.allocate_without_prototype(object_value::block(new_block)); - - receiver.block_mut().set_fragmented(); - - process.set_panic_handler(panic_handler); - process.prepare_for_collection(false); - - let result = trace_with_moving(&process, false); - - assert_eq!(result.marked, 3); - assert_eq!(result.evacuated, 3); - assert_eq!(result.promoted, 0); - - { - let handler = process.panic_handler().unwrap(); - let block = handler.block_value().unwrap(); - - assert!(handler.is_marked()); - - assert!(block.receiver.is_marked()); - - assert!(block - .captures_from - .as_ref() - .unwrap() - .get_local(0) - .is_marked()); - } - } - - #[test] - fn test_trace_without_moving_with_mature() { - let (_machine, block, process) = setup(); - let pointer1 = process.allocate_empty(); - let pointer2 = process.allocate_empty(); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - let code = process.context().code.clone(); - let new_block = Block::new( - code, - None, - ObjectPointer::integer(1), - block.global_scope, - ); - - process.set_register(0, pointer1); - - process.push_context(ExecutionContext::from_block(&new_block, None)); - - process.set_register(0, pointer2); - process.set_register(1, mature); - - pointer1.block_mut().set_fragmented(); - - process.prepare_for_collection(true); - - let result = trace_without_moving(&process, true); - - assert_eq!(result.marked, 3); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - - assert!(mature.is_marked()); - } - - #[test] - fn test_trace_with_moving_without_mature() { - let (_machine, block, process) = setup(); - let pointer1 = process.allocate_empty(); - let pointer2 = process.allocate_empty(); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - let code = process.context().code.clone(); - let new_block = Block::new( - code, - None, - ObjectPointer::integer(1), - block.global_scope, - ); - - process.set_register(0, pointer1); - - process.push_context(ExecutionContext::from_block(&new_block, None)); - - process.set_register(0, pointer2); - process.set_register(1, mature); - - pointer1.block_mut().set_fragmented(); - - process.prepare_for_collection(false); - - let result = trace_with_moving(&process, false); - - assert_eq!(result.marked, 2); - assert_eq!(result.evacuated, 2); - assert_eq!(result.promoted, 0); - - assert_eq!(mature.is_marked(), false); - } - - #[test] - fn test_trace_with_moving_with_mature() { - let (_machine, block, process) = setup(); - let pointer1 = process.allocate_empty(); - let pointer2 = process.allocate_empty(); - - let mature = process - .local_data_mut() - .allocator - .allocate_mature(Object::new(object_value::none())); - - let code = process.context().code.clone(); - let new_block = Block::new( - code, - None, - ObjectPointer::integer(1), - block.global_scope, - ); - - process.set_register(0, pointer1); - - process.push_context(ExecutionContext::from_block(&new_block, None)); - - process.set_register(0, pointer2); - process.set_register(1, mature); - - pointer1.block_mut().set_fragmented(); - - process.prepare_for_collection(true); - - let result = trace_with_moving(&process, true); - - assert_eq!(result.marked, 3); - assert_eq!(result.evacuated, 2); - assert_eq!(result.promoted, 0); - - assert!(mature.is_marked()); - } -} diff --git a/vm/src/gc/mailbox_collector.rs b/vm/src/gc/mailbox_collector.rs deleted file mode 100644 index 11abfaaea..000000000 --- a/vm/src/gc/mailbox_collector.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! Functions for performing garbage collection of a process mailbox. -use crate::gc::collector; -use crate::gc::profile::Profile; -use crate::gc::trace_result::TraceResult; -use crate::mailbox::Mailbox; -use crate::process::RcProcess; -use crate::vm::state::RcState; - -pub fn collect(vm_state: &RcState, process: &RcProcess, profile: &mut Profile) { - let local_data = process.local_data_mut(); - let mailbox = &mut local_data.mailbox; - - profile.prepare.start(); - - let lock = mailbox.write_lock.lock(); - let move_objects = mailbox.allocator.prepare_for_collection(); - - profile.prepare.stop(); - profile.trace.start(); - - let trace_result = trace(&process, &mailbox, move_objects); - - profile.trace.stop(); - profile.reclaim.start(); - - mailbox.allocator.reclaim_blocks(vm_state); - process.update_mailbox_collection_statistics(&vm_state.config); - - drop(lock); // unlock as soon as possible - - profile.reclaim.stop(); - profile.suspended.stop(); - - vm_state.scheduler.schedule(process.clone()); - - profile.total.stop(); - profile.populate_tracing_statistics(&trace_result); -} - -pub fn trace( - process: &RcProcess, - mailbox: &Mailbox, - move_objects: bool, -) -> TraceResult { - let roots = unsafe { mailbox.mailbox_pointers() }; - - if move_objects { - collector::trace_pointers_with_moving(process, roots, false) - } else { - collector::trace_pointers_without_moving(roots, false) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::Config; - use crate::vm::state::State; - use crate::vm::test::setup; - - #[test] - fn test_collect() { - let (_machine, _block, process) = setup(); - let state = State::with_rc(Config::new(), &[]); - let mut profile = Profile::young(); - let local_data = process.local_data_mut(); - - local_data - .mailbox - .send_from_external(process.allocate_empty()); - - local_data.mailbox.allocator.prepare_for_collection(); - - collect(&state, &process, &mut profile); - - assert!(local_data - .mailbox - .external - .iter() - .next() - .unwrap() - .is_marked()); - - assert_eq!(profile.marked, 1); - assert_eq!(profile.evacuated, 0); - assert_eq!(profile.promoted, 0); - } - - #[test] - fn test_trace_without_moving() { - let (_machine, _block, process) = setup(); - - let local_data = process.local_data_mut(); - - local_data - .mailbox - .send_from_external(process.allocate_empty()); - - local_data.mailbox.allocator.prepare_for_collection(); - - let result = trace(&process, &local_data.mailbox, false); - - assert_eq!(result.marked, 1); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_trace_with_moving() { - let (_machine, _block, process) = setup(); - - let local_data = process.local_data_mut(); - - local_data - .mailbox - .send_from_external(process.allocate_empty()); - - local_data - .mailbox - .external - .iter_mut() - .next() - .unwrap() - .block_mut() - .set_fragmented(); - - local_data.mailbox.allocator.prepare_for_collection(); - - let result = trace(&process, &local_data.mailbox, true); - - assert_eq!(result.marked, 1); - assert_eq!(result.evacuated, 1); - assert_eq!(result.promoted, 0); - } -} diff --git a/vm/src/gc/mod.rs b/vm/src/gc/mod.rs index 6ba40ed13..a449cab97 100644 --- a/vm/src/gc/mod.rs +++ b/vm/src/gc/mod.rs @@ -1,8 +1,5 @@ -pub mod collector; -pub mod finished_collector; -pub mod heap_collector; -pub mod mailbox_collector; -pub mod profile; -pub mod request; -pub mod trace_result; -pub mod work_list; +pub mod collection; +pub mod coordinator; +pub mod remembered_set; +pub mod statistics; +pub mod tracer; diff --git a/vm/src/gc/profile.rs b/vm/src/gc/profile.rs deleted file mode 100644 index c170e7590..000000000 --- a/vm/src/gc/profile.rs +++ /dev/null @@ -1,132 +0,0 @@ -use crate::gc::trace_result::TraceResult; -use crate::timer::Timer; - -#[derive(Debug, Eq, PartialEq)] -pub enum CollectionType { - /// A young generation collection. - Young, - - /// A young + full collection. - Full, - - /// A mailbox collection. - Mailbox, - - /// A collection for a finished process. - Finished, -} - -pub struct Profile { - /// The type of garbage collection that was performed. - pub collection_type: CollectionType, - - /// The number of marked objects. - pub marked: usize, - - /// The number of evacuated objects. - pub evacuated: usize, - - /// The number of objects promoted to the full generation. - pub promoted: usize, - - /// The total garbage collection time. - pub total: Timer, - - /// The time spent preparing a collection. - pub prepare: Timer, - - /// The time spent tracing through live objects. - pub trace: Timer, - - /// The time spent reclaiming blocks. - pub reclaim: Timer, - - /// The total time the process was suspended. - pub suspended: Timer, -} - -impl Profile { - pub fn new(collection_type: CollectionType) -> Self { - Profile { - collection_type, - marked: 0, - evacuated: 0, - promoted: 0, - total: Timer::now(), - prepare: Timer::new(), - trace: Timer::new(), - reclaim: Timer::new(), - suspended: Timer::now(), - } - } - - pub fn young() -> Self { - Self::new(CollectionType::Young) - } - - pub fn full() -> Self { - Self::new(CollectionType::Full) - } - - pub fn mailbox() -> Self { - Self::new(CollectionType::Mailbox) - } - - pub fn finished() -> Self { - Self::new(CollectionType::Finished) - } - - pub fn populate_tracing_statistics(&mut self, result: &TraceResult) { - self.marked = result.marked; - self.evacuated = result.evacuated; - self.promoted = result.promoted; - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::gc::trace_result::TraceResult; - - #[test] - fn test_new() { - let profile = Profile::new(CollectionType::Young); - - assert_eq!(profile.collection_type, CollectionType::Young); - assert_eq!(profile.marked, 0); - assert_eq!(profile.evacuated, 0); - assert_eq!(profile.promoted, 0); - } - - #[test] - fn test_young() { - let profile = Profile::young(); - - assert_eq!(profile.collection_type, CollectionType::Young); - } - - #[test] - fn test_full() { - let profile = Profile::full(); - - assert_eq!(profile.collection_type, CollectionType::Full); - } - - #[test] - fn test_mailbox() { - let profile = Profile::mailbox(); - - assert_eq!(profile.collection_type, CollectionType::Mailbox); - } - - #[test] - fn test_populate_tracing_statistics() { - let mut profile = Profile::new(CollectionType::Young); - - profile.populate_tracing_statistics(&TraceResult::with(1, 2, 3)); - - assert_eq!(profile.marked, 1); - assert_eq!(profile.evacuated, 2); - assert_eq!(profile.promoted, 3); - } -} diff --git a/vm/src/gc/remembered_set.rs b/vm/src/gc/remembered_set.rs new file mode 100644 index 000000000..e4acce3c6 --- /dev/null +++ b/vm/src/gc/remembered_set.rs @@ -0,0 +1,411 @@ +//! Remembering of mature objects containing pointers to young objects. +use crate::object_pointer::ObjectPointer; +use parking_lot::Mutex; +use std::mem; +use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; + +/// The number of values that can be put into a chunk. +const CHUNK_VALUES: usize = 4; + +/// A single chunk of remembered pointers. +pub struct Chunk { + /// The next chunk in the remembered set. + next: Option>, + + /// The index for the next value in the chunk. + index: AtomicUsize, + + /// The values to store in this chunk. + values: [ObjectPointer; CHUNK_VALUES], +} + +impl Chunk { + fn boxed() -> (Box, *mut Chunk) { + let chunk = Chunk { + next: None, + index: AtomicUsize::new(0), + values: [ObjectPointer::null(); CHUNK_VALUES], + }; + + let boxed = Box::new(chunk); + let ptr = &*boxed as *const _ as *mut _; + + (boxed, ptr) + } + + /// Remembers a pointer in the current chunk. + /// + /// This method returns `true` if the pointer was remembered. + fn remember(&mut self, value: ObjectPointer) -> bool { + loop { + let index = self.index.load(Ordering::Acquire); + + if index == CHUNK_VALUES { + return false; + } + + let next = index + 1; + + if self.index.compare_and_swap(index, next, Ordering::AcqRel) + == index + { + self.values[index] = value; + + return true; + } + } + } + + /// Remembers a pointer without any synchronisaton. + unsafe fn remember_fast(&mut self, value: ObjectPointer) -> bool { + let index = *self.index.get_mut(); + + if index == CHUNK_VALUES { + return false; + } + + *self.index.get_mut() += 1; + self.values[index] = value; + + true + } +} + +/// A collection of pointers to mature objects that contain pointers to young +/// objects. +/// +/// Values can be added to a remembered set, and an iterator can be obtained to +/// iterate over these values. Removing individual values is not supported, +/// instead one must prune the entire remembered set. +pub struct RememberedSet { + /// The first chunk in the remembered set. + head: Box, + + /// A pointer to the last chunk in the remembered set. New values will be + /// allocated into this chunk. + tail: AtomicPtr, + + /// A lock used when allocating a new chunk. + lock: Mutex<()>, +} + +impl RememberedSet { + /// Creates a new remembered set with a single chunk. + pub fn new() -> RememberedSet { + let (head, tail) = Chunk::boxed(); + + RememberedSet { + head, + tail: AtomicPtr::new(tail), + lock: Mutex::new(()), + } + } + + /// Remembers a pointer in the remembered set. + /// + /// This method supports concurrent operations and does not require you to + /// use a lock of sorts. + pub fn remember(&self, value: ObjectPointer) { + loop { + let tail_ptr = self.tail.load(Ordering::Acquire); + let mut tail = unsafe { &mut *tail_ptr }; + + if tail.remember(value) { + return; + } + + let _lock = self.lock.lock(); + + if self.tail.load(Ordering::Acquire) != tail_ptr { + continue; + } + + let (chunk, new_tail_ptr) = Chunk::boxed(); + + tail.next = Some(chunk); + self.tail.store(new_tail_ptr, Ordering::Release); + } + } + + /// Remembers a pointer in the remembered set, without synchronisation. + unsafe fn remember_fast(&mut self, value: ObjectPointer) { + loop { + let tail_ptr = *self.tail.get_mut(); + let mut tail = &mut *tail_ptr; + + if tail.remember_fast(value) { + return; + } + + let (chunk, new_tail_ptr) = Chunk::boxed(); + + tail.next = Some(chunk); + *self.tail.get_mut() = new_tail_ptr; + } + } + + /// Returns an iterator over the pointers in the remembered set. + /// + /// This method takes a mutable reference to `self` as iteration can not + /// take place when the set is modified concurrently. + pub fn iter(&mut self) -> RememberedSetIterator { + RememberedSetIterator { + chunk: &*self.head, + index: 0, + } + } + + /// Prunes the remembered set by removing pointers to unmarked objects. + pub fn prune(&mut self) { + let (mut head, tail) = Chunk::boxed(); + + // After this `head` is the old head, and `self.head` will be an + // empty chunk. + mem::swap(&mut head, &mut self.head); + *self.tail.get_mut() = tail; + + let mut current = Some(head); + + while let Some(mut chunk) = current { + for value in &chunk.values { + if value.is_null() { + // Once we encounter a NULL value there can not be any + // non-NULL values that follow it. + break; + } + + if !value.is_marked() { + // Pointers that are not marked should no longer be + // remembered. + continue; + } + + unsafe { + self.remember_fast(*value); + } + } + + current = chunk.next.take(); + } + } + + /// Returns `true` if this RememberedSet is empty. + pub fn is_empty(&self) -> bool { + self.head.values[0].is_null() + } +} + +pub struct RememberedSetIterator<'a> { + chunk: &'a Chunk, + index: usize, +} + +impl<'a> Iterator for RememberedSetIterator<'a> { + type Item = &'a ObjectPointer; + + fn next(&mut self) -> Option<&'a ObjectPointer> { + if self.index == CHUNK_VALUES { + if let Some(chunk) = self.chunk.next.as_ref() { + self.chunk = chunk; + self.index = 0; + } else { + return None; + } + } + + let value = &self.chunk.values[self.index]; + + if value.is_null() { + None + } else { + self.index += 1; + + Some(value) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::arc_without_weak::ArcWithoutWeak; + use crate::immix::block::Block; + use crate::object_pointer::ObjectPointer; + use std::mem; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::thread; + + #[test] + fn test_remember_single_pointer() { + let mut rem_set = RememberedSet::new(); + + rem_set.remember(ObjectPointer::integer(4)); + + let mut iter = rem_set.iter(); + + assert!(iter.next() == Some(&ObjectPointer::integer(4))); + assert!(iter.next().is_none()); + } + + #[test] + fn test_remember_two_chunks_of_pointers() { + let mut rem_set = RememberedSet::new(); + + for i in 0..8 { + rem_set.remember(ObjectPointer::integer(i as i64)); + } + + let mut iter = rem_set.iter(); + + // We don't use a loop here so that test failures point to the right + // line. + assert!(iter.next() == Some(&ObjectPointer::integer(0))); + assert!(iter.next() == Some(&ObjectPointer::integer(1))); + assert!(iter.next() == Some(&ObjectPointer::integer(2))); + assert!(iter.next() == Some(&ObjectPointer::integer(3))); + + assert!(iter.next() == Some(&ObjectPointer::integer(4))); + assert!(iter.next() == Some(&ObjectPointer::integer(5))); + assert!(iter.next() == Some(&ObjectPointer::integer(6))); + assert!(iter.next() == Some(&ObjectPointer::integer(7))); + + assert!(iter.next().is_none()); + } + + #[test] + fn test_remember_with_threads() { + // This test is not super accurate, as any race conditions will likely + // be timing sensitive. However, it's the least we can do without + // relying on (potentially large) third-party libraries. + for _ in 0..128 { + let mut rem_set = ArcWithoutWeak::new(RememberedSet::new()); + let mut threads = Vec::with_capacity(2); + let wait = ArcWithoutWeak::new(AtomicBool::new(true)); + + for _ in 0..2 { + let rem_set_clone = rem_set.clone(); + let wait_clone = wait.clone(); + + threads.push(thread::spawn(move || { + while wait_clone.load(Ordering::Relaxed) { + // Spin... + } + + for i in 0..4 { + rem_set_clone.remember(ObjectPointer::integer(i)) + } + })); + } + + wait.store(false, Ordering::Relaxed); + + for thread in threads { + thread.join().unwrap(); + } + + assert_eq!(rem_set.iter().count(), 8); + + // 8 values fit in two chunks, and we only allocate the third chunk + // when reaching value 9. + assert!(rem_set.head.next.is_some()); + assert!(rem_set.head.next.as_ref().unwrap().next.is_none()); + } + } + + #[test] + fn test_chunk_memory_size() { + assert_eq!(mem::size_of::(), 48); + } + + #[test] + fn test_prune_remembered_set() { + let mut rem_set = RememberedSet::new(); + let mut block = Block::boxed(); + + let ptr1 = ObjectPointer::new(block.request_pointer().unwrap()); + let ptr2 = ObjectPointer::new(block.request_pointer().unwrap()); + let ptr3 = ObjectPointer::new(block.request_pointer().unwrap()); + + ptr1.mark(); + ptr2.mark(); + + rem_set.remember(ptr1); + rem_set.remember(ptr2); + rem_set.remember(ptr3); + rem_set.prune(); + + let mut iter = rem_set.iter(); + + assert!(iter.next() == Some(&ptr1)); + assert!(iter.next() == Some(&ptr2)); + assert!(iter.next().is_none()); + } + + #[test] + fn test_prune_remembered_set_with_two_chunks() { + let mut rem_set = RememberedSet::new(); + let mut block = Block::boxed(); + + let ptr1 = ObjectPointer::new(block.request_pointer().unwrap()); + let ptr2 = ObjectPointer::new(block.request_pointer().unwrap()); + let ptr3 = ObjectPointer::new(block.request_pointer().unwrap()); + let ptr4 = ObjectPointer::new(block.request_pointer().unwrap()); + let ptr5 = ObjectPointer::new(block.request_pointer().unwrap()); + + ptr1.mark(); + ptr2.mark(); + ptr3.mark(); + ptr4.mark(); + + rem_set.remember(ptr1); + rem_set.remember(ptr2); + rem_set.remember(ptr3); + rem_set.remember(ptr4); + rem_set.remember(ptr5); + rem_set.prune(); + + let mut iter = rem_set.iter(); + + assert!(iter.next() == Some(&ptr1)); + assert!(iter.next() == Some(&ptr2)); + assert!(iter.next() == Some(&ptr3)); + assert!(iter.next() == Some(&ptr4)); + assert!(iter.next().is_none()); + } + + #[test] + fn test_is_empty() { + let rem_set = RememberedSet::new(); + + assert!(rem_set.is_empty()); + + rem_set.remember(ObjectPointer::integer(1)); + + assert_eq!(rem_set.is_empty(), false); + } + + #[test] + fn test_update_forwarded_pointer() { + let mut rem_set = RememberedSet::new(); + let mut block = Block::boxed(); + + let ptr1 = ObjectPointer::new(block.request_pointer().unwrap()); + let ptr2 = ObjectPointer::new(block.request_pointer().unwrap()); + + rem_set.remember(ptr1); + ptr1.get_mut().forward_to(ptr2); + + // The idea of this test is thread A traces through a pointer in the + // remembered set, then updates it. If we then re-retrieve the same + // pointer we should get the new value. + rem_set + .iter() + .next() + .unwrap() + .pointer() + .get_mut() + .resolve_forwarding_pointer(); + + assert!(*rem_set.iter().next().unwrap() == ptr2); + } +} diff --git a/vm/src/gc/request.rs b/vm/src/gc/request.rs deleted file mode 100644 index cdd178437..000000000 --- a/vm/src/gc/request.rs +++ /dev/null @@ -1,148 +0,0 @@ -//! Garbage Collection Requests -//! -//! A garbage collection request specifies what to collect (a heap or mailbox), -//! and what process to collect. - -use crate::gc::finished_collector; -use crate::gc::heap_collector; -use crate::gc::mailbox_collector; -use crate::gc::profile::Profile; -use crate::process::RcProcess; -use crate::vm::state::RcState; - -pub enum CollectionType { - /// A request to collect the regular heap of a process. - Heap, - - /// A request to collect the mailbox heap of a process. - Mailbox, - - /// A request to collect a process after it finished. - Finished, -} - -pub struct Request { - pub vm_state: RcState, - pub collection_type: CollectionType, - pub process: RcProcess, - pub profile: Profile, -} - -impl Request { - pub fn new( - collection_type: CollectionType, - vm_state: RcState, - process: RcProcess, - ) -> Self { - let profile = match collection_type { - CollectionType::Heap => { - if process.should_collect_mature_generation() { - Profile::full() - } else { - Profile::young() - } - } - CollectionType::Mailbox => Profile::mailbox(), - CollectionType::Finished => Profile::finished(), - }; - - Request { - vm_state, - collection_type, - process, - profile, - } - } - - /// Returns a request for collecting a process' heap. - pub fn heap(vm_state: RcState, process: RcProcess) -> Self { - Self::new(CollectionType::Heap, vm_state, process) - } - - /// Returns a request for collecting a process' mailbox. - pub fn mailbox(vm_state: RcState, process: RcProcess) -> Self { - Self::new(CollectionType::Mailbox, vm_state, process) - } - - /// Returns a request for collecting all process data after it finished. - pub fn finished(vm_state: RcState, process: RcProcess) -> Self { - Self::new(CollectionType::Finished, vm_state, process) - } - - /// Performs the garbage collection request. - pub fn perform(&mut self) { - match self.collection_type { - CollectionType::Heap => heap_collector::collect( - &self.vm_state, - &self.process, - &mut self.profile, - ), - CollectionType::Mailbox => mailbox_collector::collect( - &self.vm_state, - &self.process, - &mut self.profile, - ), - CollectionType::Finished => finished_collector::collect( - &self.vm_state, - &self.process, - &mut self.profile, - ), - }; - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::Config; - use crate::vm::state::State; - use crate::vm::test::setup; - - #[test] - fn test_new() { - let (_machine, _block, process) = setup(); - let state = State::with_rc(Config::new(), &[]); - let request = Request::new(CollectionType::Heap, state, process); - - assert!(match request.collection_type { - CollectionType::Heap => true, - _ => false, - }); - } - - #[test] - fn test_heap() { - let (_machine, _block, process) = setup(); - let state = State::with_rc(Config::new(), &[]); - let request = Request::heap(state, process); - - assert!(match request.collection_type { - CollectionType::Heap => true, - _ => false, - }); - } - - #[test] - fn test_mailbox() { - let (_machine, _block, process) = setup(); - let state = State::with_rc(Config::new(), &[]); - let request = Request::mailbox(state, process); - - assert!(match request.collection_type { - CollectionType::Mailbox => true, - _ => false, - }); - } - - #[test] - fn test_perform() { - let (_machine, _block, process) = setup(); - let state = State::with_rc(Config::new(), &[]); - let mut request = Request::heap(state, process.clone()); - - process.set_register(0, process.allocate_empty()); - request.perform(); - - assert!(process.get_register(0).is_marked()); - } -} diff --git a/vm/src/gc/statistics.rs b/vm/src/gc/statistics.rs new file mode 100644 index 000000000..0fa391a09 --- /dev/null +++ b/vm/src/gc/statistics.rs @@ -0,0 +1,101 @@ +//! Types for storing garbage collection statistics. +use std::ops::{Add, AddAssign}; +use std::time::Duration; + +/// Statistics produced by a single thread tracing objects. +pub struct TraceStatistics { + /// The number of marked objects. + pub marked: usize, + + /// The number of promoted objects. + pub promoted: usize, + + /// The number of evacuated objects. + pub evacuated: usize, +} + +impl TraceStatistics { + pub fn new() -> Self { + TraceStatistics { + marked: 0, + promoted: 0, + evacuated: 0, + } + } +} + +impl Add for TraceStatistics { + type Output = TraceStatistics; + + fn add(self, other: Self::Output) -> Self::Output { + Self { + marked: self.marked + other.marked, + promoted: self.promoted + other.promoted, + evacuated: self.evacuated + other.evacuated, + } + } +} + +impl AddAssign for TraceStatistics { + fn add_assign(&mut self, other: Self) { + *self = Self { + marked: self.marked + other.marked, + promoted: self.promoted + other.promoted, + evacuated: self.evacuated + other.evacuated, + } + } +} + +/// Statistics about a single garbage collection. +pub struct CollectionStatistics { + /// The total time spent garbage collecting. + pub duration: Duration, + + /// The statistics produced by tracing objects. + pub trace: TraceStatistics, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_trace_statistics_add() { + let mut stat1 = TraceStatistics::new(); + let mut stat2 = TraceStatistics::new(); + + stat1.marked = 1; + stat1.promoted = 1; + stat1.evacuated = 1; + + stat2.marked = 1; + stat2.promoted = 1; + stat2.evacuated = 1; + + let stat3 = stat1 + stat2; + + assert_eq!(stat3.marked, 2); + assert_eq!(stat3.promoted, 2); + assert_eq!(stat3.evacuated, 2); + } + + #[test] + fn test_trace_statistics_add_assign() { + let mut stat1 = TraceStatistics::new(); + let mut stat2 = TraceStatistics::new(); + + stat1.marked = 1; + stat1.promoted = 1; + stat1.evacuated = 1; + + stat2.marked = 1; + stat2.promoted = 1; + stat2.evacuated = 1; + + stat1 += stat2; + + assert_eq!(stat1.marked, 2); + assert_eq!(stat1.promoted, 2); + assert_eq!(stat1.evacuated, 2); + } +} diff --git a/vm/src/gc/trace_result.rs b/vm/src/gc/trace_result.rs deleted file mode 100644 index c82b7c3a2..000000000 --- a/vm/src/gc/trace_result.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! Statistics produced by tracing object graphs. -use std::ops::Add; - -pub struct TraceResult { - /// The number of marked objects. - pub marked: usize, - - /// The number of objects that were evacuated. - pub evacuated: usize, - - /// The number of objects that were promoted to the mature generation. - pub promoted: usize, -} - -impl TraceResult { - pub fn new() -> Self { - Self::with(0, 0, 0) - } - - pub fn with(marked: usize, evacuated: usize, promoted: usize) -> Self { - TraceResult { - marked, - evacuated, - promoted, - } - } -} - -impl Add for TraceResult { - type Output = TraceResult; - - fn add(self, other: Self::Output) -> Self::Output { - TraceResult::with( - self.marked + other.marked, - self.evacuated + other.evacuated, - self.promoted + other.promoted, - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_new() { - let result = TraceResult::new(); - - assert_eq!(result.marked, 0); - assert_eq!(result.evacuated, 0); - assert_eq!(result.promoted, 0); - } - - #[test] - fn test_add() { - let a = TraceResult::with(5, 10, 15); - let b = TraceResult::with(1, 2, 3); - let c = a + b; - - assert_eq!(c.marked, 6); - assert_eq!(c.evacuated, 12); - assert_eq!(c.promoted, 18); - } -} diff --git a/vm/src/gc/tracer.rs b/vm/src/gc/tracer.rs new file mode 100644 index 000000000..068388aa4 --- /dev/null +++ b/vm/src/gc/tracer.rs @@ -0,0 +1,490 @@ +//! Tracing and marking of live objects. +use crate::arc_without_weak::ArcWithoutWeak; +use crate::gc::statistics::TraceStatistics; +use crate::object::ObjectStatus; +use crate::object_pointer::{ObjectPointer, ObjectPointerPointer}; +use crate::process::RcProcess; +use crossbeam_deque::{Injector, Steal, Stealer, Worker}; +use std::sync::atomic::{spin_loop_hint, AtomicUsize, Ordering}; + +/// The raw tracing loop, with the part for actually tracing and marking being +/// supplied as an argument. +/// +/// This macro is used to implement the two tracing loops (non-moving and +/// moving), without having to duplicate the code manually. +/// +/// The tracing loop terminates automatically once all workers run out of work, +/// and takes care of not terminating threads prematurely. +/// +/// Much of the work of this macro is delegated to separate methods, as +/// otherwise we'd end up with quite a few nested loops; which gets hard to +/// read. +macro_rules! trace_loop { + ($self: expr, $work: expr) => { + loop { + $self.steal_from_global(); + + $work; + + if $self.has_global_jobs() { + continue; + } + + $self.set_idle(); + + if $self.steal_from_worker() { + $self.set_busy(); + continue; + } + + // Depending on how fast a thread runs, a thread may reach this + // point while there is still work left to be done. For example, the + // following series of events can take place: + // + // 1. Thread A is working. + // 2. Thread B is spinning in the "while" above, trying + // to steal work. + // 3. Thread B steals work from A. + // 4. Thread A runs out of work. + // 5. Thread A enters the "while" loop and observes all + // worker sto be idle. + // 6. Thread A reaches this point. + // 7. Thread B increments "busy" and restarts its loop, + // processing the work it stole earlier. + // + // To prevent thread A from terminating when new work may be + // produced, we double check both the queue sizes and the number of + // busy workers. Just checking the queue sizes is not enough. If a + // worker popped a job and is still processing it, its queue might + // be empty but new work may still be produced. + // + // Since the "busy" counter is incremented _before_ a worker starts, + // checking both should ensure we never terminate a thread until we + // are certain all work has been completed. + if $self.should_terminate() { + break; + } + + $self.set_busy(); + } + }; +} + +/// A pool of Tracers all tracing the same process. +pub struct Pool { + /// The process of which objects are being traced. + process: RcProcess, + + /// A global queue to steal jobs from. + global_queue: Injector, + + /// The list of queues we can steal work from. + stealers: Vec>, + + /// An integer storing the number of busy tracers in a pool. + busy: AtomicUsize, +} + +impl Pool { + pub fn new( + process: RcProcess, + threads: usize, + ) -> (ArcWithoutWeak, Vec) { + let mut workers = Vec::new(); + let mut stealers = Vec::new(); + + for _ in 0..threads { + let worker = Worker::new_fifo(); + let stealer = worker.stealer(); + + workers.push(worker); + stealers.push(stealer); + } + + let state = ArcWithoutWeak::new(Self { + process, + global_queue: Injector::new(), + stealers, + busy: AtomicUsize::new(threads), + }); + + let tracers = workers + .into_iter() + .map(|worker| Tracer::new(worker, state.clone())) + .collect(); + + (state, tracers) + } + + pub fn schedule(&self, pointer: ObjectPointerPointer) { + self.global_queue.push(pointer); + } +} + +/// A single thread tracing through live objects. +/// +/// A Tracer can trace objects with or without the need for moving objects, and +/// both approaches use a specialised method for this to ensure optimal +/// performance. +/// +/// A Tracer does not check the age of an object to determine if it should be +/// traced or not. Any mature object that is reachable is always marked, as +/// young collections do not reset mature mark states and mature collections +/// would mark any live objects again. +/// +/// A Tracer may steal work from other Tracers in a Tracer pool, balancing work +/// across CPU cores. Due to the inherent racy nature of work-stealing its +/// possible one or more tracers don't perform any work. This can happen if +/// other Tracers steal all work before our Tracer can even begin. +pub struct Tracer { + /// The local queue of objects to trace. + queue: Worker, + + /// The pool of tracers this Tracer belongs to. + pool: ArcWithoutWeak, +} + +impl Tracer { + pub fn new( + queue: Worker, + pool: ArcWithoutWeak, + ) -> Self { + Self { queue, pool } + } + + /// Traces through all live objects, without moving any objects. + pub fn trace_without_moving(&self) -> TraceStatistics { + let mut stats = TraceStatistics::new(); + + trace_loop!( + self, + while let Some(pointer_pointer) = self.queue.pop() { + let pointer = pointer_pointer.get(); + + if pointer.is_marked() { + continue; + } + + pointer.mark(); + + stats.marked += 1; + + pointer.get().each_pointer(|child| { + self.queue.push(child); + }); + } + ); + + stats + } + + /// Traces through all live objects, moving them if needed. + pub fn trace_with_moving(&self) -> TraceStatistics { + let mut stats = TraceStatistics::new(); + + trace_loop!( + self, + while let Some(pointer_pointer) = self.queue.pop() { + let pointer = pointer_pointer.get_mut(); + + if pointer.is_marked() { + continue; + } + + match pointer.status() { + ObjectStatus::Resolve => { + pointer.resolve_forwarding_pointer() + } + ObjectStatus::Promote => { + self.promote_mature(pointer); + + stats.promoted += 1; + stats.marked += 1; + + pointer.mark(); + + // When promoting an object we already trace it, so we + // don't need to trace it again below. + continue; + } + ObjectStatus::Evacuate => { + self.evacuate(pointer); + + stats.evacuated += 1; + } + ObjectStatus::PendingMove => { + self.queue.push(pointer_pointer.clone()); + continue; + } + _ => {} + } + + pointer.mark(); + stats.marked += 1; + + pointer.get().each_pointer(|child| { + self.queue.push(child); + }); + } + ); + + stats + } + + /// Traces a promoted object to see if it should be remembered in the + /// remembered set. + fn trace_promoted_object(&self, promoted: ObjectPointer) { + let mut remember = false; + + promoted.get().each_pointer(|child| { + if !remember && child.get().is_young() { + self.pool.process.remember_object(promoted); + + remember = true; + } + + self.queue.push(child); + }); + } + + /// Promotes an object to the mature generation. + /// + /// The pointer to promote is updated to point to the new location. + fn promote_mature(&self, pointer: &mut ObjectPointer) { + let local_data = self.pool.process.local_data_mut(); + let old_obj = pointer.get_mut(); + let new_pointer = local_data.allocator.allocate_mature(old_obj.take()); + + old_obj.forward_to(new_pointer); + + pointer.resolve_forwarding_pointer(); + + self.trace_promoted_object(*pointer); + } + + // Evacuates a pointer. + // + // The pointer to evacuate is updated to point to the new location. + fn evacuate(&self, pointer: &mut ObjectPointer) { + // When evacuating an object we must ensure we evacuate the object into + // the same bucket. + let local_data = self.pool.process.local_data_mut(); + let bucket = pointer.block_mut().bucket_mut().unwrap(); + let old_obj = pointer.get_mut(); + let new_obj = old_obj.take(); + + let (_, new_pointer) = + bucket.allocate(&local_data.allocator.global_allocator, new_obj); + + old_obj.forward_to(new_pointer); + + pointer.resolve_forwarding_pointer(); + } + + fn steal_from_global(&self) { + loop { + match self.pool.global_queue.steal_batch(&self.queue) { + Steal::Empty | Steal::Success(_) => break, + Steal::Retry => {} + }; + + spin_loop_hint(); + } + } + + fn steal_from_worker(&self) -> bool { + while self.pool.busy.load(Ordering::Acquire) > 0 { + for stealer in self.pool.stealers.iter() { + loop { + match stealer.steal_batch(&self.queue) { + Steal::Empty => break, + Steal::Retry => {} + Steal::Success(_) => return true, + } + } + } + + spin_loop_hint(); + } + + false + } + + fn should_terminate(&self) -> bool { + self.pool.stealers.iter().all(|x| x.is_empty()) + && self.pool.busy.load(Ordering::Acquire) == 0 + } + + fn has_global_jobs(&self) -> bool { + !self.pool.global_queue.is_empty() + } + + fn set_busy(&self) { + self.pool.busy.fetch_add(1, Ordering::Release); + } + + fn set_idle(&self) { + self.pool.busy.fetch_sub(1, Ordering::Release); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::object::Object; + use crate::object_value; + use crate::vm::test::setup; + + fn prepare(process: RcProcess) -> (ArcWithoutWeak, Tracer) { + let (pool, mut tracers) = Pool::new(process, 1); + + (pool, tracers.pop().unwrap()) + } + + #[test] + fn test_promote_mature() { + let (_machine, _block, process) = setup(); + let (_pool, tracer) = prepare(process.clone()); + let mut pointer = + process.allocate_without_prototype(object_value::float(15.0)); + + let old_address = pointer.raw.raw as usize; + + tracer.promote_mature(&mut pointer); + + let new_address = pointer.raw.raw as usize; + + assert!(old_address != new_address); + assert!(pointer.is_mature()); + assert_eq!(pointer.float_value().unwrap(), 15.0); + } + + #[test] + fn test_evacuate() { + let (_machine, _block, process) = setup(); + let (_pool, tracer) = prepare(process.clone()); + let mut pointer = + process.allocate_without_prototype(object_value::float(15.0)); + + let old_address = pointer.raw.raw as usize; + + tracer.evacuate(&mut pointer); + + let new_address = pointer.raw.raw as usize; + + assert!(old_address != new_address); + assert_eq!(pointer.float_value().unwrap(), 15.0); + } + + #[test] + fn test_trace_with_moving_with_marked_mature() { + let (_machine, _block, process) = setup(); + let (pool, tracer) = prepare(process.clone()); + let young_parent = process.allocate_empty(); + let young_child = process.allocate_empty(); + + young_parent.add_attribute(&process, young_child, young_child); + young_parent.block_mut().set_fragmented(); + + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + mature.block_mut().set_fragmented(); + mature.mark(); + + pool.schedule(young_parent.pointer()); + pool.schedule(mature.pointer()); + + let stats = tracer.trace_with_moving(); + + assert_eq!(stats.marked, 2); + assert_eq!(stats.evacuated, 2); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_trace_with_moving_with_unmarked_mature() { + let (_machine, _block, process) = setup(); + let (pool, tracer) = prepare(process.clone()); + let young_parent = process.allocate_empty(); + let young_child = process.allocate_empty(); + + young_parent.add_attribute(&process, young_child, young_child); + young_parent.block_mut().set_fragmented(); + + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + mature.block_mut().set_fragmented(); + + pool.schedule(young_parent.pointer()); + pool.schedule(mature.pointer()); + + let stats = tracer.trace_with_moving(); + + assert_eq!(stats.marked, 3); + assert_eq!(stats.evacuated, 3); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_trace_without_moving_with_marked_mature() { + let (_machine, _block, process) = setup(); + let (pool, tracer) = prepare(process.clone()); + let young_parent = process.allocate_empty(); + let young_child = process.allocate_empty(); + + young_parent.add_attribute(&process, young_child, young_child); + + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + mature.mark(); + + pool.schedule(young_parent.pointer()); + pool.schedule(mature.pointer()); + + let stats = tracer.trace_without_moving(); + + assert!(young_parent.is_marked()); + assert!(young_child.is_marked()); + + assert_eq!(stats.marked, 2); + assert_eq!(stats.evacuated, 0); + assert_eq!(stats.promoted, 0); + } + + #[test] + fn test_trace_without_moving_with_unmarked_mature() { + let (_machine, _block, process) = setup(); + let (pool, tracer) = prepare(process.clone()); + let young_parent = process.allocate_empty(); + let young_child = process.allocate_empty(); + + young_parent.add_attribute(&process, young_child, young_child); + + let mature = process + .local_data_mut() + .allocator + .allocate_mature(Object::new(object_value::none())); + + pool.schedule(young_parent.pointer()); + pool.schedule(mature.pointer()); + + let stats = tracer.trace_without_moving(); + + assert!(young_parent.is_marked()); + assert!(young_child.is_marked()); + assert!(mature.is_marked()); + + assert_eq!(stats.marked, 3); + assert_eq!(stats.evacuated, 0); + assert_eq!(stats.promoted, 0); + } +} diff --git a/vm/src/gc/work_list.rs b/vm/src/gc/work_list.rs deleted file mode 100644 index 33a89363e..000000000 --- a/vm/src/gc/work_list.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! Lists of pointers to mark. -//! -//! A WorkList can be used to store and retrieve the pointers to mark during the -//! tracing phase of a garbage collection cycle. -//! -//! # Prefetching -//! -//! The WorkList structure uses prefetching (when supported) to reduce the -//! amount of cache misses. The code used for this is based on the technique -//! described in "Effective Prefetch for Mark-Sweep Garbage Collection" by -//! Garner et al (2007). A PDF version of this paper can be found at -//! . -//! -//! If the buffer is empty we prefetch up to 8 pointers, this improves -//! performance drastically compared to just prefetching one pointer at a time. -//! In the best case scenario using this technique can improve tracing -//! performance by 20-30%. -use crate::object_pointer::ObjectPointerPointer; -use crate::prefetch; -use std::collections::VecDeque; - -/// The number of pointers to prefetch when the buffer is empty. This size is -/// based on the number of pointers that fit in a typical single cache line (= -/// 64 bytes on most common processors). -pub const PREFETCH_BUFFER_SIZE: usize = 8; - -/// The amount of values to reserve space for in the mark stack. We anticipate -/// many pointers to be stored so preallocating this space reduces the number -/// of reallocations necessary. -pub const STACK_RESERVE_SIZE: usize = 128; - -pub struct WorkList { - /// All the pointers that need to be marked. - pub stack: VecDeque, - - /// Pointers that have already been prefetched and should be marked before - /// those in the stack. - pub prefetch_buffer: VecDeque, -} - -impl WorkList { - pub fn new() -> Self { - WorkList { - stack: VecDeque::with_capacity(STACK_RESERVE_SIZE), - prefetch_buffer: VecDeque::with_capacity(PREFETCH_BUFFER_SIZE), - } - } - - pub fn push(&mut self, pointer: ObjectPointerPointer) { - self.stack.push_back(pointer); - } - - pub fn pop(&mut self) -> Option { - if self.prefetch_buffer.is_empty() { - self.prefetch(PREFETCH_BUFFER_SIZE); - } - - if let Some(pointer) = self.prefetch_buffer.pop_front() { - self.prefetch(1); - Some(pointer) - } else { - None - } - } - - #[inline(always)] - pub fn prefetch(&mut self, amount: usize) { - for _ in 0..amount { - if let Some(prefetch) = self.stack.pop_front() { - self.push_to_prefetch_buffer(prefetch); - } else { - break; - } - } - } - - #[inline(always)] - pub fn push_to_prefetch_buffer(&mut self, pointer: ObjectPointerPointer) { - prefetch::prefetch_read(pointer.raw); - - self.prefetch_buffer.push_back(pointer); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::object_pointer::ObjectPointer; - - fn fake_pointer(address: usize) -> ObjectPointerPointer { - ObjectPointerPointer { - raw: address as *const ObjectPointer, - } - } - - #[test] - fn test_push() { - let mut worklist = WorkList::new(); - - worklist.push(fake_pointer(0x4)); - - assert_eq!(worklist.stack.len(), 1); - assert_eq!(worklist.prefetch_buffer.len(), 0); - } - - #[test] - fn test_pop_empty() { - let mut worklist = WorkList::new(); - - assert!(worklist.pop().is_none()); - } - - #[test] - fn test_pop_non_empty() { - let mut worklist = WorkList::new(); - - worklist.push(fake_pointer(0x1)); - worklist.push(fake_pointer(0x2)); - worklist.push(fake_pointer(0x3)); - - assert_eq!(worklist.pop().unwrap().raw as usize, 0x1); - assert_eq!(worklist.prefetch_buffer.len(), 2); - } - - #[test] - fn test_prefetch_empty() { - let mut worklist = WorkList::new(); - - worklist.prefetch(5); - - assert!(worklist.prefetch_buffer.is_empty()); - } - - #[test] - fn test_prefetch_non_empty() { - let mut worklist = WorkList::new(); - - worklist.push(fake_pointer(0x1)); - worklist.push(fake_pointer(0x2)); - worklist.push(fake_pointer(0x3)); - worklist.prefetch(2); - - assert_eq!(worklist.stack.len(), 1); - assert_eq!(worklist.prefetch_buffer.len(), 2); - } -} diff --git a/vm/src/immix/block.rs b/vm/src/immix/block.rs index a408815c0..8a14f1bc1 100644 --- a/vm/src/immix/block.rs +++ b/vm/src/immix/block.rs @@ -13,7 +13,7 @@ use std::alloc::{self, Layout}; use std::mem; use std::ops::Drop; use std::ptr; -use std::thread; +use std::sync::atomic::spin_loop_hint; /// The number of bytes in a block. pub const BLOCK_SIZE: usize = 8 * 1024; @@ -41,7 +41,7 @@ pub const OBJECTS_PER_BLOCK: usize = BLOCK_SIZE / BYTES_PER_OBJECT; pub const OBJECTS_PER_LINE: usize = LINE_SIZE / BYTES_PER_OBJECT; /// The first slot objects can be allocated into. The first 4 slots (a single -/// line or 128 bytes of memory) are reserved for the mark bitmap. +/// line or 128 bytes of memory) are reserved for the mark bytemap. pub const OBJECT_START_SLOT: usize = LINE_SIZE / BYTES_PER_OBJECT; /// The first line objects can be allocated into. @@ -51,11 +51,11 @@ pub const LINE_START_SLOT: usize = 1; pub const FIRST_OBJECT_BYTE_OFFSET: usize = OBJECT_START_SLOT * BYTES_PER_OBJECT; -/// The mask to apply to go from a pointer to the mark bitmap's start. -pub const OBJECT_BITMAP_MASK: isize = !(BLOCK_SIZE as isize - 1); +/// The mask to apply to go from a pointer to the mark bytemap's start. +pub const OBJECT_BYTEMAP_MASK: isize = !(BLOCK_SIZE as isize - 1); /// The mask to apply to go from a pointer to the line's start. -pub const LINE_BITMAP_MASK: isize = !(LINE_SIZE as isize - 1); +pub const LINE_BYTEMAP_MASK: isize = !(LINE_SIZE as isize - 1); unsafe fn heap_layout_for_block() -> Layout { Layout::from_size_align_unchecked(BLOCK_SIZE, BLOCK_SIZE) @@ -146,17 +146,17 @@ pub struct Block { /// allocated into or beyond this pointer. pub end_pointer: DerefPointer, - /// The memory to use for the mark bitmap and allocating objects. The first + /// The memory to use for the mark bytemap and allocating objects. The first /// 128 bytes of this field are reserved and used for storing a BlockHeader. /// /// Memory is aligned to the block size. pub lines: RawObjectPointer, - /// Bitmap used to track which lines contain one or more reachable objects. - pub used_lines_bitmap: LineMap, + /// Bytemap used to track which lines contain one or more reachable objects. + pub used_lines_bytemap: LineMap, - /// Bitmap used for tracking which object slots are live. - pub marked_objects_bitmap: ObjectMap, + /// Bytemap used for tracking which object slots are live. + pub marked_objects_bytemap: ObjectMap, } unsafe impl Send for Block {} @@ -174,8 +174,8 @@ impl Block { let mut block = Box::new(Block { lines, - marked_objects_bitmap: ObjectMap::new(), - used_lines_bitmap: LineMap::new(), + marked_objects_bytemap: ObjectMap::new(), + used_lines_bytemap: LineMap::new(), free_pointer: DerefPointer::null(), end_pointer: DerefPointer::null(), }); @@ -194,16 +194,6 @@ impl Block { block } - /// Prepares this block for a garbage collection cycle. - pub fn prepare_for_collection(&mut self) { - self.used_lines_bitmap.swap_mark_value(); - self.marked_objects_bitmap.reset(); - } - - pub fn update_line_map(&mut self) { - self.used_lines_bitmap.reset_previous_marks(); - } - /// Returns an immutable reference to the header of this block. #[inline(always)] pub fn header(&self) -> &BlockHeader { @@ -245,6 +235,10 @@ impl Block { self.header_mut().fragmented = true; } + pub fn clear_fragmentation_status(&mut self) { + self.header_mut().fragmented = false; + } + pub fn is_fragmented(&self) -> bool { self.header().fragmented } @@ -255,7 +249,7 @@ impl Block { /// Returns true if all lines in this block are available. pub fn is_empty(&self) -> bool { - self.used_lines_bitmap.is_empty() + self.used_lines_bytemap.is_empty() } /// Returns a pointer to the first address to be used for objects. @@ -335,7 +329,7 @@ impl Block { // observed as being greater than the end pointer. Since the end // pointer will be updated very soon, we can just spin for a // little while. - thread::yield_now(); + spin_loop_hint(); continue; } @@ -347,39 +341,9 @@ impl Block { } } - /// Requests a new pointer to use for an object allocated by a mutator. - /// - /// Unlike `Block::request_pointer()`, this method _does not_ use atomic - /// operations for bump allocations. This means it _can not_ be used by any - /// collector threads. - pub unsafe fn request_pointer_for_mutator( - &mut self, - ) -> Option { - loop { - let current = self.free_pointer.pointer; - let end = self.end_pointer.pointer; - - if current == end { - if current == self.end_address() { - return None; - } - - if self.find_available_hole(current, end) { - continue; - } else { - return None; - } - } - - self.free_pointer = DerefPointer::from_pointer(current.offset(1)); - - return Some(current); - } - } - pub fn line_index_of_pointer(&self, pointer: RawObjectPointer) -> usize { let first_line = self.lines as usize; - let line_addr = (pointer as isize & LINE_BITMAP_MASK) as usize; + let line_addr = (pointer as isize & LINE_BYTEMAP_MASK) as usize; (line_addr - first_line) / LINE_SIZE } @@ -416,12 +380,17 @@ impl Block { self.set_free_pointer(start_addr); self.set_end_pointer(end_addr); - self.reset_mark_bitmaps(); + self.used_lines_bytemap.reset(); + self.marked_objects_bytemap.reset(); + } + + pub fn prepare_for_collection(&mut self) { + self.used_lines_bytemap.swap_mark_value(); + self.marked_objects_bytemap.reset(); } - pub fn reset_mark_bitmaps(&mut self) { - self.used_lines_bitmap.reset(); - self.marked_objects_bitmap.reset(); + pub fn update_line_map(&mut self) { + self.used_lines_bytemap.reset_previous_marks(); } /// Finalizes all unmarked objects right away. @@ -440,7 +409,7 @@ impl Block { let mut holes = 0; for index in LINE_START_SLOT..LINES_PER_BLOCK { - let is_set = self.used_lines_bitmap.is_set(index); + let is_set = self.used_lines_bytemap.is_set(index); if in_hole && is_set { in_hole = false; @@ -456,12 +425,12 @@ impl Block { } /// Returns the number of marked lines in this block. - pub fn marked_lines_count(&self) -> usize { - self.used_lines_bitmap.len() + pub fn marked_lines_count(&mut self) -> usize { + self.used_lines_bytemap.len() } /// Returns the number of available lines in this block. - pub fn available_lines_count(&self) -> usize { + pub fn available_lines_count(&mut self) -> usize { (LINES_PER_BLOCK - 1) - self.marked_lines_count() } @@ -483,7 +452,7 @@ impl Block { // Find the start of the hole. while line < LINES_PER_BLOCK { - if !self.used_lines_bitmap.is_set(line) { + if !self.used_lines_bytemap.is_set(line) { new_free = self.pointer_for_hole_starting_at_line(line); found_hole = true; @@ -495,7 +464,7 @@ impl Block { // Find the end of the hole. while line < LINES_PER_BLOCK { - if self.used_lines_bitmap.is_set(line) { + if self.used_lines_bytemap.is_set(line) { new_end = self.pointer_for_hole_starting_at_line(line); break; } @@ -551,7 +520,9 @@ mod tests { #[test] fn test_block_header_type_size() { - assert!(mem::size_of::() <= LINE_SIZE); + // Block headers must be smaller than or equal to the size of a single + // line. + assert_eq!(mem::size_of::(), 40); } #[test] @@ -593,27 +564,15 @@ mod tests { assert!(block.bucket().is_none()); } - #[test] - fn test_block_prepare_for_collection() { - let mut block = Block::boxed(); - - block.used_lines_bitmap.set(1); - block.marked_objects_bitmap.set(1); - block.prepare_for_collection(); - - assert!(block.used_lines_bitmap.is_set(1)); - assert_eq!(block.marked_objects_bitmap.is_set(1), false); - } - #[test] fn test_block_update_line_map() { let mut block = Block::boxed(); - block.used_lines_bitmap.set(1); - block.prepare_for_collection(); + block.used_lines_bytemap.set(1); + block.used_lines_bytemap.swap_mark_value(); block.update_line_map(); - assert!(block.used_lines_bitmap.is_empty()); + assert_eq!(block.used_lines_bytemap.is_set(1), false); } #[test] @@ -650,7 +609,7 @@ mod tests { assert!(block.is_empty()); - block.used_lines_bitmap.set(1); + block.used_lines_bytemap.set(1); assert_eq!(block.is_empty(), false); } @@ -680,23 +639,12 @@ mod tests { }); } - #[test] - fn test_block_request_pointer_for_mutator() { - let mut block = Block::boxed(); - - assert!(unsafe { block.request_pointer_for_mutator().is_some() }); - - assert_eq!(block.free_pointer(), unsafe { - block.start_address().offset(1) - }); - } - #[test] fn test_block_request_pointer_advances_hole() { let mut block = Block::boxed(); let start = block.start_address(); - block.used_lines_bitmap.set(2); + block.used_lines_bytemap.set(2); find_available_hole!(block); @@ -710,23 +658,42 @@ mod tests { } #[test] - fn test_block_request_pointer_for_mutator_advances_hole() { + fn test_request_pointer_after_preparing_collection() { let mut block = Block::boxed(); - let start = block.start_address(); - block.used_lines_bitmap.set(2); + // We simulate a block starting with a hole, followed by an empty line, + // followed by another hole, that we allocate into. This can happen when + // a survivor space is allocated into when it still has recyclable + // blocks, such as during evacuation. + block.used_lines_bytemap.set(2); + block.recycle(); + block.prepare_for_collection(); + + // At this point line 2 should not be allocated into, because it was + // still marked as live. + for i in 0..8 { + let pointer = ObjectPointer::new(block.request_pointer().unwrap()); - find_available_hole!(block); + Object::new(ObjectValue::Integer(i)).write_to(pointer.raw.raw); + } - for _ in 0..4 { - unsafe { - block.request_pointer_for_mutator(); - } + for i in 0..4 { + let raw_pointer = unsafe { block.start_address().add(i) }; + + assert!(ObjectPointer::new(raw_pointer).is_integer()); } - assert!(unsafe { block.request_pointer_for_mutator().is_some() }); + for i in 4..8 { + let raw_pointer = unsafe { block.start_address().add(i) }; - assert_eq!(block.free_pointer(), unsafe { start.offset(9) }); + assert!(ObjectPointer::new(raw_pointer).get().value.is_none()); + } + + for i in 8..12 { + let raw_pointer = unsafe { block.start_address().add(i) }; + + assert!(ObjectPointer::new(raw_pointer).is_integer()); + } } #[test] @@ -769,7 +736,7 @@ mod tests { let mut block = Block::boxed(); // First line is used - block.used_lines_bitmap.set(1); + block.used_lines_bytemap.set(1); block.recycle(); assert_eq!(block.free_pointer(), unsafe { @@ -779,8 +746,8 @@ mod tests { assert_eq!(block.end_pointer(), block.end_address()); // first line is available, followed by a used line - block.used_lines_bitmap.reset(); - block.used_lines_bitmap.set(2); + block.used_lines_bytemap.reset(); + block.used_lines_bytemap.set(2); block.recycle(); assert_eq!(block.free_pointer(), block.start_address()); @@ -797,15 +764,15 @@ mod tests { let pointer1 = Object::new(ObjectValue::None) .write_to(block.request_pointer().unwrap()); - block.used_lines_bitmap.set(1); + block.used_lines_bytemap.set(1); find_available_hole!(block); let pointer2 = Object::new(ObjectValue::None) .write_to(block.request_pointer().unwrap()); - block.used_lines_bitmap.set(2); - block.used_lines_bitmap.set(3); + block.used_lines_bytemap.set(2); + block.used_lines_bytemap.set(3); find_available_hole!(block); @@ -822,7 +789,7 @@ mod tests { let mut block = Block::boxed(); let start = block.start_address(); - block.used_lines_bitmap.set(1); + block.used_lines_bytemap.set(1); find_available_hole!(block); @@ -835,8 +802,8 @@ mod tests { let mut block = Block::boxed(); let start = block.start_address(); - block.used_lines_bitmap.set(1); - block.used_lines_bitmap.set(3); + block.used_lines_bytemap.set(1); + block.used_lines_bytemap.set(3); find_available_hole!(block); @@ -849,7 +816,7 @@ mod tests { let mut block = Block::boxed(); for index in 1..LINES_PER_BLOCK { - block.used_lines_bitmap.set(index); + block.used_lines_bytemap.set(index); } find_available_hole!(block); @@ -862,9 +829,8 @@ mod tests { fn test_block_find_available_hole_recycle() { let mut block = Block::boxed(); - block.used_lines_bitmap.set(1); - block.used_lines_bitmap.set(2); - block.used_lines_bitmap.reset_previous_marks(); + block.used_lines_bytemap.set(1); + block.used_lines_bytemap.set(2); find_available_hole!(block); @@ -877,9 +843,9 @@ mod tests { fn test_block_find_available_hole_pointer_range() { let mut block = Block::boxed(); - block.used_lines_bitmap.set(1); - block.used_lines_bitmap.set(2); - block.used_lines_bitmap.set(LINES_PER_BLOCK - 1); + block.used_lines_bytemap.set(1); + block.used_lines_bytemap.set(2); + block.used_lines_bytemap.set(LINES_PER_BLOCK - 1); find_available_hole!(block); @@ -909,8 +875,8 @@ mod tests { block.set_end_pointer(start_addr); block.set_bucket(&mut bucket as *mut Bucket); - block.used_lines_bitmap.set(1); - block.marked_objects_bitmap.set(1); + block.used_lines_bytemap.set(1); + block.marked_objects_bytemap.set(1); block.reset(); @@ -919,8 +885,8 @@ mod tests { assert!(block.free_pointer() == block.start_address()); assert!(block.end_pointer() == block.end_address()); assert!(block.bucket().is_none()); - assert!(block.used_lines_bitmap.is_empty()); - assert!(block.marked_objects_bitmap.is_empty()); + assert!(block.used_lines_bytemap.is_empty()); + assert!(block.marked_objects_bytemap.is_empty()); } #[test] @@ -929,21 +895,29 @@ mod tests { let raw_pointer = block.request_pointer().unwrap(); let pointer = ObjectPointer::new(raw_pointer); - Object::new(ObjectValue::Float(10.0)).write_to(raw_pointer); + { + let mut obj = Object::new(ObjectValue::Float(10.0)); + + obj.add_attribute(pointer, pointer); + obj.write_to(raw_pointer); + } + + block.finalize(); - pointer.finalize(); + let obj = pointer.get(); - assert!(pointer.get().value.is_none()); - assert_eq!(pointer.is_finalizable(), false); + assert!(obj.attributes.is_null()); + assert!(obj.prototype.is_null()); + assert!(obj.value.is_none()); } #[test] fn test_block_update_hole_count() { let mut block = Block::boxed(); - block.used_lines_bitmap.set(1); - block.used_lines_bitmap.set(3); - block.used_lines_bitmap.set(10); + block.used_lines_bytemap.set(1); + block.used_lines_bytemap.set(3); + block.used_lines_bytemap.set(10); block.update_hole_count(); @@ -956,7 +930,7 @@ mod tests { assert_eq!(block.marked_lines_count(), 0); - block.used_lines_bitmap.set(1); + block.used_lines_bytemap.set(1); assert_eq!(block.marked_lines_count(), 1); } @@ -967,8 +941,30 @@ mod tests { assert_eq!(block.available_lines_count(), LINES_PER_BLOCK - 1); - block.used_lines_bitmap.set(1); + block.used_lines_bytemap.set(1); assert_eq!(block.available_lines_count(), LINES_PER_BLOCK - 2); } + + #[test] + fn test_clear_fragmentation_status() { + let mut block = Block::boxed(); + + block.set_fragmented(); + block.clear_fragmentation_status(); + + assert_eq!(block.is_fragmented(), false); + } + + #[test] + fn test_prepare_for_collection() { + let mut block = Block::boxed(); + + block.used_lines_bytemap.set(1); + block.marked_objects_bytemap.set(1); + block.prepare_for_collection(); + + assert!(block.used_lines_bytemap.is_set(1)); + assert_eq!(block.marked_objects_bytemap.is_set(1), false); + } } diff --git a/vm/src/immix/bucket.rs b/vm/src/immix/bucket.rs index cd8421931..f5d9a8a43 100644 --- a/vm/src/immix/bucket.rs +++ b/vm/src/immix/bucket.rs @@ -3,15 +3,15 @@ //! A Bucket contains a sequence of Immix blocks that all contain objects of the //! same age. use crate::deref_pointer::DerefPointer; -use crate::immix::block::Block; +use crate::immix::block::{Block, MAX_HOLES}; use crate::immix::block_list::BlockList; use crate::immix::global_allocator::RcGlobalAllocator; +use crate::immix::histogram::MINIMUM_BIN; use crate::immix::histograms::Histograms; use crate::object::Object; use crate::object_pointer::ObjectPointer; -use crate::vm::state::RcState; +use crate::vm::state::State; use parking_lot::Mutex; -use rayon::prelude::*; use std::cell::UnsafeCell; macro_rules! lock_bucket { @@ -45,9 +45,6 @@ pub struct Bucket { /// The age of the objects in the current bucket. pub age: i8, - - /// The objects in this bucket should be promoted to the mature generation. - pub promote: bool, } unsafe impl Send for Bucket {} @@ -63,28 +60,18 @@ impl Bucket { blocks: BlockList::new(), current_block: DerefPointer::null(), age, - promote: false, lock: UnsafeCell::new(Mutex::new(())), } } pub fn reset_age(&mut self) { self.age = 0; - self.promote = false; } pub fn increment_age(&mut self) { self.age += 1; } - pub fn number_of_blocks(&self) -> u32 { - // The maximum value of u32 is 4 294 967 295. With every block being 32 - // KB in size, this means we have an upper limit of 128 TB per process. - // That seems more than enough, and allows us to more efficiently store - // this number compared to using an u64/usize. - self.blocks.len() as u32 - } - pub fn current_block(&self) -> Option> { let pointer = self.current_block.atomic_load(); @@ -124,6 +111,8 @@ impl Bucket { /// /// The return value is a tuple containing a boolean that indicates if a new /// block was requested, and the pointer to the allocated object. + /// + /// This method can safely be used concurrently by different threads. pub fn allocate( &mut self, global_allocator: &RcGlobalAllocator, @@ -173,148 +162,120 @@ impl Bucket { } } - /// Allocates an object for a mutator into this bucket + /// Reclaims the blocks in this bucket /// - /// The return value is the same as `Bucket::allocate()`. + /// Recyclable blocks are scheduled for re-use by the allocator, empty + /// blocks are to be returned to the global pool, and full blocks are kept. /// - /// This method does not use synchronisation, so it _can not_ be safely used - /// from a collector thread. - pub unsafe fn allocate_for_mutator( + /// The return value is the total number of blocks after reclaiming + /// finishes. + pub fn reclaim_blocks( &mut self, - global_allocator: &RcGlobalAllocator, - object: Object, - ) -> (bool, ObjectPointer) { - let mut new_block = false; - - loop { - let mut advance_block = false; - - if let Some(mut current) = self.current_block() { - for block in current.iter_mut() { - if block.is_fragmented() { - // The block is fragmented, so skip it. The next time we - // find an available block we'll set it as the current - // block. - advance_block = true; - - continue; - } - - if let Some(raw_pointer) = - block.request_pointer_for_mutator() - { - if advance_block { - self.current_block = DerefPointer::new(block); - } - - return (new_block, object.write_to(raw_pointer)); - } - } - } + state: &State, + histograms: &mut Histograms, + ) -> usize { + let mut to_release = BlockList::new(); + let mut amount = 0; + + // We perform this work sequentially, as performing this in parallel + // would require multiple passes over the list of input blocks. We found + // that performing this work in parallel using Rayon ended up being + // about 20% slower, likely due to: + // + // 1. The overhead of distributing work across threads. + // 2. The list of blocks being a linked list, which can't be easily + // split to balance load across threads. + for mut block in self.blocks.drain() { + block.update_line_map(); - new_block = true; - self.add_block(global_allocator.request_block()); - } - } + if block.is_empty() { + block.reset(); + to_release.push_front(block); + } else { + let holes = block.update_hole_count(); - /// Returns true if this bucket contains blocks that need to be evacuated. - pub fn should_evacuate(&self) -> bool { - // The Immix paper states that one should evacuate when there are one or - // more recyclable or fragmented blocks. In IVM all objects are the same - // size and thus it's not possible to have any recyclable blocks left by - // the time we start a collection (as they have all been consumed). As - // such we don't check for these and instead only check for fragmented - // blocks. - self.blocks.iter().any(Block::is_fragmented) - } + // Clearing the fragmentation status is done so a block does + // not stay fragmented until it has been evacuated entirely. + // This ensures we don't keep evacuating objects when this + // may no longer be needed. + block.clear_fragmentation_status(); - /// Reclaims the blocks in this bucket - /// - /// Recyclable blocks are scheduled for re-use by the allocator, empty - /// blocks are to be returned to the global pool, and full blocks are kept. - pub fn reclaim_blocks(&mut self, state: &RcState, histograms: &Histograms) { - self.blocks - .pointers() - .into_par_iter() - .for_each(|mut block| { - block.update_line_map(); - - if block.is_empty() { - block.reset(); - } else { - let holes = block.update_hole_count(); - - if holes > 0 { + if holes > 0 { + if holes >= MINIMUM_BIN { histograms.marked.increment( holes, - block.marked_lines_count() as u16, + block.marked_lines_count() as u32, ); - - block.recycle(); } + + block.recycle(); } - }); - // We partition the blocks in sequence so we don't need to synchronise - // access to the destination lists. - for block in self.blocks.drain() { - if block.is_empty() { - state.global_allocator.add_block(block); - } else { + amount += 1; + self.blocks.push_front(block); } } + state.global_allocator.add_blocks(&mut to_release); + self.reset_current_block(); + + amount } /// Prepares this bucket for a collection. - /// - /// Returns true if evacuation is needed for this bucket. - pub fn prepare_for_collection(&mut self, histograms: &Histograms) -> bool { - let mut available: isize = 0; + pub fn prepare_for_collection( + &mut self, + histograms: &mut Histograms, + evacuate: bool, + ) { let mut required: isize = 0; - let evacuate = self.should_evacuate(); + let mut available: isize = 0; for block in self.blocks.iter_mut() { let holes = block.holes(); - if evacuate && holes > 0 { - let count = block.available_lines_count() as u16; + // We ignore blocks with only a single hole, as those are not + // fragmented and not worth evacuating. This also ensures we ignore + // blocks added since the last collection, which will have a hole + // count of 1. + if evacuate && holes >= MINIMUM_BIN { + let lines = block.available_lines_count() as u32; - histograms.available.increment(holes, count); + histograms.available.increment(holes, lines); - available += count as isize; - } + available += lines as isize; + }; + // We _must_ reset the bytemaps _after_ calculating the above + // statistics, as those statistics depend on the mark values in + // these maps. block.prepare_for_collection(); } if available > 0 { - let mut iter = histograms.marked.iter(); - let mut min_bin = None; + let mut min_bin = 0; + let mut bin = MAX_HOLES; - while available > required { - if let Some(bin) = iter.next() { - required += histograms.marked.get(bin) as isize; - available -= histograms.available.get(bin) as isize; + // Bucket 1 refers to blocks with only a single hole. Blocks with + // just one hole aren't fragmented, so we ignore those here. + while available > required && bin >= MINIMUM_BIN { + required += histograms.marked.get(bin) as isize; + available -= histograms.available.get(bin) as isize; - min_bin = Some(bin); - } else { - break; - } + min_bin = bin; + bin -= 1; } - if let Some(bin) = min_bin { + if min_bin > 0 { for block in self.blocks.iter_mut() { - if block.holes() >= bin { + if block.holes() >= min_bin { block.set_fragmented(); } } } } - - evacuate } } @@ -367,11 +328,9 @@ mod tests { fn test_reset_age() { let mut bucket = Bucket::with_age(4); - bucket.promote = true; bucket.reset_age(); assert_eq!(bucket.age, 0); - assert_eq!(bucket.promote, false); } #[test] @@ -455,14 +414,14 @@ mod tests { let state = State::with_rc(Config::new(), &[]); let global_alloc = global_allocator(); let mut bucket = Bucket::new(); - let histos = Histograms::new(); + let mut histos = Histograms::new(); let (_, pointer) = bucket.allocate(&global_alloc, Object::new(object_value::none())); pointer.mark(); - bucket.reclaim_blocks(&state, &histos); + bucket.reclaim_blocks(&state, &mut histos); assert_eq!(bucket.blocks.len(), 1); @@ -480,89 +439,6 @@ mod tests { ); } - #[test] - fn test_allocate_for_mutator_without_blocks() { - let global_alloc = global_allocator(); - let mut bucket = Bucket::new(); - - let (new_block, pointer) = unsafe { - bucket.allocate_for_mutator( - &global_alloc, - Object::new(object_value::none()), - ) - }; - - assert!(new_block); - assert!(pointer.get().value.is_none()); - - let block = pointer.block(); - - assert!( - block.free_pointer() == unsafe { block.start_address().offset(1) } - ); - - unsafe { - bucket.allocate_for_mutator( - &global_alloc, - Object::new(object_value::none()), - ); - } - - assert!( - block.free_pointer() == unsafe { block.start_address().offset(2) } - ); - } - - #[test] - fn test_allocate_for_mutator_with_recyclable_blocks() { - let state = State::with_rc(Config::new(), &[]); - let global_alloc = global_allocator(); - let mut bucket = Bucket::new(); - let histos = Histograms::new(); - - let (_, pointer) = unsafe { - bucket.allocate_for_mutator( - &global_alloc, - Object::new(object_value::none()), - ) - }; - - pointer.mark(); - - bucket.reclaim_blocks(&state, &histos); - - assert_eq!(bucket.blocks.len(), 1); - - let (new_block, new_pointer) = unsafe { - bucket.allocate_for_mutator( - &global_alloc, - Object::new(object_value::float(4.0)), - ) - }; - - assert_eq!(new_block, false); - assert!(pointer.get().value.is_none()); - assert!(new_pointer.get().value.is_float()); - - let head = bucket.blocks.head().unwrap(); - - assert!( - head.free_pointer() == unsafe { head.start_address().offset(5) } - ); - } - - #[test] - fn test_should_evacuate_with_fragmented_blocks() { - let mut bucket = Bucket::new(); - let mut block = Block::boxed(); - - block.set_fragmented(); - - bucket.add_block(block); - - assert!(bucket.should_evacuate()); - } - #[test] fn test_reclaim_blocks() { let mut bucket = Bucket::new(); @@ -570,22 +446,25 @@ mod tests { let block2 = Block::boxed(); let mut block3 = Block::boxed(); let state = State::with_rc(Config::new(), &[]); - let histos = Histograms::new(); + let mut histos = Histograms::new(); - block1.used_lines_bitmap.set(LINES_PER_BLOCK - 1); - block3.used_lines_bitmap.set(2); + block1.used_lines_bytemap.set(LINES_PER_BLOCK - 1); + + block3.used_lines_bytemap.set(2); bucket.add_block(block1); bucket.add_block(block2); bucket.add_block(block3); - bucket.reclaim_blocks(&state, &histos); + + let total = bucket.reclaim_blocks(&state, &mut histos); assert_eq!(bucket.blocks.len(), 2); + assert_eq!(total, 2); assert_eq!(bucket.blocks[0].holes(), 1); assert_eq!(bucket.blocks[1].holes(), 2); - assert_eq!(histos.marked.get(1), 1); + assert_eq!(histos.marked.get(1), 0); // Bucket 1 should not be set assert_eq!(histos.marked.get(2), 1); } @@ -593,70 +472,63 @@ mod tests { fn test_reclaim_blocks_full() { let mut bucket = Bucket::new(); let mut block = Block::boxed(); - let histos = Histograms::new(); + let mut histos = Histograms::new(); let state = State::with_rc(Config::new(), &[]); for i in 0..LINES_PER_BLOCK { - block.used_lines_bitmap.set(i); + block.used_lines_bytemap.set(i); } bucket.add_block(block); - bucket.reclaim_blocks(&state, &histos); + + let total = bucket.reclaim_blocks(&state, &mut histos); assert_eq!(bucket.blocks.len(), 1); + assert_eq!(total, 1); assert_eq!(bucket.current_block.is_null(), false); } #[test] fn test_prepare_for_collection_without_evacuation() { let mut bucket = Bucket::new(); - let histos = Histograms::new(); + let mut histos = Histograms::new(); bucket.add_block(Block::boxed()); - bucket.current_block().unwrap().used_lines_bitmap.set(1); + bucket.current_block().unwrap().used_lines_bytemap.set(1); - assert_eq!(bucket.prepare_for_collection(&histos), false); + bucket.prepare_for_collection(&mut histos, false); // No evacuation needed means the available histogram is not updated. assert_eq!(histos.available.get(1), 0); let block = bucket.current_block().unwrap(); - assert!(block.marked_objects_bitmap.is_empty()); + assert!(block.marked_objects_bytemap.is_empty()); } #[test] fn test_prepare_for_collection_with_evacuation() { let mut bucket = Bucket::new(); - let mut block1 = Block::boxed(); - let histos = Histograms::new(); - let block2 = Block::boxed(); + let block1 = Block::boxed(); + let mut block2 = Block::boxed(); + let mut histos = Histograms::new(); - block1.used_lines_bitmap.set(1); - block1.set_fragmented(); + block2.used_lines_bytemap.set(1); + block2.used_lines_bytemap.set(3); + block2.update_hole_count(); + block2.set_fragmented(); bucket.add_block(block1); bucket.add_block(block2); + histos.marked.increment(2, 1); - // Normally the collector updates the mark histogram at the end of a - // cycle. Since said code is not executed by the function we're testing - // we'll update this histogram manually. - histos.marked.increment(1, 1); - - assert!(bucket.prepare_for_collection(&histos)); + bucket.prepare_for_collection(&mut histos, true); - assert_eq!( - histos.available.get(1), - // We added two blocks worth of lines. Line 0 in every block is - // reserved, meaning we have to subtract two lines from the number - // of available ones. We also marked line 1 of block 1 as in use, so - // we have to subtract another line. - ((LINES_PER_BLOCK * 2) - 3) as u16 - ); + assert_eq!(histos.available.get(2), (LINES_PER_BLOCK - 3) as u32); let block = bucket.current_block().unwrap(); assert!(block.is_fragmented()); - assert!(block.marked_objects_bitmap.is_empty()); + assert!(block.marked_objects_bytemap.is_empty()); } } diff --git a/vm/src/immix/bytemap.rs b/vm/src/immix/bytemap.rs index 6ad925d9b..c73f3dcc0 100644 --- a/vm/src/immix/bytemap.rs +++ b/vm/src/immix/bytemap.rs @@ -4,6 +4,7 @@ //! are in use. An ObjectMap is used for marking objects while tracing. use crate::immix::block::{LINES_PER_BLOCK, OBJECTS_PER_BLOCK}; use std::mem; +use std::ptr; use std::sync::atomic::{AtomicU8, Ordering}; /// A bytemap for marking objects when they are traced. @@ -12,87 +13,125 @@ pub struct ObjectMap { } /// A bytemap for marking lines when tracing objects. +/// +/// Line maps cycle between two different mark values every collection, instead +/// of always using "1" to mark an entry. This is necessary as during a +/// collection we want to: +/// +/// 1. Be able to determine which lines are live _after_ the collection. +/// 2. Be able to evacuate objects in a bucket _without_ overwriting still live +/// objects. +/// +/// Imagine the eden space is collected, survivor space 1 is about half way +/// through its recyclable blocks, and we need to evacuate objects in survivor +/// space 1: +/// +/// X = used object slot +/// _ = available object slot +/// +/// 1 2 3 4 5 6 7 = line number +/// +------------------------------------------------+ +/// | XXXX | XXXX | XXXX | XX__ | ____ | XXXX | XXXX | = block +/// +------------------------------------------------+ +/// ^ ^ +/// A: allocation cursor B: next full line +/// +/// Here "A" indicates where the next object will be allocated into, and "B" +/// indicates the next full line. If we blindly reset the line map, and "B" +/// includes one or more live objects (that we still have to trace through), we +/// would end up overwriting those objects. +/// +/// Toggling the mark value between 1 and 2 allows us to prevent this from +/// happening. At collection time we first swap the value, then trace through +/// all objects. At the end of a collection we reset all entries using an old +/// value to 0, and then check what to do with the block. +/// +/// We currently store the mark value in the LineMap, which means an additional +/// byte of space is required. Storing this elsewhere would require passing it +/// as an argument to a whole bunch of methods. For a 1GB heap the extra byte +/// (including alignment of 8 bytes) would require 1MB of additional space. pub struct LineMap { values: [AtomicU8; LINES_PER_BLOCK], mark_value: u8, } pub trait Bytemap { - fn max_entries(&self) -> usize; fn values(&self) -> &[AtomicU8]; + fn values_mut(&mut self) -> &mut [AtomicU8]; - /// Sets the given index in the bitmap. - /// - /// # Examples - /// - /// let mut bitmap = ObjectMap::new(); - /// - /// bitmap.set(4); + fn reset(&mut self); + + /// The value to use for marking an entry. + fn mark_value(&self) -> u8 { + 1 + } + + /// Sets the given index in the bytemap. fn set(&mut self, index: usize) { - self.values()[index].store(1, Ordering::Release); + self.values()[index].store(self.mark_value(), Ordering::Release); } - /// Unsets the given index in the bitmap. - /// - /// # Examples - /// - /// let mut bitmap = ObjectMap::new(); - /// - /// bitmap.set(4); - /// bitmap.unset(4); + /// Unsets the given index in the bytemap. fn unset(&mut self, index: usize) { self.values()[index].store(0, Ordering::Release); } /// Returns `true` if a given index is set. - /// - /// # Examples - /// - /// let mut bitmap = ObjectMap::new(); - /// - /// bitmap.is_set(1); // => false - /// - /// bitmap.set(1); - /// - /// bitmap.is_set(1); // => true fn is_set(&self, index: usize) -> bool { - self.values()[index].load(Ordering::Acquire) != 0 + self.values()[index].load(Ordering::Acquire) > 0 } - /// Returns true if the bitmap is empty. + /// Returns true if the bytemap is empty. + /// + /// The number of values in a bytemap is a multiple of 2, and thus a + /// multiple of the word size of the current architecture. Since we store + /// bytes in the bytemap, this allows us to read multiple bytes at once. + /// This in turn allows us to greatly speed up checking if a bytemap is + /// empty. + /// + /// The downside of this is that this method can not be used safely while + /// the bytemap is also being modified. + #[cfg_attr(feature = "cargo-clippy", allow(cast_ptr_alignment))] fn is_empty(&self) -> bool { - for value in self.values().iter() { - if value.load(Ordering::Acquire) != 0 { + let mut offset = 0; + + while offset < self.values().len() { + // The cast to *mut usize here is important so that reads read a + // single word, not a byte. + let value = unsafe { + let ptr = self.values().as_ptr().add(offset) as *const usize; + + *ptr + }; + + if value > 0 { return false; } + + offset += mem::size_of::(); } true } - /// Resets the bitmap. - fn reset(&mut self) { - for index in 0..self.max_entries() { - self.unset(index); - } - } - - /// The number of indexes set in the bitmap. - fn len(&self) -> usize { - let mut count = 0; + /// Returns the number of indexes set in the bytemap. + /// + /// This method can not be used if the bytemap is modified concurrently. + fn len(&mut self) -> usize { + let mut amount = 0; - for value in self.values().iter() { - if value.load(Ordering::Acquire) != 0 { - count += 1; + for value in self.values_mut().iter_mut() { + if *value.get_mut() > 0 { + amount += 1; } } - count + amount } } impl ObjectMap { - /// Returns a new, empty object bitmap. + /// Returns a new, empty object bytemap. pub fn new() -> ObjectMap { let values = [0_u8; OBJECTS_PER_BLOCK]; @@ -103,7 +142,7 @@ impl ObjectMap { } impl LineMap { - /// Returns a new, empty line bitmap. + /// Returns a new, empty line bytemap. pub fn new() -> LineMap { let values = [0_u8; LINES_PER_BLOCK]; @@ -113,10 +152,6 @@ impl LineMap { } } - pub fn set(&mut self, index: usize) { - self.values[index].store(self.mark_value, Ordering::Release); - } - pub fn swap_mark_value(&mut self) { if self.mark_value == 1 { self.mark_value = 2; @@ -127,11 +162,11 @@ impl LineMap { /// Resets marks from previous marking cycles. pub fn reset_previous_marks(&mut self) { - for index in 0..self.max_entries() { - let current = self.values[index].load(Ordering::Acquire); + for index in 0..LINES_PER_BLOCK { + let current = self.values[index].get_mut(); - if current != self.mark_value { - self.values[index].store(0, Ordering::Release); + if *current != self.mark_value { + *current = 0; } } } @@ -143,8 +178,15 @@ impl Bytemap for ObjectMap { &self.values } - fn max_entries(&self) -> usize { - OBJECTS_PER_BLOCK + #[inline(always)] + fn values_mut(&mut self) -> &mut [AtomicU8] { + &mut self.values + } + + fn reset(&mut self) { + unsafe { + ptr::write_bytes(self.values.as_mut_ptr(), 0, OBJECTS_PER_BLOCK); + } } } @@ -154,8 +196,19 @@ impl Bytemap for LineMap { &self.values } - fn max_entries(&self) -> usize { - LINES_PER_BLOCK + #[inline(always)] + fn values_mut(&mut self) -> &mut [AtomicU8] { + &mut self.values + } + + fn mark_value(&self) -> u8 { + self.mark_value + } + + fn reset(&mut self) { + unsafe { + ptr::write_bytes(self.values.as_mut_ptr(), 0, LINES_PER_BLOCK); + } } } @@ -230,18 +283,6 @@ mod tests { assert!(line_map.is_set(1)); } - #[test] - fn test_line_map_set_swap_marks() { - let mut line_map = LineMap::new(); - - line_map.set(1); - line_map.swap_mark_value(); - line_map.set(2); - - assert!(line_map.is_set(1)); - assert!(line_map.is_set(2)); - } - #[test] fn test_line_map_unset() { let mut line_map = LineMap::new(); @@ -261,6 +302,11 @@ mod tests { line_map.set(1); assert_eq!(line_map.is_empty(), false); + + line_map.unset(1); + line_map.set(60); + + assert_eq!(line_map.is_empty(), false); } #[test] @@ -289,32 +335,4 @@ mod tests { // grow due to some change. assert_eq!(size_of::(), LINES_PER_BLOCK + 1); } - - #[test] - fn test_line_map_swap_mark_value() { - let mut line_map = LineMap::new(); - - assert_eq!(line_map.mark_value, 1); - - line_map.swap_mark_value(); - - assert_eq!(line_map.mark_value, 2); - } - - #[test] - fn test_line_map_reset_previous_marks() { - let mut line_map = LineMap::new(); - - line_map.set(1); - line_map.set(2); - - line_map.swap_mark_value(); - - line_map.set(3); - line_map.reset_previous_marks(); - - assert_eq!(line_map.is_set(1), false); - assert_eq!(line_map.is_set(2), false); - assert_eq!(line_map.is_set(3), true); - } } diff --git a/vm/src/immix/copy_object.rs b/vm/src/immix/copy_object.rs index c746c9548..128530dd8 100644 --- a/vm/src/immix/copy_object.rs +++ b/vm/src/immix/copy_object.rs @@ -104,68 +104,6 @@ pub trait CopyObject: Sized { self.allocate_copy(copy) } - - /// Performs a deep move of the given pointer. - /// - /// This will copy over the object to the current heap, while _moving_ all - /// related data from the old object into the new one. - fn move_object(&mut self, to_copy_ptr: ObjectPointer) -> ObjectPointer { - if to_copy_ptr.is_permanent() { - return to_copy_ptr; - } - - let to_copy = to_copy_ptr.get_mut(); - - let value_copy = match to_copy.value.take() { - ObjectValue::Array(mut array) => { - for index in 0..array.len() { - array[index] = self.move_object(array[index]); - } - - ObjectValue::Array(array) - } - ObjectValue::Block(mut block) => { - if let Some(captures_from) = block.captures_from.as_mut() { - captures_from.move_pointers_to(self); - } - - block.receiver = self.move_object(block.receiver); - - ObjectValue::Block(block) - } - ObjectValue::Binding(mut binding) => { - binding.move_pointers_to(self); - - ObjectValue::Binding(binding) - } - value => value, - }; - - let mut copy = if let Some(proto_ptr) = to_copy.take_prototype() { - let proto_copy = self.move_object(proto_ptr); - - Object::with_prototype(value_copy, proto_copy) - } else { - Object::new(value_copy) - }; - - if let Some(map) = to_copy.attributes_map() { - let mut map_copy = AttributesMap::default(); - - for (key, val) in map.iter() { - let key_copy = self.move_object(*key); - let val_copy = self.move_object(*val); - - map_copy.insert(key_copy, val_copy); - } - - copy.set_attributes_map(map_copy); - } - - to_copy.drop_attributes(); - - self.allocate_copy(copy) - } } #[cfg(test)] @@ -373,172 +311,4 @@ mod tests { assert_eq!(local1_copy.float_value().unwrap(), 20.0); assert_eq!(local2_copy.float_value().unwrap(), 15.0); } - - #[test] - fn test_move_none() { - let mut dummy = DummyAllocator::new(); - let pointer = dummy.allocator.allocate_empty(); - let copy = dummy.move_object(pointer); - - assert!(copy.get().value.is_none()); - } - - #[test] - fn test_move_with_prototype() { - let mut dummy = DummyAllocator::new(); - let pointer = dummy.allocator.allocate_empty(); - let proto = dummy.allocator.allocate_empty(); - - pointer.get_mut().set_prototype(proto); - - let copy = dummy.move_object(pointer); - - assert!(copy.get().prototype().is_some()); - assert!(pointer.get().prototype().is_none()); - } - - #[test] - fn test_move_with_attributes() { - let mut dummy = DummyAllocator::new(); - let ptr1 = dummy.allocator.allocate_empty(); - let ptr2 = dummy.allocator.allocate_empty(); - let name = dummy.allocator.allocate_empty(); - - ptr1.get_mut().add_attribute(name, ptr2); - - let copy = dummy.move_object(ptr1); - - assert_eq!(ptr1.is_finalizable(), false); - assert!(ptr1.get().attributes_map().is_none()); - - assert!(copy.is_finalizable()); - assert!(copy.get().attributes_map().is_some()); - } - - #[test] - fn test_move_integer() { - let mut dummy = DummyAllocator::new(); - let pointer = dummy - .allocator - .allocate_without_prototype(object_value::integer(5)); - - let copy = dummy.move_object(pointer); - - assert!(pointer.get().value.is_none()); - - assert!(copy.get().value.is_integer()); - assert_eq!(copy.integer_value().unwrap(), 5); - } - - #[test] - fn test_move_float() { - let mut dummy = DummyAllocator::new(); - let pointer = dummy - .allocator - .allocate_without_prototype(object_value::float(2.5)); - - let copy = dummy.move_object(pointer); - - assert!(pointer.get().value.is_none()); - - assert!(copy.get().value.is_float()); - assert_eq!(copy.get().value.as_float().unwrap(), 2.5); - } - - #[test] - fn test_move_string() { - let mut dummy = DummyAllocator::new(); - let pointer = dummy - .allocator - .allocate_without_prototype(object_value::string("a".to_string())); - - let copy = dummy.move_object(pointer); - - assert!(pointer.get().value.is_none()); - - assert!(copy.get().value.is_string()); - assert_eq!(copy.string_value().unwrap().as_slice(), "a"); - } - - #[test] - fn test_move_array() { - let mut dummy = DummyAllocator::new(); - let ptr1 = dummy.allocator.allocate_empty(); - let ptr2 = dummy.allocator.allocate_empty(); - let array = dummy - .allocator - .allocate_without_prototype(object_value::array(vec![ptr1, ptr2])); - - let copy = dummy.move_object(array); - - assert!(array.get().value.is_none()); - - assert!(copy.get().value.is_array()); - assert_eq!(copy.get().value.as_array().unwrap().len(), 2); - } - - #[test] - fn test_move_block() { - let mut dummy = DummyAllocator::new(); - let state = state(); - let cc = CompiledCode::new( - state.intern_string("a".to_string()), - state.intern_string("a".to_string()), - 1, - Vec::new(), - ); - - let scope = GlobalScope::new(); - - let block = Block::new( - DerefPointer::new(&cc), - None, - ObjectPointer::integer(1), - GlobalScopePointer::new(&scope), - ); - - let ptr = dummy - .allocator - .allocate_without_prototype(object_value::block(block)); - - let copy = dummy.move_object(ptr); - - assert!(ptr.get().value.is_none()); - assert!(copy.get().value.is_block()); - } - - #[test] - fn test_move_binding() { - let mut dummy = DummyAllocator::new(); - - let local1 = dummy - .allocator - .allocate_without_prototype(object_value::float(15.0)); - - let local2 = dummy - .allocator - .allocate_without_prototype(object_value::float(20.0)); - - let receiver = dummy - .allocator - .allocate_without_prototype(object_value::float(12.0)); - - let mut binding1 = Binding::with_rc(1, ObjectPointer::integer(1)); - let mut binding2 = Binding::with_rc(1, receiver); - - binding1.set_local(0, local1); - binding2.set_local(0, local2); - - let binding_ptr = dummy - .allocator - .allocate_without_prototype(object_value::binding(binding2)); - - let binding_move_ptr = dummy.move_object(binding_ptr); - let binding_move = binding_move_ptr.get(); - let binding_val = binding_move.value.as_binding().unwrap(); - - assert!(binding_val.local_exists(0)); - assert!(binding_ptr.get().value.is_none()); - assert_eq!(binding_val.receiver.float_value().unwrap(), 12.0); - } } diff --git a/vm/src/immix/generation_config.rs b/vm/src/immix/generation_config.rs index 65e8b5486..a3f0e85aa 100644 --- a/vm/src/immix/generation_config.rs +++ b/vm/src/immix/generation_config.rs @@ -1,4 +1,5 @@ //! Configuration for heap generations. +use crate::config::Config; pub struct GenerationConfig { /// The maximum number of blocks that can be allocated before triggering a @@ -23,10 +24,10 @@ impl GenerationConfig { /// The `blocks` argument should specify the current number of live blocks. pub fn should_increase_threshold( &self, - blocks: u32, + blocks: usize, growth_threshold: f64, ) -> bool { - let percentage = f64::from(blocks) / f64::from(self.threshold); + let percentage = blocks as f64 / f64::from(self.threshold); percentage >= growth_threshold } @@ -36,6 +37,24 @@ impl GenerationConfig { (f64::from(self.threshold) * growth_factor).ceil() as u32; } + pub fn update_after_collection( + &mut self, + config: &Config, + blocks: usize, + ) -> bool { + let max = config.heap_growth_threshold; + let factor = config.heap_growth_factor; + + self.block_allocations = 0; + + if self.should_increase_threshold(blocks, max) { + self.increment_threshold(factor); + true + } else { + false + } + } + pub fn allocation_threshold_exceeded(&self) -> bool { self.block_allocations >= self.threshold } @@ -98,4 +117,19 @@ mod tests { assert_eq!(config.block_allocations, 1); assert!(config.allocation_threshold_exceeded()); } + + #[test] + fn test_update_after_collection() { + let mut gen_config = GenerationConfig::new(1); + let mut vm_config = Config::new(); + + assert_eq!(gen_config.update_after_collection(&vm_config, 0), false); + + vm_config.heap_growth_factor = 2.0; + gen_config.threshold = 4; + gen_config.block_allocations = 4; + + assert!(gen_config.update_after_collection(&vm_config, 4)); + assert_eq!(gen_config.threshold, 8); + } } diff --git a/vm/src/immix/global_allocator.rs b/vm/src/immix/global_allocator.rs index bad6340d0..a880bc97e 100644 --- a/vm/src/immix/global_allocator.rs +++ b/vm/src/immix/global_allocator.rs @@ -6,26 +6,26 @@ use crate::arc_without_weak::ArcWithoutWeak; use crate::immix::block::Block; use crate::immix::block_list::BlockList; -use crossbeam_queue::SegQueue; +use parking_lot::Mutex; pub type RcGlobalAllocator = ArcWithoutWeak; /// Structure used for storing the state of the global allocator. pub struct GlobalAllocator { - blocks: SegQueue>, + blocks: Mutex, } impl GlobalAllocator { /// Creates a new GlobalAllocator with a number of blocks pre-allocated. pub fn with_rc() -> RcGlobalAllocator { ArcWithoutWeak::new(GlobalAllocator { - blocks: SegQueue::new(), + blocks: Mutex::new(BlockList::new()), }) } /// Requests a new free block from the pool pub fn request_block(&self) -> Box { - if let Ok(block) = self.blocks.pop() { + if let Some(block) = self.blocks.lock().pop_front() { block } else { Block::boxed() @@ -34,26 +34,27 @@ impl GlobalAllocator { /// Adds a block to the pool so it can be re-used. pub fn add_block(&self, block: Box) { - self.blocks.push(block); + self.blocks.lock().push_front(block); } /// Adds multiple blocks to the global allocator. pub fn add_blocks(&self, to_add: &mut BlockList) { - for block in to_add.drain() { - self.add_block(block); - } + let mut blocks = self.blocks.lock(); + + blocks.append(to_add); } } #[cfg(test)] mod tests { use super::*; + use crate::immix::block_list::BlockList; #[test] fn test_new() { let alloc = GlobalAllocator::with_rc(); - assert!(alloc.blocks.pop().is_err()); + assert!(alloc.blocks.lock().pop_front().is_none()); } #[test] @@ -64,7 +65,7 @@ mod tests { alloc.add_block(block); alloc.request_block(); - assert!(alloc.blocks.pop().is_err()); + assert!(alloc.blocks.lock().pop_front().is_none()); } #[test] @@ -74,6 +75,18 @@ mod tests { alloc.add_block(block); - assert!(alloc.blocks.pop().is_ok()); + assert!(alloc.blocks.lock().pop_front().is_some()); + } + + #[test] + fn test_add_blocks() { + let alloc = GlobalAllocator::with_rc(); + let mut blocks = BlockList::new(); + + blocks.push_front(alloc.request_block()); + blocks.push_front(alloc.request_block()); + alloc.add_blocks(&mut blocks); + + assert_eq!(alloc.blocks.lock().len(), 2); } } diff --git a/vm/src/immix/histogram.rs b/vm/src/immix/histogram.rs index 74b363ad9..1f8617f6d 100644 --- a/vm/src/immix/histogram.rs +++ b/vm/src/immix/histogram.rs @@ -3,22 +3,19 @@ //! A Histogram is used to track the distribution of marked and available lines //! across Immix blocks. Each bin represents the number of holes with the values //! representing the number of marked lines. -//! -//! Histograms are of a fixed size and use atomic operations for incrementing -//! bucket values, allowing concurrent use of the same histogram. use crate::chunk::Chunk; -use std::sync::atomic::{AtomicU16, Ordering}; -const DEFAULT_VALUE: u16 = 0; +/// The minimum bin number that we care about when obtaining the most fragmented +/// bins. +/// +/// Bins 0 and 1 are not interesting, because blocks with 0 or 1 holes are not +/// used for calculating fragmentation statistics. +pub const MINIMUM_BIN: usize = 2; pub struct Histogram { - values: Chunk, -} - -/// Iterator for traversing the most fragmented bins in a histogram. -pub struct HistogramIterator<'a> { - histogram: &'a Histogram, - index: isize, + // We use a u32 as this allows for 4 294 967 295 lines per bucket, which + // equals roughly 512 GB of lines. + values: Chunk, } impl Histogram { @@ -32,62 +29,30 @@ impl Histogram { /// /// Bounds checking is not performed, as the garbage collector never uses an /// out of bounds index. - pub fn increment(&self, index: usize, value: u16) { - self.values[index].fetch_add(value, Ordering::Release); + pub fn increment(&mut self, index: usize, value: u32) { + debug_assert!(index < self.values.len()); + + self.values[index] += value; } /// Returns the value for the given bin. /// /// Bounds checking is not performed, as the garbage collector never uses an /// out of bounds index. - pub fn get(&self, index: usize) -> u16 { - self.values[index].load(Ordering::Acquire) - } - - /// Returns the most fragmented bin. - pub fn most_fragmented_bin(&self) -> usize { - for bin in (0..self.values.len()).rev() { - if self.values[bin].load(Ordering::Acquire) > DEFAULT_VALUE { - return bin; - } - } - - 0 - } - - /// Returns an iterator for traversing the most fragmented bins in - /// descending order. - pub fn iter(&self) -> HistogramIterator { - HistogramIterator { - index: self.most_fragmented_bin() as isize, - histogram: self, - } + pub fn get(&self, index: usize) -> u32 { + debug_assert!( + index < self.values.len(), + "index is {} but the length is {}", + index, + self.values.len() + ); + + self.values[index] } /// Removes all values from the histogram. pub fn reset(&mut self) { - for index in 0..self.values.len() { - self.values[index].store(DEFAULT_VALUE, Ordering::Release); - } - } -} - -impl<'a> Iterator for HistogramIterator<'a> { - type Item = usize; - - fn next(&mut self) -> Option { - while self.index >= 0 { - let index = self.index as usize; - let value = self.histogram.get(index as usize); - - self.index -= 1; - - if value > 0 { - return Some(index); - } - } - - None + self.values.reset(); } } @@ -104,7 +69,7 @@ mod tests { #[test] fn test_increment() { - let histo = Histogram::new(1); + let mut histo = Histogram::new(1); histo.increment(0, 10); @@ -113,7 +78,7 @@ mod tests { #[test] fn test_increment_successive() { - let histo = Histogram::new(1); + let mut histo = Histogram::new(1); histo.increment(0, 5); histo.increment(0, 5); @@ -121,32 +86,6 @@ mod tests { assert_eq!(histo.get(0), 10); } - #[test] - fn test_most_fragmented_bin() { - let histo = Histogram::new(2); - - histo.increment(0, 5); - histo.increment(1, 7); - - assert_eq!(histo.most_fragmented_bin(), 1); - } - - #[test] - fn test_iter() { - let histo = Histogram::new(3); - - histo.increment(0, 10); - histo.increment(1, 20); - histo.increment(2, 25); - - let mut iter = histo.iter(); - - assert_eq!(iter.next().unwrap(), 2); - assert_eq!(iter.next().unwrap(), 1); - assert_eq!(iter.next().unwrap(), 0); - assert!(iter.next().is_none()); - } - #[test] fn test_reset() { let mut histo = Histogram::new(1); diff --git a/vm/src/immix/histograms.rs b/vm/src/immix/histograms.rs index acd774caa..a0c881df6 100644 --- a/vm/src/immix/histograms.rs +++ b/vm/src/immix/histograms.rs @@ -17,8 +17,8 @@ unsafe impl Sync for Histograms {} impl Histograms { pub fn new() -> Self { Self { - available: Histogram::new(MAX_HOLES), - marked: Histogram::new(LINES_PER_BLOCK), + available: Histogram::new(MAX_HOLES + 1), + marked: Histogram::new(LINES_PER_BLOCK + 1), } } diff --git a/vm/src/immix/local_allocator.rs b/vm/src/immix/local_allocator.rs index f4c226cde..993ffe547 100644 --- a/vm/src/immix/local_allocator.rs +++ b/vm/src/immix/local_allocator.rs @@ -3,18 +3,17 @@ //! The LocalAllocator lives in a Process and is used for allocating memory on a //! process heap. use crate::config::Config; -use crate::gc::work_list::WorkList; +use crate::gc::remembered_set::RememberedSet; use crate::immix::bucket::{Bucket, MATURE}; use crate::immix::copy_object::CopyObject; use crate::immix::generation_config::GenerationConfig; use crate::immix::global_allocator::RcGlobalAllocator; use crate::immix::histograms::Histograms; use crate::object::Object; -use crate::object_pointer::ObjectPointer; +use crate::object_pointer::{ObjectPointer, ObjectPointerPointer}; use crate::object_value; use crate::object_value::ObjectValue; -use crate::vm::state::RcState; -use fnv::FnvHashSet; +use crate::vm::state::State; /// The maximum age of a bucket in the young generation. pub const YOUNG_MAX_AGE: i8 = 2; @@ -35,12 +34,13 @@ pub struct LocalAllocator { pub mature_histograms: Histograms, /// The position of the eden bucket in the young generation. - pub eden_index: usize, + /// + /// This is a u8 to conserve space, as we'll never have more than 255 + /// buckets to choose from. + pub eden_index: u8, - /// The remembered set of this process. This set is not synchronized via a - /// lock of sorts. As such the collector must ensure this process is - /// suspended upon examining the remembered set. - pub remembered_set: FnvHashSet, + /// A collection of mature objects that contain pointers to young objects. + pub remembered_set: RememberedSet, /// The bucket to use for the mature generation. pub mature_generation: Bucket, @@ -50,6 +50,14 @@ pub struct LocalAllocator { /// The configuration for the mature generation. pub mature_config: GenerationConfig, + + /// A boolean indicating if we should evacuate objects in the young + /// generation. + evacuate_young: bool, + + /// A boolean indicating if we should evacuate objects in the mature + /// generation. + evacuate_mature: bool, } impl LocalAllocator { @@ -70,7 +78,9 @@ impl LocalAllocator { mature_generation: Bucket::with_age(MATURE), young_config: GenerationConfig::new(config.young_threshold), mature_config: GenerationConfig::new(config.mature_threshold), - remembered_set: FnvHashSet::default(), + remembered_set: RememberedSet::new(), + evacuate_young: false, + evacuate_mature: false, } } @@ -79,11 +89,11 @@ impl LocalAllocator { } pub fn eden_space(&self) -> &Bucket { - &self.young_generation[self.eden_index] + &self.young_generation[self.eden_index as usize] } pub fn eden_space_mut(&mut self) -> &mut Bucket { - &mut self.young_generation[self.eden_index] + &mut self.young_generation[self.eden_index as usize] } pub fn should_collect_young(&self) -> bool { @@ -98,50 +108,68 @@ impl LocalAllocator { /// /// Returns true if objects have to be moved around. pub fn prepare_for_collection(&mut self, mature: bool) -> bool { - let mut move_objects = false; + let mut move_objects = self.evacuate_young; for bucket in &mut self.young_generation { - if bucket.prepare_for_collection(&self.young_histograms) { - move_objects = true; - } + bucket.prepare_for_collection( + &mut self.young_histograms, + self.evacuate_young, + ); - if bucket.promote { + if bucket.age == YOUNG_MAX_AGE { move_objects = true; } } if mature { - if self - .mature_generation - .prepare_for_collection(&self.mature_histograms) - { + self.mature_generation.prepare_for_collection( + &mut self.mature_histograms, + self.evacuate_mature, + ); + + if self.evacuate_mature { move_objects = true; } - } else if self.has_remembered_objects() { - self.prepare_remembered_objects_for_collection(); } move_objects } /// Reclaims blocks in the young (and mature) generation. - pub fn reclaim_blocks(&mut self, state: &RcState, mature: bool) { + pub fn reclaim_blocks(&mut self, state: &State, mature: bool) { + self.reclaim_young_blocks(&state); + + if mature { + self.reclaim_mature_blocks(&state); + } + } + + fn reclaim_young_blocks(&mut self, state: &State) { self.young_histograms.reset(); + let mut blocks = 0; + for bucket in &mut self.young_generation { - bucket.reclaim_blocks(state, &self.young_histograms); + blocks += bucket.reclaim_blocks(state, &mut self.young_histograms); } - if mature { - self.mature_histograms.reset(); + self.increment_young_ages(); - self.mature_generation - .reclaim_blocks(state, &self.mature_histograms); - } else { - for block in self.mature_generation.blocks.iter_mut() { - block.update_line_map(); - } - } + self.evacuate_young = self + .young_config + .update_after_collection(&state.config, blocks); + } + + fn reclaim_mature_blocks(&mut self, state: &State) { + self.mature_histograms.reset(); + + let blocks = self + .mature_generation + .reclaim_blocks(state, &mut self.mature_histograms); + + self.evacuate_mature = self + .mature_config + .update_after_collection(&state.config, blocks); } pub fn allocate_with_prototype( @@ -169,7 +197,9 @@ impl LocalAllocator { } pub fn allocate_eden(&mut self, object: Object) -> ObjectPointer { - let (new_block, pointer) = self.allocate_eden_raw(object); + let (new_block, pointer) = self.young_generation + [self.eden_index as usize] + .allocate(&self.global_allocator, object); if new_block { self.young_config.increment_allocations(); @@ -179,7 +209,9 @@ impl LocalAllocator { } pub fn allocate_mature(&mut self, object: Object) -> ObjectPointer { - let (new_block, pointer) = self.allocate_mature_raw(object); + let (new_block, pointer) = self + .mature_generation + .allocate(&self.global_allocator, object); if new_block { self.mature_config.increment_allocations(); @@ -195,104 +227,54 @@ impl LocalAllocator { bucket.reset_age(); } else { bucket.increment_age(); - bucket.promote = bucket.age == YOUNG_MAX_AGE; } if bucket.age == 0 { - self.eden_index = index; + self.eden_index = index as u8; } } } - pub fn update_collection_statistics( - &mut self, - config: &Config, - mature: bool, - ) { - self.update_young_collection_statistics(config); - - if mature { - self.update_mature_collection_statistics(config); - } - } - - pub fn update_young_collection_statistics(&mut self, config: &Config) { - self.increment_young_ages(); - - self.young_config.block_allocations = 0; - - let blocks = self.number_of_young_blocks(); - let max = config.heap_growth_threshold; - let factor = config.heap_growth_factor; - - if self.young_config.should_increase_threshold(blocks, max) { - self.young_config.increment_threshold(factor); - } - } - - pub fn update_mature_collection_statistics(&mut self, config: &Config) { - self.update_young_collection_statistics(config); - - self.mature_config.block_allocations = 0; - - let blocks = self.mature_generation.number_of_blocks(); - let max = config.heap_growth_threshold; - let factor = config.heap_growth_factor; - - if self.mature_config.should_increase_threshold(blocks, max) { - self.mature_config.increment_threshold(factor); + pub fn remember_object(&mut self, pointer: ObjectPointer) { + if pointer.is_remembered() { + return; } - } - pub fn number_of_young_blocks(&self) -> u32 { - self.young_generation - .iter() - .map(Bucket::number_of_blocks) - .sum() - } + pointer.mark_as_remembered(); - pub fn has_remembered_objects(&self) -> bool { - !self.remembered_set.is_empty() - } - - pub fn remember_object(&mut self, pointer: ObjectPointer) { - self.remembered_set.insert(pointer); + self.remembered_set.remember(pointer); } pub fn prune_remembered_objects(&mut self) { - self.remembered_set.retain(ObjectPointer::is_marked); + self.remembered_set.prune(); } - pub fn prepare_remembered_objects_for_collection(&mut self) { - for pointer in &self.remembered_set { - // We prepare the entire block because this is simpler than having - // to figure out if we can unmark a line or not (since a line may - // contain multiple objects). - pointer.block_mut().prepare_for_collection(); + pub fn each_remembered_pointer(&mut self, mut callback: F) + where + F: FnMut(ObjectPointerPointer), + { + if self.remembered_set.is_empty() { + return; } - } - - pub fn remembered_pointers(&self) -> WorkList { - let mut pointers = WorkList::new(); - - for pointer in &self.remembered_set { - pointers.push(pointer.pointer()); - } - - pointers - } - fn allocate_eden_raw(&mut self, object: Object) -> (bool, ObjectPointer) { - unsafe { - self.young_generation[self.eden_index] - .allocate_for_mutator(&self.global_allocator, object) + for pointer in self.remembered_set.iter() { + // In a young collection we want to (re-)trace all remembered + // objects. Mark values for the mature space are only updated during + // a mature collection. We don't care about nested mature objects, + // as those will be in the remembered set if they contain any young + // pointers. + // + // Line mark states should remain as-is, so we don't promote mature + // objects into already used mature lines. + // + // Because of this, all we need to do here is unmark the mature + // objects we have remembered; ensuring we will trace them for any + // young pointers. + pointer.unmark(); + + callback(pointer.pointer()); } } - - fn allocate_mature_raw(&mut self, object: Object) -> (bool, ObjectPointer) { - self.mature_generation - .allocate(&self.global_allocator, object) - } } impl CopyObject for LocalAllocator { @@ -343,9 +325,37 @@ mod tests { assert_eq!(alloc.prepare_for_collection(true), false); - alloc.young_generation[0].promote = true; + alloc.young_generation[0].increment_age(); + alloc.young_generation[0].increment_age(); + + assert!(alloc.prepare_for_collection(true)); + } + + #[test] + fn test_prepare_for_collection_with_promotion() { + let (_, mut alloc) = local_allocator(); + + alloc.increment_young_ages(); + alloc.increment_young_ages(); + alloc.increment_young_ages(); - assert_eq!(alloc.prepare_for_collection(true), true); + assert!(alloc.prepare_for_collection(false)); + } + + #[test] + fn test_each_remembered_pointer() { + let (_, mut alloc) = local_allocator(); + let mature = alloc.allocate_mature(Object::new(object_value::none())); + + alloc.remember_object(mature); + mature.mark(); + + let mut pointers = Vec::new(); + + alloc.each_remembered_pointer(|ptr| pointers.push(ptr)); + + assert!(pointers.pop().is_some()); + assert_eq!(mature.is_marked(), false); } #[test] @@ -444,7 +454,6 @@ mod tests { assert_eq!(alloc.young_generation[0].age, 2); assert_eq!(alloc.young_generation[1].age, 1); assert_eq!(alloc.young_generation[2].age, 0); - assert_eq!(alloc.young_generation[0].promote, true); assert_eq!(alloc.eden_index, 2); alloc.increment_young_ages(); @@ -452,19 +461,13 @@ mod tests { assert_eq!(alloc.young_generation[0].age, 0); assert_eq!(alloc.young_generation[1].age, 2); assert_eq!(alloc.young_generation[2].age, 1); - assert_eq!(alloc.young_generation[1].promote, true); assert_eq!(alloc.eden_index, 0); alloc.increment_young_ages(); assert_eq!(alloc.young_generation[0].age, 1); - assert_eq!(alloc.young_generation[0].promote, false); - assert_eq!(alloc.young_generation[1].age, 0); - assert_eq!(alloc.young_generation[1].promote, false); - assert_eq!(alloc.young_generation[2].age, 2); - assert_eq!(alloc.young_generation[2].promote, true); assert_eq!(alloc.eden_index, 1); alloc.increment_young_ages(); @@ -494,36 +497,37 @@ mod tests { alloc.remember_object(pointer); - assert!(alloc.has_remembered_objects()); + assert_eq!(alloc.remembered_set.is_empty(), false); + assert!(pointer.is_remembered()); } #[test] - fn test_prune_remembered_objects() { + fn test_remember_object_already_remembered() { let (_, mut alloc) = local_allocator(); - let ptr1 = alloc.allocate_empty(); - let ptr2 = alloc.allocate_empty(); - - ptr1.mark(); + let pointer = alloc.allocate_empty(); - alloc.remember_object(ptr1); - alloc.remember_object(ptr2); - alloc.prune_remembered_objects(); + alloc.remember_object(pointer); + alloc.remember_object(pointer); - assert_eq!(alloc.remembered_set.contains(&ptr1), true); - assert_eq!(alloc.remembered_set.contains(&ptr2), false); + assert_eq!(alloc.remembered_set.iter().count(), 1); } #[test] - fn test_prepare_remembered_objects_for_collection() { + fn test_prune_remembered_objects() { let (_, mut alloc) = local_allocator(); let ptr1 = alloc.allocate_empty(); + let ptr2 = alloc.allocate_empty(); ptr1.mark(); alloc.remember_object(ptr1); - alloc.prepare_remembered_objects_for_collection(); + alloc.remember_object(ptr2); + alloc.prune_remembered_objects(); + + let mut iter = alloc.remembered_set.iter(); - assert_eq!(ptr1.is_marked(), false); + assert!(iter.next() == Some(&ptr1)); + assert!(iter.next().is_none()); } #[test] diff --git a/vm/src/immix/mailbox_allocator.rs b/vm/src/immix/mailbox_allocator.rs deleted file mode 100644 index 8b9ec310f..000000000 --- a/vm/src/immix/mailbox_allocator.rs +++ /dev/null @@ -1,128 +0,0 @@ -//! Allocator for process mailboxes -//! -//! Each mailbox has its own allocator and its own heap. Incoming messages are -//! copied into this heap. When a message is received its copied from the -//! mailbox heap to the process local heap. - -use crate::config::Config; -use crate::immix::bucket::{Bucket, MAILBOX}; -use crate::immix::copy_object::CopyObject; -use crate::immix::generation_config::GenerationConfig; -use crate::immix::global_allocator::RcGlobalAllocator; -use crate::immix::histograms::Histograms; -use crate::object::Object; -use crate::object_pointer::ObjectPointer; -use crate::vm::state::RcState; - -pub struct MailboxAllocator { - global_allocator: RcGlobalAllocator, - - /// The histograms to use for collecting garbage. - pub histograms: Histograms, - - /// The bucket to allocate objects into. - pub bucket: Bucket, - - /// The heap configuration. - pub config: GenerationConfig, -} - -impl MailboxAllocator { - pub fn new(global_allocator: RcGlobalAllocator, config: &Config) -> Self { - MailboxAllocator { - global_allocator, - histograms: Histograms::new(), - bucket: Bucket::with_age(MAILBOX), - config: GenerationConfig::new(config.mailbox_threshold), - } - } - - pub fn allocate(&mut self, object: Object) -> ObjectPointer { - let (new_block, pointer) = unsafe { - self.bucket - .allocate_for_mutator(&self.global_allocator, object) - }; - - if new_block { - self.config.increment_allocations(); - } - - pointer - } - - /// Prepares a garbage collection cycle, returns true if objects have to be - /// moved around. - pub fn prepare_for_collection(&mut self) -> bool { - self.bucket.prepare_for_collection(&self.histograms) - } - - /// Returns unused blocks to the global allocator. - pub fn reclaim_blocks(&mut self, state: &RcState) { - self.histograms.reset(); - self.bucket.reclaim_blocks(state, &self.histograms); - } - - pub fn should_collect(&self) -> bool { - self.config.allocation_threshold_exceeded() - } - - pub fn update_collection_statistics(&mut self, config: &Config) { - self.config.block_allocations = 0; - - let blocks = self.bucket.number_of_blocks(); - - if self - .config - .should_increase_threshold(blocks, config.heap_growth_threshold) - { - self.config.increment_threshold(config.heap_growth_factor); - } - } -} - -impl CopyObject for MailboxAllocator { - fn allocate_copy(&mut self, object: Object) -> ObjectPointer { - self.allocate(object) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::Config; - use crate::immix::copy_object::CopyObject; - use crate::immix::global_allocator::GlobalAllocator; - use crate::immix::local_allocator::LocalAllocator; - use crate::object::Object; - use crate::object_value; - - fn mailbox_allocator() -> MailboxAllocator { - let global = GlobalAllocator::with_rc(); - - MailboxAllocator::new(global, &Config::new()) - } - - #[test] - fn test_allocate() { - let mut alloc = mailbox_allocator(); - let pointer = alloc.allocate(Object::new(object_value::none())); - - assert!(pointer.is_mailbox()); - assert!(pointer.get().value.is_none()); - } - - #[test] - fn test_copy_object() { - let mut mbox_alloc = mailbox_allocator(); - let global_alloc = mbox_alloc.global_allocator.clone(); - let mut local_alloc = LocalAllocator::new(global_alloc, &Config::new()); - - let original = - local_alloc.allocate_without_prototype(object_value::float(5.0)); - - let copy = mbox_alloc.copy_object(original); - - assert!(copy.is_mailbox()); - assert!(copy.get().value.is_float()); - } -} diff --git a/vm/src/immix/mod.rs b/vm/src/immix/mod.rs index 3d1f3e22a..cd7d9a853 100644 --- a/vm/src/immix/mod.rs +++ b/vm/src/immix/mod.rs @@ -8,5 +8,4 @@ pub mod global_allocator; pub mod histogram; pub mod histograms; pub mod local_allocator; -pub mod mailbox_allocator; pub mod permanent_allocator; diff --git a/vm/src/immix/permanent_allocator.rs b/vm/src/immix/permanent_allocator.rs index f8bf54e3b..6fa7c48d4 100644 --- a/vm/src/immix/permanent_allocator.rs +++ b/vm/src/immix/permanent_allocator.rs @@ -45,10 +45,7 @@ impl PermanentAllocator { } fn allocate(&mut self, object: Object) -> ObjectPointer { - let (_, pointer) = unsafe { - self.bucket - .allocate_for_mutator(&self.global_allocator, object) - }; + let (_, pointer) = self.bucket.allocate(&self.global_allocator, object); pointer.mark(); pointer diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 0bae5c14d..a2f4b67cc 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -49,5 +49,4 @@ pub mod socket; pub mod stacktrace; pub mod string_pool; pub mod tagged_pointer; -pub mod timer; pub mod vm; diff --git a/vm/src/mailbox.rs b/vm/src/mailbox.rs index 72a7f89c5..5a51d19e9 100644 --- a/vm/src/mailbox.rs +++ b/vm/src/mailbox.rs @@ -1,208 +1,80 @@ -use crate::config::Config; -use crate::gc::work_list::WorkList; -use crate::immix::copy_object::CopyObject; -use crate::immix::global_allocator::RcGlobalAllocator; -use crate::immix::mailbox_allocator::MailboxAllocator; -use crate::object_pointer::ObjectPointer; -use parking_lot::Mutex; -use std::collections::LinkedList; -use std::sync::atomic::{AtomicUsize, Ordering}; +use crate::object_pointer::{ObjectPointer, ObjectPointerPointer}; +use std::collections::VecDeque; pub struct Mailbox { - /// Messages sent from external processes. - pub external: LinkedList, - - /// Messages that were moved from the external to the internal queue, or - /// were sent by the owning process itself. - pub internal: LinkedList, - - /// The allocator to use for storing messages. - pub allocator: MailboxAllocator, - - /// A lock to use when synchronising various operations, such as sending - /// messages from external processes. - pub write_lock: Mutex<()>, - - /// The number of messages stored in this mailbox. - /// - /// Since messages can be stored in either the synchronised external half - /// or the unsynchronised internal half, obtaining this number would be - /// expensive. Storing it separately and using atomic operations to access - /// it allows us to more efficiently retrieve this number, at the cost of a - /// little bit of extra memory. - amount: AtomicUsize, + /// The messages stored in this mailbox. + messages: VecDeque, } impl Mailbox { - pub fn new(global_allocator: RcGlobalAllocator, config: &Config) -> Self { + pub fn new() -> Self { Mailbox { - external: LinkedList::new(), - internal: LinkedList::new(), - allocator: MailboxAllocator::new(global_allocator, config), - write_lock: Mutex::new(()), - amount: AtomicUsize::new(0), + messages: VecDeque::new(), } } - pub fn send_from_external(&mut self, original: ObjectPointer) { - let _lock = self.write_lock.lock(); - - self.external - .push_back(self.allocator.copy_object(original)); - - self.amount.fetch_add(1, Ordering::AcqRel); + pub fn send(&mut self, message: ObjectPointer) { + self.messages.push_back(message); } - pub fn send_from_self(&mut self, pointer: ObjectPointer) { - self.internal.push_back(pointer); - self.amount.fetch_add(1, Ordering::AcqRel); + pub fn receive(&mut self) -> Option { + self.messages.pop_front() } - /// Returns a tuple containing a boolean and an optional message. - /// - /// If the boolean is set to `true`, the returned pointer must be copied to - /// a process' local heap. - pub fn receive(&mut self) -> (bool, Option) { - if self.internal.is_empty() { - let _lock = self.write_lock.lock(); - - self.internal.append(&mut self.external); - } - - if let Some(pointer) = self.internal.pop_front() { - self.amount.fetch_sub(1, Ordering::AcqRel); - - (pointer.is_mailbox(), Some(pointer)) - } else { - (false, None) + pub fn each_pointer(&self, mut callback: F) + where + F: FnMut(ObjectPointerPointer), + { + for message in &self.messages { + callback(message.pointer()); } } - /// This method is unsafe because it does not explicitly synchronise access - /// to `self.external`, instead this is up to the caller. - pub unsafe fn mailbox_pointers(&self) -> WorkList { - let mut pointers = WorkList::new(); - - for pointer in self.internal.iter().chain(self.external.iter()) { - pointers.push(pointer.pointer()); - } - - pointers - } - - pub fn local_pointers(&self) -> WorkList { - let mut pointers = WorkList::new(); - - for pointer in &self.internal { - if !pointer.is_mailbox() { - pointers.push(pointer.pointer()); - } - } - - pointers - } - pub fn has_messages(&self) -> bool { - self.amount.load(Ordering::Acquire) > 0 + !self.messages.is_empty() } } #[cfg(test)] mod tests { use super::*; - use crate::config::Config; - use crate::immix::global_allocator::GlobalAllocator; use crate::object_pointer::ObjectPointer; - use crate::vm::test::setup; #[test] - fn test_send_from_self() { - let config = Config::new(); - let mut mailbox = Mailbox::new(GlobalAllocator::with_rc(), &config); - - assert_eq!(mailbox.amount.load(Ordering::Acquire), 0); + fn test_send_receive() { + let mut mailbox = Mailbox::new(); - mailbox.send_from_self(ObjectPointer::integer(5)); + mailbox.send(ObjectPointer::integer(4)); - assert_eq!(mailbox.amount.load(Ordering::Acquire), 1); + assert!(mailbox.receive() == Some(ObjectPointer::integer(4))) } #[test] - fn test_send_from_external() { - let config = Config::new(); - let mut mailbox = Mailbox::new(GlobalAllocator::with_rc(), &config); + fn test_each_pointer() { + let mut mailbox = Mailbox::new(); - assert_eq!(mailbox.amount.load(Ordering::Acquire), 0); + mailbox.send(ObjectPointer::new(0x1 as _)); - mailbox.send_from_external(ObjectPointer::integer(5)); + let mut pointers = Vec::new(); - assert_eq!(mailbox.amount.load(Ordering::Acquire), 1); - } + mailbox.each_pointer(|ptr| pointers.push(ptr)); - #[test] - fn test_receive_without_messages() { - let config = Config::new(); - let mut mailbox = Mailbox::new(GlobalAllocator::with_rc(), &config); - - let (must_copy, message) = mailbox.receive(); - - assert_eq!(must_copy, false); - assert!(message.is_none()); - } - - #[test] - fn test_receive_with_external_message() { - let config = Config::new(); - let mut mailbox = Mailbox::new(GlobalAllocator::with_rc(), &config); - - mailbox.send_from_external(ObjectPointer::integer(5)); - - let (must_copy, message) = mailbox.receive(); - - assert_eq!(must_copy, false); - assert_eq!(mailbox.amount.load(Ordering::Acquire), 0); - assert!(message == Some(ObjectPointer::integer(5))); - } - - #[test] - fn test_receive_with_external_message_with_copying() { - let config = Config::new(); - let mut mailbox = Mailbox::new(GlobalAllocator::with_rc(), &config); - let (_machine, _block, process) = setup(); - - let message = process.allocate_empty(); - - mailbox.send_from_external(message); - - let (must_copy, message) = mailbox.receive(); - - assert_eq!(must_copy, true); - assert_eq!(mailbox.amount.load(Ordering::Acquire), 0); - assert!(message.unwrap().get().value.is_none()); - } - - #[test] - fn test_receive_with_internal_message() { - let config = Config::new(); - let mut mailbox = Mailbox::new(GlobalAllocator::with_rc(), &config); + let pointer_pointer = pointers.pop(); - mailbox.send_from_self(ObjectPointer::integer(5)); + assert!(pointer_pointer.is_some()); - let (must_copy, message) = mailbox.receive(); + pointer_pointer.unwrap().get_mut().raw.raw = 0x4 as _; - assert_eq!(must_copy, false); - assert_eq!(mailbox.amount.load(Ordering::Acquire), 0); - assert!(message == Some(ObjectPointer::integer(5))); + assert!(mailbox.receive() == Some(ObjectPointer::new(0x4 as _))); } #[test] - fn test_has_messages() { - let config = Config::new(); - let mut mailbox = Mailbox::new(GlobalAllocator::with_rc(), &config); + fn test_has_messagess() { + let mut mailbox = Mailbox::new(); assert_eq!(mailbox.has_messages(), false); - mailbox.send_from_self(ObjectPointer::integer(5)); + mailbox.send(ObjectPointer::integer(5)); assert!(mailbox.has_messages()); } diff --git a/vm/src/object.rs b/vm/src/object.rs index e17de313c..a028efa4d 100644 --- a/vm/src/object.rs +++ b/vm/src/object.rs @@ -7,8 +7,9 @@ use fnv::FnvHashMap; use std::ops::Drop; use std::ptr; -use crate::gc::work_list::WorkList; -use crate::object_pointer::{ObjectPointer, RawObjectPointer}; +use crate::object_pointer::{ + ObjectPointer, ObjectPointerPointer, RawObjectPointer, +}; use crate::object_value::ObjectValue; use crate::tagged_pointer::TaggedPointer; @@ -50,6 +51,9 @@ pub const PENDING_FORWARD_BIT: usize = 0; /// The bit to set for objects that have been forwarded. pub const FORWARDED_BIT: usize = 1; +/// The bit to set for objects stored in a remembered set. +pub const REMEMBERED_BIT: usize = 2; + /// The mask to apply when installing a forwarding pointer. pub const FORWARDING_MASK: usize = 0x3; @@ -62,12 +66,16 @@ pub struct Object { /// on-demand and default to a NULL pointer. /// /// This pointer may be tagged to store extra information. The following - /// bits can be set: + /// lower bits can be set: + /// + /// * 000: this field contains a regular pointer. + /// * 001: this object is in the process of being forwarded. + /// * 010: this object has been forwarded, and this field is set to the + /// target `ObjectPointer`. + /// * 100: this object has been remembered in the remembered set. /// - /// * 00: this field contains a regular pointer. - /// * 01: this object is in the process of being forwarded. - /// * 10: this object has been forwarded, and this field is set to the - /// target object. + /// Multiple bits can be set as well. For example, `101` would mean the + /// object is remembered and being forwarded. pub attributes: TaggedPointer, /// A native Rust value (e.g. a String) that belongs to this object. @@ -226,35 +234,37 @@ impl Object { self.attributes = TaggedPointer::new(Box::into_raw(Box::new(attrs))); } - /// Pushes all pointers in this object into the given Vec. - pub fn push_pointers(&self, pointers: &mut WorkList) { + pub fn each_pointer(&self, mut callback: F) + where + F: FnMut(ObjectPointerPointer), + { if !self.prototype.is_null() { - pointers.push(self.prototype.pointer()); + callback(self.prototype.pointer()); } if let Some(map) = self.attributes_map() { // Attribute keys are interned strings, which don't need to be // marked. for (_, pointer) in map.iter() { - pointers.push(pointer.pointer()); + callback(pointer.pointer()); } } match self.value { ObjectValue::Array(ref array) => { for pointer in array.iter() { - pointers.push(pointer.pointer()); + callback(pointer.pointer()); } } ObjectValue::Block(ref block) => { if let Some(captures_from) = block.captures_from.as_ref() { - captures_from.push_pointers(pointers); + captures_from.each_pointer(|v| callback(v)); } - pointers.push(block.receiver.pointer()); + callback(block.receiver.pointer()); } ObjectValue::Binding(ref binding) => { - binding.push_pointers(pointers); + binding.each_pointer(|v| callback(v)); } _ => {} } @@ -265,19 +275,12 @@ impl Object { let mut new_obj = Object::with_prototype(self.value.take(), self.prototype); - if let Some(attributes) = self.take_attributes() { - new_obj.attributes = attributes; - } - - new_obj - } - - pub fn take_attributes(&mut self) -> Option> { - if !self.has_attributes() { - return None; - } + // When taking over the attributes we want to automatically inherit the + // "remembered" bit, but not the forwarding bits. + let attrs = (self.attributes.raw as usize & !FORWARDING_MASK) + as *mut AttributesMap; - let attrs = self.attributes.without_tags(); + new_obj.attributes = TaggedPointer::new(attrs); // When the object is being forwarded we don't want to lose this status // by just setting the attributes to NULL. Doing so could result in @@ -285,7 +288,7 @@ impl Object { self.attributes = TaggedPointer::with_bit(0x0 as _, PENDING_FORWARD_BIT); - Some(attrs) + new_obj } /// Tries to mark this object as pending a forward. @@ -320,6 +323,19 @@ impl Object { self.attributes.atomic_store(new_attrs); } + /// Marks this object as being remembered. + /// + /// This does not use atomic operations and thus should not be called + /// concurrently for the same pointer. + pub fn mark_as_remembered(&mut self) { + self.attributes.set_bit(REMEMBERED_BIT); + } + + /// Returns true if this object has been remembered in a remembered set. + pub fn is_remembered(&self) -> bool { + self.attributes.atomic_bit_is_set(REMEMBERED_BIT) + } + /// Returns true if this object is forwarded. pub fn is_forwarded(&self) -> bool { self.attributes.atomic_bit_is_set(FORWARDED_BIT) @@ -332,13 +348,21 @@ impl Object { /// Returns true if an attributes map has been allocated. pub fn has_attributes(&self) -> bool { - !self.attributes.is_null() && !self.is_forwarded() + if self.is_forwarded() { + return false; + } + + !self.attributes.untagged().is_null() } pub fn drop_attributes(&mut self) { - if let Some(attributes) = self.take_attributes() { - drop(unsafe { Box::from_raw(attributes.untagged()) }); + if !self.has_attributes() { + return; } + + drop(unsafe { Box::from_raw(self.attributes.untagged()) }); + + self.attributes = TaggedPointer::null(); } pub fn write_to(self, raw_pointer: RawObjectPointer) -> ObjectPointer { @@ -570,22 +594,22 @@ mod tests { } #[test] - fn test_object_push_pointers_without_pointers() { + fn test_object_each_pointer_without_pointers() { let obj = new_object(); - let mut pointers = WorkList::new(); + let mut pointers = Vec::new(); - obj.push_pointers(&mut pointers); + obj.each_pointer(|ptr| pointers.push(ptr)); - assert!(pointers.pop().is_none()); + assert!(pointers.is_empty()); } #[test] - fn test_object_push_pointers_with_attributes() { + fn test_object_each_pointer_with_attributes() { let mut obj = new_object(); - let mut pointers = WorkList::new(); + let mut pointers = Vec::new(); obj.add_attribute(fake_pointer(), fake_pointer()); - obj.push_pointers(&mut pointers); + obj.each_pointer(|ptr| pointers.push(ptr)); let pointer_pointer = pointers.pop().unwrap(); @@ -597,12 +621,12 @@ mod tests { } #[test] - fn test_object_push_pointers_with_array() { - let mut pointers = WorkList::new(); + fn test_object_each_pointer_with_array() { + let mut pointers = Vec::new(); let obj = Object::new(ObjectValue::Array(Box::new(vec![fake_pointer()]))); - obj.push_pointers(&mut pointers); + obj.each_pointer(|ptr| pointers.push(ptr)); let pointer_pointer = pointers.pop().unwrap(); @@ -614,7 +638,7 @@ mod tests { } #[test] - fn test_object_push_pointers_with_block() { + fn test_object_each_pointer_with_block() { let state = State::with_rc(Config::new(), &[]); let binding = Binding::with_rc(0, fake_pointer()); let code = CompiledCode::new( @@ -633,9 +657,9 @@ mod tests { ); let obj = Object::new(ObjectValue::Block(Box::new(block))); - let mut pointers = WorkList::new(); + let mut pointers = Vec::new(); - obj.push_pointers(&mut pointers); + obj.each_pointer(|ptr| pointers.push(ptr)); while let Some(pointer_pointer) = pointers.pop() { pointer_pointer.get_mut().raw.raw = 0x5 as _; @@ -650,15 +674,15 @@ mod tests { } #[test] - fn test_object_push_pointers_with_binding() { + fn test_object_each_pointer_with_binding() { let mut binding = Binding::with_rc(1, fake_pointer()); binding.set_local(0, fake_pointer()); let obj = Object::new(ObjectValue::Binding(binding.clone())); - let mut pointers = WorkList::new(); + let mut pointers = Vec::new(); - obj.push_pointers(&mut pointers); + obj.each_pointer(|ptr| pointers.push(ptr)); while let Some(pointer_pointer) = pointers.pop() { pointer_pointer.get_mut().raw.raw = 0x5 as _; @@ -669,12 +693,12 @@ mod tests { } #[test] - fn test_object_push_pointers_with_prototype() { + fn test_object_each_pointer_with_prototype() { let mut obj = Object::new(ObjectValue::None); - let mut pointers = WorkList::new(); + let mut pointers = Vec::new(); obj.set_prototype(fake_pointer()); - obj.push_pointers(&mut pointers); + obj.each_pointer(|ptr| pointers.push(ptr)); while let Some(pointer_pointer) = pointers.pop() { pointer_pointer.get_mut().raw.raw = 0x5 as _; @@ -700,21 +724,78 @@ mod tests { } #[test] - fn test_object_take_attributes() { + fn test_object_take_remembered_object() { + let mut obj = Object::new(ObjectValue::Float(10.0)); + + obj.mark_as_remembered(); + + let new_obj = obj.take(); + + assert_eq!(obj.is_remembered(), false); + assert!(new_obj.is_remembered()); + } + + #[test] + fn test_object_has_attributes() { let mut obj = Object::new(ObjectValue::Float(10.0)); let map = AttributesMap::default(); obj.set_attributes_map(map); - let attr_opt = obj.take_attributes(); + assert!(obj.has_attributes()); + } + + #[test] + fn test_object_has_attributes_forwarded() { + let mut obj = Object::new(ObjectValue::Float(10.0)); + + obj.attributes = TaggedPointer::new(0x1 as *mut _); + obj.attributes.set_bit(PENDING_FORWARD_BIT); + obj.attributes.set_bit(FORWARDED_BIT); - assert!(attr_opt.is_some()); + assert_eq!(obj.has_attributes(), false); + } + + #[test] + fn test_object_has_attributes_remembered() { + let mut obj = Object::new(ObjectValue::Float(10.0)); + let map = AttributesMap::default(); + + obj.set_attributes_map(map); + obj.mark_as_remembered(); + + assert!(obj.has_attributes()); + } + + #[test] + fn test_object_mark_as_remembered() { + let mut obj = Object::new(ObjectValue::Float(10.0)); + + assert_eq!(obj.is_remembered(), false); + + obj.mark_as_remembered(); + + assert!(obj.is_remembered()); + } + + #[test] + fn test_object_mark_as_forwarded_for_remembered_object() { + let mut obj = Object::new(ObjectValue::Float(10.0)); + + obj.mark_as_remembered(); + obj.mark_for_forward(); + + assert!(obj.is_remembered()); assert!(obj.attributes.bit_is_set(PENDING_FORWARD_BIT)); - assert!(obj.attributes.is_null()); + } - unsafe { - Box::from_raw(attr_opt.unwrap().untagged()); - } + #[test] + fn test_object_has_attributes_remembered_without_attributes() { + let mut obj = Object::new(ObjectValue::Float(10.0)); + + obj.mark_as_remembered(); + + assert_eq!(obj.has_attributes(), false); } #[test] @@ -725,8 +806,23 @@ mod tests { obj.forward_to(object_pointer_for(&target)); assert!(obj.is_forwarded()); - assert!(!obj.attributes.is_null()); - assert!(object_pointer_for(&obj).is_forwarded()); + assert!(obj.attributes.bit_is_set(PENDING_FORWARD_BIT)); + assert!(obj.attributes.bit_is_set(FORWARDED_BIT)); + } + + #[test] + fn test_object_forward_to_remembered_object() { + let mut obj = new_object(); + let target = new_object(); + + obj.mark_as_remembered(); + obj.forward_to(object_pointer_for(&target)); + + assert!(obj.is_forwarded()); + assert!(obj.attributes.bit_is_set(PENDING_FORWARD_BIT)); + assert!(obj.attributes.bit_is_set(FORWARDED_BIT)); + + assert_eq!(obj.is_remembered(), false); } #[test] diff --git a/vm/src/object_pointer.rs b/vm/src/object_pointer.rs index 431f31fa7..d9b36f5c3 100644 --- a/vm/src/object_pointer.rs +++ b/vm/src/object_pointer.rs @@ -12,6 +12,7 @@ use std::i16; use std::i32; use std::i64; use std::i8; +use std::ptr; use std::u16; use std::u32; use std::u64; @@ -80,9 +81,8 @@ pub struct ObjectPointer { /// The underlying tagged pointer. This pointer can have the following last /// two bits set: /// - /// 00: the pointer is a regular pointer - /// 01: the pointer is a tagged integer - /// 10: the pointer is a forwarding pointer + /// * 00: the pointer is a regular pointer + /// * 01: the pointer is a tagged integer pub raw: TaggedPointer, } @@ -106,7 +106,7 @@ pub const INTEGER_BIT: usize = 0; fn block_header_of<'a>( pointer: RawObjectPointer, ) -> &'a mut block::BlockHeader { - let addr = (pointer as isize & block::OBJECT_BITMAP_MASK) as usize; + let addr = (pointer as isize & block::OBJECT_BYTEMAP_MASK) as usize; unsafe { let ptr = addr as *mut block::BlockHeader; @@ -175,21 +175,29 @@ impl ObjectPointer { let block = self.block(); - if block.is_fragmented() { - if self.get_mut().mark_for_forward() { - ObjectStatus::Evacuate + // If an object resides on a fragmented block _and_ needs to be + // promoted, we can just promote it right away; instead of first + // evacuating it and _then_ promoting it. + // + // If we instead evacuate such an object it may end up surviving too + // many collections before being promoted. + if block.bucket().unwrap().age == YOUNG_MAX_AGE { + return if self.get_mut().mark_for_forward() { + ObjectStatus::Promote } else { ObjectStatus::PendingMove - } - } else if block.bucket().unwrap().promote { - if self.get_mut().mark_for_forward() { - ObjectStatus::Promote + }; + } + + if block.is_fragmented() { + return if self.get_mut().mark_for_forward() { + ObjectStatus::Evacuate } else { ObjectStatus::PendingMove - } - } else { - ObjectStatus::OK + }; } + + ObjectStatus::OK } /// Replaces the current pointer with a pointer to the forwarded object. @@ -267,8 +275,21 @@ impl ObjectPointer { let object_index = block.object_index_of_pointer(pointer); let line_index = block.line_index_of_pointer(pointer); - block.marked_objects_bitmap.set(object_index); - block.used_lines_bitmap.set(line_index); + block.marked_objects_bytemap.set(object_index); + block.used_lines_bytemap.set(line_index); + } + + /// Unmarks the current object. + /// + /// The line mark state is not changed. + pub fn unmark(&self) { + let pointer = self.raw.untagged(); + let header = block_header_of(pointer); + let block = header.block_mut(); + + let object_index = block.object_index_of_pointer(pointer); + + block.marked_objects_bytemap.unset(object_index); } /// Returns true if the current object is marked. @@ -288,7 +309,19 @@ impl ObjectPointer { let block = header.block_mut(); let index = block.object_index_of_pointer(pointer); - block.marked_objects_bitmap.is_set(index) + block.marked_objects_bytemap.is_set(index) + } + + /// Marks the object this pointer points to as being remembered in a + /// remembered set. + pub fn mark_as_remembered(&self) { + self.get_mut().mark_as_remembered(); + } + + /// Returns `true` if the object this pointer points to has been remembered + /// in a remembered set. + pub fn is_remembered(&self) -> bool { + self.get().is_remembered() } /// Returns a mutable reference to the block this pointer belongs to. @@ -315,7 +348,14 @@ impl ObjectPointer { return; } - drop(self.get_mut().take()); + unsafe { + ptr::drop_in_place(self.raw.raw); + + // We zero out the memory so future finalize() calls for the same + // object (before other allocations take place) don't try to free + // the memory again. + ptr::write_bytes(self.raw.raw, 0, 1); + } } /// Adds an attribute to the object this pointer points to. @@ -763,8 +803,24 @@ mod tests { fn test_object_pointer_status_promote() { let mut allocator = local_allocator(); let mut pointer = allocator.allocate_empty(); + let bucket = pointer.block_mut().bucket_mut().unwrap(); - pointer.block_mut().bucket_mut().unwrap().promote = true; + bucket.increment_age(); + bucket.increment_age(); + + assert_eq!(pointer.status(), ObjectStatus::Promote); + assert_eq!(pointer.status(), ObjectStatus::PendingMove); + } + + #[test] + fn test_object_pointer_status_promote_from_fragmented_block() { + let mut allocator = local_allocator(); + let mut pointer = allocator.allocate_empty(); + let bucket = pointer.block_mut().bucket_mut().unwrap(); + + bucket.increment_age(); + bucket.increment_age(); + pointer.block_mut().set_fragmented(); assert_eq!(pointer.status(), ObjectStatus::Promote); assert_eq!(pointer.status(), ObjectStatus::PendingMove); @@ -982,8 +1038,33 @@ mod tests { pointer.mark(); - assert!(pointer.block().marked_objects_bitmap.is_set(4)); - assert!(pointer.block().used_lines_bitmap.is_set(1)); + assert!(pointer.block().marked_objects_bytemap.is_set(4)); + assert!(pointer.block().used_lines_bytemap.is_set(1)); + } + + #[test] + fn test_object_pointer_unmark() { + let mut allocator = local_allocator(); + let pointer = allocator.allocate_empty(); + + pointer.mark(); + pointer.unmark(); + + assert_eq!(pointer.block().marked_objects_bytemap.is_set(4), false); + + assert!(pointer.block().used_lines_bytemap.is_set(1)); + } + + #[test] + fn test_object_pointer_mark_as_remembered() { + let mut allocator = local_allocator(); + let pointer = allocator.allocate_empty(); + + assert_eq!(pointer.is_remembered(), false); + + pointer.mark_as_remembered(); + + assert!(pointer.is_remembered()); } #[test] @@ -1162,6 +1243,22 @@ mod tests { assert_eq!(ObjectPointer::integer(5).is_finalizable(), false); } + #[test] + fn test_finalize() { + let mut allocator = local_allocator(); + let ptr1 = + allocator.allocate_without_prototype(ObjectValue::Integer(10)); + + let ptr2 = allocator.allocate_empty(); + + ptr1.get_mut().add_attribute(ptr2, ptr2); + ptr1.finalize(); + + let obj1 = ptr1.get(); + + assert!(obj1.attributes.is_null()); + } + #[test] fn test_usize_value() { let ptr = ObjectPointer::integer(5); diff --git a/vm/src/process.rs b/vm/src/process.rs index 2ca424689..b62771f7e 100644 --- a/vm/src/process.rs +++ b/vm/src/process.rs @@ -4,20 +4,20 @@ use crate::block::Block; use crate::compiled_code::CompiledCodePointer; use crate::config::Config; use crate::execution_context::ExecutionContext; -use crate::gc::work_list::WorkList; use crate::global_scope::GlobalScopePointer; use crate::immix::block_list::BlockList; use crate::immix::copy_object::CopyObject; use crate::immix::global_allocator::RcGlobalAllocator; use crate::immix::local_allocator::LocalAllocator; use crate::mailbox::Mailbox; -use crate::object_pointer::ObjectPointer; +use crate::object_pointer::{ObjectPointer, ObjectPointerPointer}; use crate::object_value; use crate::scheduler::timeouts::Timeout; use crate::tagged_pointer::{self, TaggedPointer}; -use crate::vm::state::RcState; +use crate::vm::state::State; use num_bigint::BigInt; use num_traits::FromPrimitive; +use parking_lot::Mutex; use std::cell::UnsafeCell; use std::i64; use std::mem; @@ -65,13 +65,11 @@ pub struct LocalData { /// The process-local memory allocator. pub allocator: LocalAllocator, - /// The mailbox for sending/receiving messages. + /// The mailbox of this process. /// - /// The Mailbox is stored in LocalData as a Mailbox uses internal locking - /// while still allowing a receiver to mutate it without a lock. This means - /// some operations need a &mut self, which won't be possible if a Mailbox - /// is stored directly in a Process. - pub mailbox: Mailbox, + /// We store this in LocalData so that we can borrow fields from LocalData + /// while also borrowing the mailbox. + pub mailbox: Mutex, // A block to execute in the event of a panic. pub panic_handler: ObjectPointer, @@ -132,11 +130,11 @@ impl Process { let local_data = LocalData { allocator: LocalAllocator::new(global_allocator.clone(), config), context: Box::new(context), - mailbox: Mailbox::new(global_allocator, config), panic_handler: ObjectPointer::null(), blocking: false, main: false, thread_id: None, + mailbox: Mutex::new(Mailbox::new()), }; ArcWithoutWeak::new(Process { @@ -385,37 +383,25 @@ impl Process { local_data.allocator.allocate_without_prototype(value) } - pub fn send_message_from_external_process(&self, message: ObjectPointer) { - self.local_data_mut().mailbox.send_from_external(message); + pub fn send_message_from_external_process( + &self, + message_to_copy: ObjectPointer, + ) { + let local_data = self.local_data_mut(); + + // The lock must be acquired first, as the receiving process may be + // garbage collected at this time. + let mut mailbox = local_data.mailbox.lock(); + + mailbox.send(local_data.allocator.copy_object(message_to_copy)); } pub fn send_message_from_self(&self, message: ObjectPointer) { - self.local_data_mut().mailbox.send_from_self(message); + self.local_data_mut().mailbox.lock().send(message); } - /// Returns a message from the mailbox. pub fn receive_message(&self) -> Option { - let local_data = self.local_data_mut(); - let (should_copy, pointer_opt) = local_data.mailbox.receive(); - - if let Some(mailbox_pointer) = pointer_opt { - let pointer = if should_copy { - // When another process sends us a message, the message will be - // copied onto the mailbox heap. We can't directly use such a - // pointer, as it might be garbage collected when it no longer - // resides in the mailbox (e.g. after a receive). - // - // To work around this, we move the data from the mailbox heap - // into the process' local heap. - local_data.allocator.move_object(mailbox_pointer) - } else { - mailbox_pointer - }; - - Some(pointer) - } else { - None - } + self.local_data_mut().mailbox.lock().receive() } pub fn binding(&self) -> RcBinding { @@ -440,7 +426,7 @@ impl Process { } pub fn has_messages(&self) -> bool { - self.local_data().mailbox.has_messages() + self.local_data().mailbox.lock().has_messages() } pub fn should_collect_young_generation(&self) -> bool { @@ -451,18 +437,10 @@ impl Process { self.local_data().allocator.should_collect_mature() } - pub fn should_collect_mailbox(&self) -> bool { - self.local_data().mailbox.allocator.should_collect() - } - pub fn contexts(&self) -> Vec<&ExecutionContext> { self.context().contexts().collect() } - pub fn has_remembered_objects(&self) -> bool { - self.local_data().allocator.has_remembered_objects() - } - /// Write barrier for tracking cross generation writes. /// /// This barrier is based on the Steele write barrier and tracks the object @@ -483,7 +461,7 @@ impl Process { .prepare_for_collection(mature) } - pub fn reclaim_blocks(&self, state: &RcState, mature: bool) { + pub fn reclaim_blocks(&self, state: &State, mature: bool) { self.local_data_mut() .allocator .reclaim_blocks(state, mature); @@ -498,12 +476,11 @@ impl Process { } blocks.append(&mut local_data.allocator.mature_generation.blocks); - blocks.append(&mut local_data.mailbox.allocator.bucket.blocks); blocks } - pub fn reclaim_and_finalize(&self, state: &RcState) { + pub fn reclaim_and_finalize(&self, state: &State) { let mut blocks = self.reclaim_all_blocks(); for block in blocks.iter_mut() { @@ -514,23 +491,6 @@ impl Process { state.global_allocator.add_blocks(&mut blocks); } - pub fn update_collection_statistics(&self, config: &Config, mature: bool) { - let local_data = self.local_data_mut(); - - local_data - .allocator - .update_collection_statistics(config, mature); - } - - pub fn update_mailbox_collection_statistics(&self, config: &Config) { - let local_data = self.local_data_mut(); - - local_data - .mailbox - .allocator - .update_collection_statistics(config); - } - pub fn panic_handler(&self) -> Option<&ObjectPointer> { let local_data = self.local_data(); @@ -545,14 +505,30 @@ impl Process { self.local_data_mut().panic_handler = handler; } - pub fn global_pointers_to_trace(&self) -> WorkList { - let mut pointers = WorkList::new(); - + pub fn each_global_pointer(&self, mut callback: F) + where + F: FnMut(ObjectPointerPointer), + { if let Some(handler) = self.panic_handler() { - pointers.push(handler.pointer()); + callback(handler.pointer()); } + } + + pub fn each_remembered_pointer(&self, callback: F) + where + F: FnMut(ObjectPointerPointer), + { + self.local_data_mut() + .allocator + .each_remembered_pointer(callback); + } + + pub fn prune_remembered_set(&self) { + self.local_data_mut().allocator.prune_remembered_objects(); + } - pointers + pub fn remember_object(&self, pointer: ObjectPointer) { + self.local_data_mut().allocator.remember_object(pointer); } pub fn waiting_for_message(&self) { @@ -610,7 +586,7 @@ mod tests { } #[test] - fn test_update_collection_statistics_without_mature() { + fn test_reclaim_blocks_without_mature() { let (machine, _block, process) = setup(); { @@ -620,7 +596,7 @@ mod tests { local_data.allocator.mature_config.increment_allocations(); } - process.update_collection_statistics(&machine.state.config, false); + process.reclaim_blocks(&machine.state, false); let local_data = process.local_data(); @@ -629,7 +605,7 @@ mod tests { } #[test] - fn test_update_collection_statistics_with_mature() { + fn test_reclaim_blocks_with_mature() { let (machine, _block, process) = setup(); { @@ -639,7 +615,7 @@ mod tests { local_data.allocator.mature_config.increment_allocations(); } - process.update_collection_statistics(&machine.state.config, true); + process.reclaim_blocks(&machine.state, true); let local_data = process.local_data(); @@ -647,29 +623,10 @@ mod tests { assert_eq!(local_data.allocator.mature_config.block_allocations, 0); } - #[test] - fn test_update_mailbox_collection_statistics() { - let (machine, _block, process) = setup(); - - process - .local_data_mut() - .mailbox - .allocator - .config - .increment_allocations(); - - process.update_mailbox_collection_statistics(&machine.state.config); - - let local_data = process.local_data(); - - assert_eq!(local_data.mailbox.allocator.config.block_allocations, 0); - } - #[test] fn test_receive_message() { let (machine, _block, process) = setup(); - // Simulate sending a message from an external process. let input_message = process .allocate(object_value::integer(14), process.allocate_empty()); @@ -677,10 +634,7 @@ mod tests { input_message.add_attribute(&process, attr, attr); - process - .local_data_mut() - .mailbox - .send_from_external(input_message); + process.send_message_from_external_process(input_message); let received = process.receive_message().unwrap(); @@ -689,6 +643,7 @@ mod tests { assert!(received.get().prototype().is_some()); assert!(received.get().attributes_map().is_some()); assert!(received.is_finalizable()); + assert!(received.raw.raw != input_message.raw.raw); } #[test] @@ -761,11 +716,9 @@ mod tests { #[test] fn test_process_type_size() { - let size = mem::size_of::(); - // This test is put in place to ensure the type size doesn't change // unintentionally. - assert!(size <= 448); + assert_eq!(mem::size_of::(), 328); } #[test] @@ -789,4 +742,17 @@ mod tests { assert!(process.identifier() > 0); } + + #[test] + fn test_each_global_pointer() { + let (_machine, _block, process) = setup(); + + process.set_panic_handler(ObjectPointer::integer(5)); + + let mut pointers = Vec::new(); + + process.each_global_pointer(|ptr| pointers.push(ptr)); + + assert_eq!(pointers.len(), 1); + } } diff --git a/vm/src/register.rs b/vm/src/register.rs index 6001d5d54..af7c553a9 100644 --- a/vm/src/register.rs +++ b/vm/src/register.rs @@ -3,8 +3,7 @@ //! Registers can be set in any particular order. However, reading from a //! register that is not set can lead to bogus data being returned. use crate::chunk::Chunk; -use crate::gc::work_list::WorkList; -use crate::object_pointer::ObjectPointer; +use crate::object_pointer::{ObjectPointer, ObjectPointerPointer}; /// Structure used for storing temporary values of a scope. pub struct Register { @@ -29,13 +28,15 @@ impl Register { self.values[register] } - /// Pushes all pointers in this register into the supplied vector. - pub fn push_pointers(&self, pointers: &mut WorkList) { + pub fn each_pointer(&self, mut callback: F) + where + F: FnMut(ObjectPointerPointer), + { for index in 0..self.values.len() { let pointer = &self.values[index]; if !pointer.is_null() { - pointers.push(pointer.pointer()); + callback(pointer.pointer()); } } } @@ -59,7 +60,7 @@ mod tests { } #[test] - fn test_push_pointers() { + fn test_each_pointer() { let mut register = Register::new(2); let pointer1 = ObjectPointer::new(0x1 as RawObjectPointer); @@ -68,9 +69,9 @@ mod tests { register.set(0, pointer1); register.set(1, pointer2); - let mut pointers = WorkList::new(); + let mut pointers = Vec::new(); - register.push_pointers(&mut pointers); + register.each_pointer(|ptr| pointers.push(ptr)); // The returned pointers should allow updating of what's stored in the // register without copying anything. diff --git a/vm/src/scheduler/generic_pool.rs b/vm/src/scheduler/generic_pool.rs deleted file mode 100644 index 6039dee0e..000000000 --- a/vm/src/scheduler/generic_pool.rs +++ /dev/null @@ -1,90 +0,0 @@ -//! Thread pool for executing generic tasks. -use crate::arc_without_weak::ArcWithoutWeak; -use crate::scheduler::generic_worker::GenericWorker; -use crate::scheduler::pool::Pool; -use crate::scheduler::pool_state::PoolState; -use crate::scheduler::queue::RcQueue; -use crate::scheduler::worker::Worker; -use std::thread; - -/// A pool of threads for running generic tasks. -pub struct GenericPool { - pub state: ArcWithoutWeak>, - - /// The base name of every thread in this pool. - name: String, -} - -impl GenericPool { - pub fn new(name: String, threads: usize) -> Self { - assert!( - threads > 0, - "A GenericPool requires at least a single thread" - ); - - Self { - name, - state: ArcWithoutWeak::new(PoolState::new(threads)), - } - } -} - -impl Pool> for GenericPool { - fn state(&self) -> &ArcWithoutWeak> { - &self.state - } - - fn spawn_thread( - &self, - id: usize, - queue: RcQueue, - callback: ArcWithoutWeak, - ) -> thread::JoinHandle<()> - where - F: Fn(&mut GenericWorker, T) + Send + 'static, - { - let state = self.state.clone(); - - thread::Builder::new() - .name(format!("{} {}", self.name, id)) - .spawn(move || { - GenericWorker::new(queue, state).run(&*callback); - }) - .unwrap() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use parking_lot::Mutex; - - #[test] - #[should_panic] - fn test_new_with_zero_threads() { - GenericPool::<()>::new("test".to_string(), 0); - } - - #[test] - fn test_spawn_thread() { - let pool = GenericPool::new("test".to_string(), 1); - let number = ArcWithoutWeak::new(Mutex::new(0)); - let number_copy = number.clone(); - - let callback = ArcWithoutWeak::new( - move |worker: &mut GenericWorker, number| { - *number_copy.lock() = number; - worker.state().terminate(); - }, - ); - - let thread = - pool.spawn_thread(0, pool.state.queues[0].clone(), callback); - - pool.schedule(10); - - thread.join().unwrap(); - - assert_eq!(*number.lock(), 10); - } -} diff --git a/vm/src/scheduler/generic_worker.rs b/vm/src/scheduler/generic_worker.rs deleted file mode 100644 index e9cbe8ba8..000000000 --- a/vm/src/scheduler/generic_worker.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! Workers for executing generic tasks. -use crate::arc_without_weak::ArcWithoutWeak; -use crate::scheduler::pool_state::PoolState; -use crate::scheduler::queue::RcQueue; -use crate::scheduler::worker::Worker; - -/// A worker that can be used for executing a wide variety of tasks, instead of -/// being limited to only executing lightweight processes. -/// -/// Generic workers do not support task pinning, or scheduling tasks directly -/// onto a specific worker. -pub struct GenericWorker { - /// The queue owned by this worker. - queue: RcQueue, - - /// The state of the pool this worker belongs to. - state: ArcWithoutWeak>, -} - -impl GenericWorker { - pub fn new(queue: RcQueue, state: ArcWithoutWeak>) -> Self { - GenericWorker { queue, state } - } -} - -impl Worker for GenericWorker { - fn run(&mut self, callback: F) - where - F: Fn(&mut Self, T), - { - while self.state.is_alive() { - if self.process_local_jobs(&callback) { - continue; - } - - if self.steal_from_other_queue() { - continue; - } - - if self.steal_from_global_queue() { - continue; - } - - self.state.park_while(|| !self.state.has_global_jobs()); - } - } - - fn state(&self) -> &PoolState { - &self.state - } - - fn queue(&self) -> &RcQueue { - &self.queue - } -} - -#[cfg(test)] -mod tests { - use super::*; - use parking_lot::Mutex; - use std::collections::HashSet; - - fn numbers() -> ( - ArcWithoutWeak>>, - ArcWithoutWeak>>, - ) { - let orig = ArcWithoutWeak::new(Mutex::new(HashSet::new())); - let copy = orig.clone(); - - (orig, copy) - } - - fn worker() -> GenericWorker { - let state = ArcWithoutWeak::new(PoolState::new(2)); - - GenericWorker::new(state.queues[0].clone(), state) - } - - #[test] - fn test_run_global_jobs() { - let (numbers, numbers_copy) = numbers(); - let mut worker = worker(); - - worker.state.push_global(10); - - worker.run(move |worker, number| { - numbers_copy.lock().insert(number); - worker.state.terminate(); - }); - - assert!(numbers.lock().contains(&10)); - assert_eq!(worker.state.queues[1].has_local_jobs(), false); - } - - #[test] - fn test_run_steal_then_terminate() { - let (numbers, numbers_copy) = numbers(); - let mut worker = worker(); - - worker.state.queues[1].push_internal(10); - - worker.run(move |worker, number| { - numbers_copy.lock().insert(number); - worker.state.terminate(); - }); - - assert!(numbers.lock().contains(&10)); - assert_eq!(worker.state.queues[1].has_local_jobs(), false); - } - - #[test] - fn test_run_steal_then_work() { - let (numbers, numbers_copy) = numbers(); - let mut worker = worker(); - - worker.state.queues[1].push_internal(10); - - // Here the order of work is: - // - // 1. Steal from other queue - // 2. Go back to processing our own queue - // 3. Terminate - worker.run(move |worker, number| { - numbers_copy.lock().insert(number); - - worker.queue.push_internal(20); - - if number == 20 { - worker.state.terminate(); - } - }); - - assert!(numbers.lock().contains(&10)); - assert!(numbers.lock().contains(&20)); - assert_eq!(worker.state.queues[1].has_local_jobs(), false); - } - - #[test] - fn test_run_work_then_terminate_steal_loop() { - let (numbers, numbers_copy) = numbers(); - let mut worker = worker(); - - worker.state.queues[0].push_internal(10); - worker.state.queues[1].push_internal(20); - - worker.run(move |worker, number| { - numbers_copy.lock().insert(number); - worker.state.terminate(); - }); - - assert_eq!(numbers.lock().contains(&10), true); - assert_eq!(numbers.lock().contains(&20), false); - - assert!(worker.state.queues[1].has_local_jobs()); - } -} diff --git a/vm/src/scheduler/mod.rs b/vm/src/scheduler/mod.rs index ae14d0945..414e3abda 100644 --- a/vm/src/scheduler/mod.rs +++ b/vm/src/scheduler/mod.rs @@ -1,9 +1,6 @@ //! Task scheduling and execution using work stealing. -pub mod generic_pool; -pub mod generic_worker; pub mod join_list; pub mod park_group; -pub mod pool; pub mod pool_state; pub mod process_pool; pub mod process_scheduler; diff --git a/vm/src/scheduler/park_group.rs b/vm/src/scheduler/park_group.rs index 349e7fd1d..58f623a7e 100644 --- a/vm/src/scheduler/park_group.rs +++ b/vm/src/scheduler/park_group.rs @@ -46,6 +46,12 @@ impl ParkGroup { lock_and_notify!(self, notify_one); } + pub fn park(&self) { + let mut lock = self.mutex.lock(); + + self.cvar.wait(&mut lock); + } + /// Parks the current thread as long as the given condition is true. pub fn park_while(&self, condition: F) where diff --git a/vm/src/scheduler/pool.rs b/vm/src/scheduler/pool.rs deleted file mode 100644 index 20eb1e8b9..000000000 --- a/vm/src/scheduler/pool.rs +++ /dev/null @@ -1,64 +0,0 @@ -use crate::arc_without_weak::ArcWithoutWeak; -use crate::scheduler::join_list::JoinList; -use crate::scheduler::pool_state::PoolState; -use crate::scheduler::queue::RcQueue; -use crate::scheduler::worker::Worker; -use std::thread; - -pub trait Pool> { - fn state(&self) -> &ArcWithoutWeak>; - - /// Spawns a single OS thread that is to consume the given queue. - fn spawn_thread( - &self, - id: usize, - queue: RcQueue, - callback: ArcWithoutWeak, - ) -> thread::JoinHandle<()> - where - F: Fn(&mut W, T) + Send + 'static; - - /// Schedules a job onto the global queue. - fn schedule(&self, job: T) { - self.state().push_global(job); - } - - /// Informs this pool it should terminate as soon as possible. - fn terminate(&self) { - self.state().terminate(); - } - - /// Starts the pool, without blocking the calling thread. - fn start(&self, callback: F) -> JoinList<()> - where - F: Fn(&mut W, T) + Send + 'static, - { - let rc_callback = ArcWithoutWeak::new(callback); - - self.spawn_threads_for_range(0, &rc_callback) - } - - /// Spawns OS threads for a range of queues, starting at the given position. - fn spawn_threads_for_range( - &self, - start_at: usize, - callback: &ArcWithoutWeak, - ) -> JoinList<()> - where - F: Fn(&mut W, T) + Send + 'static, - { - let handles = self.state().queues[start_at..] - .iter() - .enumerate() - .map(|(index, queue)| { - // When using enumerate() with a range start > 0, the first - // index is still 0. - let worker_id = start_at + index; - - self.spawn_thread(worker_id, queue.clone(), callback.clone()) - }) - .collect(); - - JoinList::new(handles) - } -} diff --git a/vm/src/scheduler/pool_state.rs b/vm/src/scheduler/pool_state.rs index d3ca4e985..5b7c1d249 100644 --- a/vm/src/scheduler/pool_state.rs +++ b/vm/src/scheduler/pool_state.rs @@ -85,6 +85,10 @@ impl PoolState { pub fn terminate(&self) { self.alive.store(false, Ordering::Release); + self.notify_all(); + } + + pub fn notify_all(&self) { self.park_group.notify_all(); } @@ -107,8 +111,14 @@ impl PoolState { mod tests { use super::*; use crate::arc_without_weak::ArcWithoutWeak; + use std::mem; use std::thread; + #[test] + fn test_memory_size() { + assert_eq!(mem::size_of::>(), 384); + } + #[test] fn test_new() { let state: PoolState<()> = PoolState::new(4); diff --git a/vm/src/scheduler/process_pool.rs b/vm/src/scheduler/process_pool.rs index ca871c2a8..d6df3a42b 100644 --- a/vm/src/scheduler/process_pool.rs +++ b/vm/src/scheduler/process_pool.rs @@ -2,11 +2,11 @@ use crate::arc_without_weak::ArcWithoutWeak; use crate::process::RcProcess; use crate::scheduler::join_list::JoinList; -use crate::scheduler::pool::Pool; use crate::scheduler::pool_state::PoolState; use crate::scheduler::process_worker::ProcessWorker; use crate::scheduler::queue::RcQueue; use crate::scheduler::worker::Worker; +use crate::vm::machine::Machine; use std::thread; /// A pool of threads for running lightweight processes. @@ -34,50 +34,73 @@ impl ProcessPool { } } + /// Schedules a job onto a specific queue. + pub fn schedule_onto_queue(&self, queue: usize, job: RcProcess) { + self.state.schedule_onto_queue(queue, job); + } + + /// Schedules a job onto the global queue. + pub fn schedule(&self, job: RcProcess) { + self.state.push_global(job); + } + + /// Informs this pool it should terminate as soon as possible. + pub fn terminate(&self) { + self.state.terminate(); + } + /// Starts the pool, blocking the current thread until the pool is /// terminated. /// /// The current thread will be used to perform jobs scheduled onto the first /// queue. - pub fn start_main(&self, callback: F) -> JoinList<()> - where - F: Fn(&mut ProcessWorker, RcProcess) + Send + 'static, - { - let rc_callback = ArcWithoutWeak::new(callback); - let join_list = self.spawn_threads_for_range(1, &rc_callback); + pub fn start_main(&self, machine: Machine) -> JoinList<()> { + let join_list = self.spawn_threads_for_range(1, machine.clone()); + let queue = self.state.queues[0].clone(); - ProcessWorker::new(0, self.state.queues[0].clone(), self.state.clone()) - .run(&*rc_callback); + ProcessWorker::new(0, queue, self.state.clone(), machine).run(); join_list } - /// Schedules a job onto a specific queue. - pub fn schedule_onto_queue(&self, queue: usize, job: RcProcess) { - self.state.schedule_onto_queue(queue, job); + /// Starts the pool, without blocking the calling thread. + pub fn start(&self, machine: Machine) -> JoinList<()> { + self.spawn_threads_for_range(0, machine) } -} -impl Pool for ProcessPool { - fn state(&self) -> &ArcWithoutWeak> { - &self.state + /// Spawns OS threads for a range of queues, starting at the given position. + fn spawn_threads_for_range( + &self, + start_at: usize, + machine: Machine, + ) -> JoinList<()> { + let mut handles = Vec::new(); + + for index in start_at..self.state.queues.len() { + let handle = self.spawn_thread( + index, + machine.clone(), + self.state.queues[index].clone(), + ); + + handles.push(handle); + } + + JoinList::new(handles) } - fn spawn_thread( + fn spawn_thread( &self, id: usize, + machine: Machine, queue: RcQueue, - callback: ArcWithoutWeak, - ) -> thread::JoinHandle<()> - where - F: Fn(&mut ProcessWorker, RcProcess) + Send + 'static, - { + ) -> thread::JoinHandle<()> { let state = self.state.clone(); thread::Builder::new() .name(format!("{} {}", self.name, id)) .spawn(move || { - ProcessWorker::new(id, queue, state).run(&*callback); + ProcessWorker::new(id, queue, state, machine).run(); }) .unwrap() } @@ -87,7 +110,6 @@ impl Pool for ProcessPool { mod tests { use super::*; use crate::vm::test::setup; - use parking_lot::Mutex; #[test] #[should_panic] @@ -97,27 +119,22 @@ mod tests { #[test] fn test_start_main() { - let pool = ProcessPool::new("test".to_string(), 1); - let (_machine, _block, process) = setup(); - let pid = ArcWithoutWeak::new(Mutex::new(10)); - let pid_copy = pid.clone(); + let (machine, _block, process) = setup(); + let pool = &machine.state.scheduler.primary_pool; pool.schedule(process.clone()); - let threads = pool.start_main(move |worker, process| { - *pid_copy.lock() = process.identifier(); - worker.state().terminate(); - }); + let threads = pool.start_main(machine.clone()); threads.join().unwrap(); - assert_eq!(*pid.lock(), process.identifier()); + assert_eq!(pool.state.is_alive(), false); } #[test] fn test_schedule_onto_queue() { - let pool = ProcessPool::new("test".to_string(), 1); - let (_machine, _block, process) = setup(); + let (machine, _block, process) = setup(); + let pool = &machine.state.scheduler.primary_pool; pool.schedule_onto_queue(0, process); @@ -126,25 +143,16 @@ mod tests { #[test] fn test_spawn_thread() { - let pool = ProcessPool::new("test".to_string(), 1); - let (_machine, _block, process) = setup(); - let pid = ArcWithoutWeak::new(Mutex::new(10)); - let pid_copy = pid.clone(); - - let callback = ArcWithoutWeak::new( - move |worker: &mut ProcessWorker, process: RcProcess| { - *pid_copy.lock() = process.identifier(); - worker.state().terminate(); - }, - ); + let (machine, _block, process) = setup(); + let pool = &machine.state.scheduler.primary_pool; let thread = - pool.spawn_thread(0, pool.state.queues[0].clone(), callback); + pool.spawn_thread(0, machine.clone(), pool.state.queues[0].clone()); pool.schedule(process.clone()); thread.join().unwrap(); - assert_eq!(*pid.lock(), process.identifier()); + assert_eq!(pool.state.has_global_jobs(), false); } } diff --git a/vm/src/scheduler/process_scheduler.rs b/vm/src/scheduler/process_scheduler.rs index 19550348c..05f3dd0ba 100644 --- a/vm/src/scheduler/process_scheduler.rs +++ b/vm/src/scheduler/process_scheduler.rs @@ -1,6 +1,5 @@ //! Scheduling and execution of lightweight Inko processes. use crate::process::RcProcess; -use crate::scheduler::pool::Pool; use crate::scheduler::process_pool::ProcessPool; /// The ID of the queue that is processed by the main thread. diff --git a/vm/src/scheduler/process_worker.rs b/vm/src/scheduler/process_worker.rs index aa6e26bdf..169fea768 100644 --- a/vm/src/scheduler/process_worker.rs +++ b/vm/src/scheduler/process_worker.rs @@ -4,12 +4,14 @@ use crate::process::RcProcess; use crate::scheduler::pool_state::PoolState; use crate::scheduler::queue::RcQueue; use crate::scheduler::worker::Worker; +use crate::vm::machine::Machine; use num_bigint::BigInt; use num_bigint::RandBigInt; use rand::distributions::uniform::{SampleBorrow, SampleUniform}; use rand::distributions::{Distribution, Standard}; use rand::rngs::ThreadRng; use rand::{thread_rng, Rng}; +use std::cell::UnsafeCell; /// The state that a worker is in. #[derive(Eq, PartialEq, Debug)] @@ -43,6 +45,9 @@ pub struct ProcessWorker { /// The state of the pool this worker belongs to. state: ArcWithoutWeak>, + /// The Machine to use for running code. + machine: UnsafeCell, + /// The mode this worker is in. mode: Mode, } @@ -53,6 +58,7 @@ impl ProcessWorker { id: usize, queue: RcQueue, state: ArcWithoutWeak>, + machine: Machine, ) -> Self { ProcessWorker { id, @@ -61,6 +67,7 @@ impl ProcessWorker { queue, state, mode: Mode::Normal, + machine: UnsafeCell::new(machine), } } @@ -130,11 +137,8 @@ impl ProcessWorker { } /// Performs a single iteration of the normal work loop. - fn normal_iteration(&mut self, callback: &F) - where - F: Fn(&mut Self, RcProcess), - { - if self.process_local_jobs(callback) { + fn normal_iteration(&mut self) { + if self.process_local_jobs() { return; } @@ -156,11 +160,8 @@ impl ProcessWorker { } /// Runs a single iteration of an exclusive work loop. - fn exclusive_iteration(&mut self, callback: &F) - where - F: Fn(&mut Self, RcProcess), - { - if self.process_local_jobs(callback) { + fn exclusive_iteration(&mut self) { + if self.process_local_jobs() { return; } @@ -168,7 +169,7 @@ impl ProcessWorker { // starving the current worker of pinned jobs. Since only one job can be // pinned to a worker, we don't need a loop here. if let Some(job) = self.queue.pop_external_job() { - callback(self, job); + self.process_job(job); return; } @@ -185,21 +186,29 @@ impl Worker for ProcessWorker { &self.queue } - /// Starts the worker, blocking the calling thread. - /// - /// This method will not return until our queue or any other queues this - /// worker has access to are terminated. - fn run(&mut self, callback: F) - where - F: Fn(&mut Self, RcProcess), - { + fn run(&mut self) { while self.state.is_alive() { match self.mode { - Mode::Normal => self.normal_iteration(&callback), - Mode::Exclusive => self.exclusive_iteration(&callback), + Mode::Normal => self.normal_iteration(), + Mode::Exclusive => self.exclusive_iteration(), }; } } + + fn process_job(&mut self, job: RcProcess) { + // When using a Machine we need both an immutable reference to it (using + // `self.machine`), and a mutable reference to pass as an argument. + // Rust does not allow this, even though in this case it's perfectly + // safe. + // + // To work around this we use UnsafeCell. We could use RefCell, but + // since we know exactly how this code is used (it's only the lines + // below that depend on this) the runtime reference counting is not + // needed. + let machine = unsafe { &*self.machine.get() }; + + machine.run_with_error_handling(self, &job); + } } #[cfg(test)] @@ -207,149 +216,99 @@ mod tests { use super::*; use crate::vm::process; use crate::vm::test::setup; - use parking_lot::Mutex; - use std::collections::HashSet; - fn pids_set() -> ( - ArcWithoutWeak>>, - ArcWithoutWeak>>, - ) { - let orig = ArcWithoutWeak::new(Mutex::new(HashSet::new())); - let copy = orig.clone(); + fn worker(machine: Machine) -> ProcessWorker { + let pool_state = machine.state.scheduler.primary_pool.state.clone(); + let queue = pool_state.queues[0].clone(); - (orig, copy) - } - - fn worker() -> ProcessWorker { - let state = ArcWithoutWeak::new(PoolState::new(2)); - - ProcessWorker::new(0, state.queues[0].clone(), state) + ProcessWorker::new(0, queue, pool_state, machine) } #[test] fn test_run_global_jobs() { - let (pids, pids_copy) = pids_set(); - let (_machine, _block, process) = setup(); - let mut worker = worker(); + let (machine, _block, process) = setup(); + let mut worker = worker(machine.clone()); worker.state.push_global(process.clone()); + worker.run(); - worker.run(move |worker, process| { - pids_copy.lock().insert(process.identifier()); - worker.state.terminate(); - }); - - assert!(pids.lock().contains(&process.identifier())); - assert_eq!(worker.state.queues[1].has_local_jobs(), false); + assert!(worker.state.pop_global().is_none()); + assert_eq!(worker.state.queues[0].has_local_jobs(), false); } #[test] fn test_run_with_external_jobs() { - let (pids, pids_copy) = pids_set(); - let (_machine, _block, process) = setup(); - let mut worker = worker(); + let (machine, _block, process) = setup(); + let mut worker = worker(machine.clone()); worker.state.queues[0].push_external(process.clone()); + worker.run(); - worker.run(move |worker, process| { - pids_copy.lock().insert(process.identifier()); - worker.state.terminate(); - }); - - assert!(pids.lock().contains(&process.identifier())); + assert_eq!(worker.state.queues[0].has_external_jobs(), false); } #[test] fn test_run_steal_then_terminate() { - let (pids, pids_copy) = pids_set(); - let (_machine, _block, process) = setup(); - let mut worker = worker(); + let (machine, _block, process) = setup(); + let mut worker = worker(machine.clone()); worker.state.queues[1].push_internal(process.clone()); + worker.run(); - worker.run(move |worker, process| { - pids_copy.lock().insert(process.identifier()); - worker.state.terminate(); - }); - - assert!(pids.lock().contains(&process.identifier())); assert_eq!(worker.state.queues[1].has_local_jobs(), false); } #[test] - fn test_run_steal_then_work() { - let (pids, pids_copy) = pids_set(); + fn test_run_work_and_steal() { let (machine, block, process) = setup(); let process2 = process::allocate(&machine.state, &block); - let process2_clone = process2.clone(); - let mut worker = worker(); + let mut worker = worker(machine.clone()); - process.set_main(); - worker.state.queues[1].push_internal(process.clone()); + worker.queue.push_internal(process2); + worker.state.queues[1].push_internal(process); // Here the order of work is: // - // 1. Steal from other queue - // 2. Go back to processing our own queue + // 1. Process local job + // 2. Steal from other queue // 3. Terminate - worker.run(move |worker, process| { - pids_copy.lock().insert(process.identifier()); - - worker.queue.push_internal(process2_clone.clone()); + worker.run(); - if !process.is_main() { - worker.state.terminate(); - } - }); - - assert!(pids.lock().contains(&process.identifier())); - assert!(pids.lock().contains(&process2.identifier())); + assert_eq!(worker.queue.has_local_jobs(), false); assert_eq!(worker.state.queues[1].has_local_jobs(), false); } #[test] fn test_run_work_then_terminate_steal_loop() { - let (pids, pids_copy) = pids_set(); let (machine, block, process) = setup(); let process2 = process::allocate(&machine.state, &block); - let mut worker = worker(); - - worker.state.queues[0].push_internal(process.clone()); - worker.state.queues[1].push_internal(process2.clone()); + let mut worker = worker(machine.clone()); - worker.run(move |worker, process| { - pids_copy.lock().insert(process.identifier()); - worker.state.terminate(); - }); - - assert_eq!(pids.lock().contains(&process.identifier()), true); - assert_eq!(pids.lock().contains(&process2.identifier()), false); + worker.state.queues[0].push_internal(process); + worker.state.queues[1].push_internal(process2); + worker.run(); + assert_eq!(worker.state.queues[0].has_local_jobs(), false); assert!(worker.state.queues[1].has_local_jobs()); } #[test] fn test_run_exclusive_iteration() { - let (pids, pids_copy) = pids_set(); - let (_machine, _block, process) = setup(); - let mut worker = worker(); + let (machine, _block, process) = setup(); + let mut worker = worker(machine.clone()); worker.enter_exclusive_mode(); - worker.queue.push_external(process.clone()); - - worker.run(move |worker, process| { - pids_copy.lock().insert(process.identifier()); - worker.state.terminate(); - }); + worker.queue.push_external(process); + worker.run(); - assert!(pids.lock().contains(&process.identifier())); + assert_eq!(worker.queue.has_external_jobs(), false); } #[test] fn test_enter_exclusive_mode() { - let mut worker = worker(); let (machine, block, process) = setup(); let process2 = process::allocate(&machine.state, &block); + let mut worker = worker(machine.clone()); worker.queue.push_internal(process); worker.queue.push_external(process2); @@ -362,7 +321,8 @@ mod tests { #[test] fn test_leave_exclusive_mode() { - let mut worker = worker(); + let (machine, _block, _process) = setup(); + let mut worker = worker(machine.clone()); worker.enter_exclusive_mode(); worker.leave_exclusive_mode(); @@ -372,7 +332,8 @@ mod tests { #[test] fn test_random_number() { - let mut worker = worker(); + let (machine, _block, _process) = setup(); + let mut worker = worker(machine.clone()); // There is no particular way we can test the exact value, so this is // just a smoke test to see if the method works or not. @@ -381,7 +342,8 @@ mod tests { #[test] fn test_random_number_between() { - let mut worker = worker(); + let (machine, _block, _process) = setup(); + let mut worker = worker(machine.clone()); let number: u8 = worker.random_number_between(0, 10); assert!(number <= 10); @@ -389,7 +351,8 @@ mod tests { #[test] fn test_random_bigint_between() { - let mut worker = worker(); + let (machine, _block, _process) = setup(); + let mut worker = worker(machine.clone()); let min = BigInt::from(0); let max = BigInt::from(10); let number = worker.random_bigint_between(&min, &max); @@ -399,7 +362,8 @@ mod tests { #[test] fn test_random_incremental_number() { - let mut worker = worker(); + let (machine, _block, _process) = setup(); + let mut worker = worker(machine.clone()); let num1 = worker.random_incremental_number(); let num2 = worker.random_incremental_number(); @@ -408,7 +372,8 @@ mod tests { #[test] fn test_random_bytes() { - let mut worker = worker(); + let (machine, _block, _process) = setup(); + let mut worker = worker(machine.clone()); let bytes = worker.random_bytes(4).unwrap(); assert_eq!(bytes.len(), 4); diff --git a/vm/src/scheduler/queue.rs b/vm/src/scheduler/queue.rs index 878ca9e0c..219518beb 100644 --- a/vm/src/scheduler/queue.rs +++ b/vm/src/scheduler/queue.rs @@ -59,7 +59,9 @@ impl Queue { } pub fn decrement_pending_external(&self) { - self.pending_external.fetch_sub(1, Ordering::Release); + if self.pending_external() > 0 { + self.pending_external.fetch_sub(1, Ordering::Release); + } } /// Pushes a job onto the deque. @@ -105,7 +107,13 @@ impl Queue { /// Pops a job from the public channel, without first moving it to the /// private Worker. pub fn pop_external_job(&self) -> Option { - self.receiver.try_recv().ok() + let job = self.receiver.try_recv().ok(); + + if job.is_some() { + self.decrement_pending_external(); + } + + job } /// Moves all jobs from the public channel into the private Worker, without @@ -252,6 +260,7 @@ mod tests { queue.push_external(10); assert_eq!(queue.pop_external_job(), Some(10)); + assert_eq!(queue.pending_external(), 0); } #[test] diff --git a/vm/src/scheduler/worker.rs b/vm/src/scheduler/worker.rs index f89bc79ba..d06d58898 100644 --- a/vm/src/scheduler/worker.rs +++ b/vm/src/scheduler/worker.rs @@ -3,30 +3,60 @@ use crate::scheduler::queue::RcQueue; /// A trait providing the basic building blocks of a worker thread. pub trait Worker { - /// Starts the worker, blocking the current thread until the worker - /// terminates. - fn run(&mut self, callback: F) - where - F: Fn(&mut Self, T); + /// Processes the given job. + fn process_job(&mut self, job: T); + /// Returns the PoolState used by this Worker. fn state(&self) -> &PoolState; + /// Returns the queue owned by this worker. fn queue(&self) -> &RcQueue; + /// Starts the worker, blocking the current thread until the worker + /// terminates. + #[cfg_attr(test, allow(unreachable_code))] + fn run(&mut self) { + while self.state().is_alive() { + if self.process_local_jobs() { + continue; + } + + if self.steal_from_other_queue() { + continue; + } + + if self.steal_from_global_queue() { + continue; + } + + #[cfg(test)] + { + // Since this method never returns unless the pool is + // terminated, calling this method in a test would deadlock the + // test. To prevent this from happening we break instead of + // sleeping when running tests. + break; + } + + self.park(); + } + } + + fn park(&self) { + self.state().park_while(|| !self.state().has_global_jobs()); + } + /// Processes all local jobs until we run out of work. /// /// This method returns true if the worker should self terminate. - fn process_local_jobs(&mut self, callback: &F) -> bool - where - F: Fn(&mut Self, T), - { + fn process_local_jobs(&mut self) -> bool { loop { if !self.state().is_alive() { return true; } if let Some(job) = self.queue().pop() { - callback(self, job); + self.process_job(job); } else { return false; } diff --git a/vm/src/tagged_pointer.rs b/vm/src/tagged_pointer.rs index 2eac25e78..2a8198d16 100644 --- a/vm/src/tagged_pointer.rs +++ b/vm/src/tagged_pointer.rs @@ -8,7 +8,7 @@ use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; /// The mask to use for untagging a pointer. -const UNTAG_MASK: usize = (!0x3) as usize; +const UNTAG_MASK: usize = (!0x7) as usize; /// Returns true if the pointer has the given bit set to 1. pub fn bit_is_set(pointer: *mut T, bit: usize) -> bool { diff --git a/vm/src/timer.rs b/vm/src/timer.rs deleted file mode 100644 index 34c144f0d..000000000 --- a/vm/src/timer.rs +++ /dev/null @@ -1,164 +0,0 @@ -//! Timer for measuring the elapsed time between two points. - -use std::time::Instant; -use std::u64; - -#[derive(Default)] -pub struct Timer { - start: Option, - stop: Option, -} - -impl Timer { - pub fn new() -> Self { - Self::default() - } - - pub fn now() -> Self { - let mut timer = Timer::new(); - - timer.start(); - - timer - } - - /// Returns the duration in nanoseconds. - /// - /// Since this method returns the time as a u64 care should be taken to - /// ensure the duration is not long enough for the value to overflow. - pub fn duration_nanosec(&self) -> u64 { - if self.finished() { - let start = self.start.unwrap(); - let stop = self.stop.unwrap(); - let duration = stop.duration_since(start); - - (duration.as_secs() * 1_000_000_000) - + u64::from(duration.subsec_nanos()) - } else { - 0 - } - } - - /// Returns the duration in milliseconds. - pub fn duration_msec(&self) -> f64 { - self.duration_nanosec() as f64 / 1_000_000.0 - } - - /// Returns the duration in seconds. - pub fn duration_sec(&self) -> f64 { - self.duration_nanosec() as f64 / 1_000_000_000.0 - } - - pub fn set_start_time(&mut self, time: Instant) { - self.start = Some(time); - } - - pub fn start(&mut self) { - self.start = Some(Instant::now()); - } - - pub fn stop(&mut self) { - self.stop = Some(Instant::now()); - } - - pub fn finished(&self) -> bool { - self.start.is_some() && self.stop.is_some() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::thread; - use std::time::Duration; - - #[test] - fn test_new() { - let timer = Timer::new(); - - assert!(timer.start.is_none()); - assert!(timer.stop.is_none()); - } - - #[test] - fn test_now() { - let timer = Timer::now(); - - assert!(timer.start.is_some()); - } - - #[test] - fn test_duration_nanosec() { - let mut timer = Timer::new(); - - timer.start(); - thread::sleep(Duration::from_millis(10)); - timer.stop(); - - assert!(timer.duration_nanosec() >= 10000000 as u64); - } - - #[test] - fn test_duration_msec() { - let mut timer = Timer::new(); - - timer.start(); - thread::sleep(Duration::from_millis(10)); - timer.stop(); - - assert!(timer.duration_msec() >= 10.0); - } - - #[test] - fn test_duration_sec() { - let mut timer = Timer::new(); - - timer.start(); - thread::sleep(Duration::from_millis(10)); - timer.stop(); - - assert!(timer.duration_sec() >= 0.01); - } - - #[test] - fn test_set_start_time() { - let mut timer = Timer::new(); - - timer.set_start_time(Instant::now()); - - assert!(timer.start.is_some()); - } - - #[test] - fn test_start() { - let mut timer = Timer::new(); - - timer.start(); - - assert!(timer.start.is_some()); - } - - #[test] - fn test_stop() { - let mut timer = Timer::new(); - - timer.stop(); - - assert!(timer.stop.is_some()); - } - - #[test] - fn test_finished() { - let mut timer = Timer::new(); - - assert_eq!(timer.finished(), false); - - timer.start(); - - assert_eq!(timer.finished(), false); - - timer.stop(); - - assert_eq!(timer.finished(), true); - } -} diff --git a/vm/src/vm/machine.rs b/vm/src/vm/machine.rs index b5e8a8856..4f18aa4bd 100644 --- a/vm/src/vm/machine.rs +++ b/vm/src/vm/machine.rs @@ -1,7 +1,7 @@ //! Virtual Machine for running instructions use crate::compiled_code::CompiledCodePointer; use crate::execution_context::ExecutionContext; -use crate::gc::request::Request as GcRequest; +use crate::gc::collection::Collection; use crate::integer_operations; use crate::module_registry::{ModuleRegistry, RcModuleRegistry}; use crate::network_poller::worker::Worker as NetworkPollerWorker; @@ -13,7 +13,6 @@ use crate::process::RcProcess; use crate::runtime_error::RuntimeError; use crate::runtime_panic; use crate::scheduler::join_list::JoinList; -use crate::scheduler::pool::Pool; use crate::scheduler::process_worker::ProcessWorker; use crate::vm::array; use crate::vm::block; @@ -34,7 +33,6 @@ use crate::vm::state::RcState; use crate::vm::string; use crate::vm::time; use num_bigint::BigInt; -use rayon::ThreadPoolBuilder; use std::i32; use std::ops::{Add, Mul, Sub}; use std::panic; @@ -178,7 +176,6 @@ impl Machine { /// This method returns true if the VM terminated successfully, false /// otherwise. pub fn start(&self, file: &str) { - self.configure_rayon(); self.schedule_main_process(file); let gc_pool_guard = self.start_gc_threads(); @@ -206,41 +203,17 @@ impl Machine { } } - fn configure_rayon(&self) { - ThreadPoolBuilder::new() - .thread_name(|idx| format!("rayon {}", idx)) - .num_threads(self.state.config.generic_parallel_threads) - .build_global() - .unwrap(); - } - fn start_primary_threads(&self) -> JoinList<()> { - let machine = self.clone(); - - self.state - .scheduler - .primary_pool - .start_main(move |worker, process| { - machine.run_with_error_handling(worker, &process) - }) + self.state.scheduler.primary_pool.start_main(self.clone()) } fn start_blocking_threads(&self) -> JoinList<()> { - let machine = self.clone(); - - self.state - .scheduler - .blocking_pool - .start(move |worker, process| { - machine.run_with_error_handling(worker, &process) - }) + self.state.scheduler.blocking_pool.start(self.clone()) } /// Starts the garbage collection threads. fn start_gc_threads(&self) -> JoinList<()> { - self.state - .gc_pool - .start(move |_, mut request| request.perform()) + self.state.gc_pool.start(self.state.clone()) } fn start_timeout_worker_thread(&self) -> thread::JoinHandle<()> { @@ -2081,10 +2054,7 @@ impl Machine { worker.leave_exclusive_mode(); } - // We must clean up _after_ removing the process from the process table - // to prevent a cleanup from happening while the process is still - // receiving messages as this could lead to memory not being reclaimed. - self.schedule_gc_for_finished_process(&process); + process.reclaim_and_finalize(&self.state); // Terminate once the main process has finished execution. if process.is_main() { @@ -2099,33 +2069,15 @@ impl Machine { /// /// Returns true if a process should be suspended for garbage collection. fn gc_safepoint(&self, process: &RcProcess) -> bool { - if process.should_collect_young_generation() { - self.schedule_gc_request(GcRequest::heap( - self.state.clone(), - process.clone(), - )); - - true - } else if process.should_collect_mailbox() { - self.schedule_gc_request(GcRequest::mailbox( - self.state.clone(), - process.clone(), - )); - - true - } else { - false + if !process.should_collect_young_generation() { + return false; } - } - - fn schedule_gc_request(&self, request: GcRequest) { - self.state.gc_pool.schedule(request); - } - fn schedule_gc_for_finished_process(&self, process: &RcProcess) { - let request = GcRequest::finished(self.state.clone(), process.clone()); + self.state + .gc_pool + .schedule(Collection::new(process.clone())); - self.schedule_gc_request(request); + true } #[inline(always)] diff --git a/vm/src/vm/state.rs b/vm/src/vm/state.rs index 9ff0d29c3..526154d66 100644 --- a/vm/src/vm/state.rs +++ b/vm/src/vm/state.rs @@ -5,7 +5,7 @@ //! etc. use crate::arc_without_weak::ArcWithoutWeak; use crate::config::Config; -use crate::gc::request::Request; +use crate::gc::coordinator::Pool as GcPool; use crate::immix::copy_object::CopyObject; use crate::immix::global_allocator::{GlobalAllocator, RcGlobalAllocator}; use crate::immix::permanent_allocator::PermanentAllocator; @@ -13,7 +13,6 @@ use crate::immutable_string::ImmutableString; use crate::network_poller::NetworkPoller; use crate::object_pointer::ObjectPointer; use crate::object_value; -use crate::scheduler::generic_pool::GenericPool; use crate::scheduler::process_scheduler::ProcessScheduler; use crate::scheduler::timeout_worker::TimeoutWorker; use crate::string_pool::StringPool; @@ -55,7 +54,7 @@ pub struct State { pub scheduler: ProcessScheduler, /// The pool to use for garbage collection. - pub gc_pool: GenericPool, + pub gc_pool: GcPool, /// The permanent memory allocator, used for global data. pub permanent_allocator: Mutex>, @@ -165,7 +164,7 @@ impl State { byte_array_prototype.set_prototype(object_proto); } - let gc_pool = GenericPool::new("GC".to_string(), config.gc_threads); + let gc_pool = GcPool::new(config.gc_threads); let mut state = State { scheduler: ProcessScheduler::new( diff --git a/vm/src/vm/test.rs b/vm/src/vm/test.rs index 03fab7f92..5b3357f8a 100644 --- a/vm/src/vm/test.rs +++ b/vm/src/vm/test.rs @@ -11,10 +11,22 @@ use crate::vm::state::State; /// Sets up a VM with a single process. pub fn setup() -> (Machine, Block, RcProcess) { - let state = State::with_rc(Config::new(), &[]); + let mut config = Config::new(); + + config.primary_threads = 2; + config.blocking_threads = 2; + config.gc_threads = 2; + config.tracer_threads = 2; + + let state = State::with_rc(config, &[]); let name = state.intern_string("a".to_string()); let machine = Machine::default(state); - let mut code = CompiledCode::new(name, name, 1, Vec::new()); + let mut code = CompiledCode::new( + name, + name, + 1, + vec![new_instruction(InstructionType::Return, vec![0])], + ); // Reserve enough space for registers/locals for most tests. code.locals = 32; @@ -39,6 +51,8 @@ pub fn setup() -> (Machine, Block, RcProcess) { Block::new(module.code(), None, machine.state.top_level, scope); let process = process::allocate(&machine.state, &block); + process.set_main(); + (block, process) };