Compressed pointers

crates/starlight/Cargo.toml

@@ -42,6 +42,7 @@ val-as-u64 = []
 enable-gc-tracking = []
 no-inline-caching = []
 valgrind = []
+compressed-pointers = []
 # enable performance counter for interpreter/gc/codegen
 perf = []
 ffi = ["libloading", "libffi"]

crates/starlight/src/bytecode.rs

@@ -2,7 +2,10 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 use crate::{
-    gc::cell::{GcPointer, Trace, Tracer},
+    gc::{
+        cell::{GcPointer, Trace, Tracer},
+        compressed_pointer::CompressedPtr,
+    },
     vm::{object::JsObject, structure::Structure, structure_chain::StructureChain},
 };
 
@@ -11,23 +14,23 @@ pub mod profile;
 #[derive(Clone, Copy, PartialEq, Eq)]
 pub enum GetByIdMode {
     Default,
-    ProtoLoad(GcPointer<JsObject> /*cached slot */),
+    ProtoLoad(CompressedPtr<JsObject> /*cached slot */),
     ArrayLength,
 }
 pub enum TypeFeedBack {
     StructureCache {
-        structure: GcPointer<Structure>,
+        structure: CompressedPtr<Structure>,
     },
     PropertyCache {
-        structure: GcPointer<Structure>,
+        structure: CompressedPtr<Structure>,
         offset: u32,
         mode: GetByIdMode,
     },
     PutByIdFeedBack {
-        new_structure: Option<GcPointer<Structure>>,
-        old_structure: Option<GcPointer<Structure>>,
+        new_structure: Option<CompressedPtr<Structure>>,
+        old_structure: Option<CompressedPtr<Structure>>,
         offset: u32,
-        structure_chain: Option<GcPointer<StructureChain>>,
+        structure_chain: Option<CompressedPtr<StructureChain>>,
     },
     None,
 }

crates/starlight/src/gc.rs

@@ -14,6 +14,8 @@
 //! `letroot!` is not required to use anymore.
 //!
 #![allow(dead_code, unused_variables)]
+use self::allocation::Space;
+use self::space_bitmap::SpaceBitmap;
 use crate::options::Options;
 use crate::vm::Runtime;
 use crate::{
@@ -24,14 +26,18 @@ use crate::{
         serializer::{Serializable, SnapshotSerializer},
     },
 };
+use std::collections::LinkedList;
 use std::intrinsics::{copy_nonoverlapping, unlikely};
+use std::mem::swap;
 use std::ops::Deref;
 use std::{any::TypeId, cmp::Ordering, fmt, marker::PhantomData};
 use std::{
     mem::size_of,
     ptr::{null_mut, NonNull},
 };
 use std::{u8, usize};
+use wtf_rs::approximate_stack_pointer;
+use yastl::Pool;
 
 /// Like C's offsetof but you can use it with GC-able objects to get offset from GC header to field.
 ///
@@ -69,6 +75,7 @@ macro_rules! offsetof {
 pub mod cell;
 pub mod snapshot;
 pub const K: usize = 1024;
+pub mod compressed_pointer;
 pub mod mem;
 pub mod os;
 pub mod pmarking;
@@ -77,6 +84,14 @@ pub mod safepoint;
 pub mod vgrs;
 #[macro_use]
 pub mod shadowstack;
+pub mod allocation;
+pub mod block;
+pub mod block_allocator;
+pub mod collector;
+pub mod constants;
+pub mod large_object_space;
+pub mod space_bitmap;
+
 pub trait MarkingConstraint {
     fn name(&self) -> &str {
         "<anonymous name>"
@@ -439,20 +454,6 @@ mod tests {
     }
 }
 
-pub mod allocation;
-pub mod block;
-pub mod block_allocator;
-pub mod collector;
-pub mod constants;
-pub mod large_object_space;
-pub mod space_bitmap;
-use std::collections::LinkedList;
-use std::mem::swap;
-use wtf_rs::approximate_stack_pointer;
-use yastl::Pool;
-
-use self::allocation::Space;
-use self::space_bitmap::SpaceBitmap;
 /// Visits garbage collected objects
 pub struct SlotVisitor {
     pub(super) queue: Vec<*mut GcPointerBase>,
@@ -463,6 +464,12 @@ unsafe impl Send for SlotVisitor {}
 unsafe impl Send for Space {}
 unsafe impl Sync for Space {}
 impl Tracer for SlotVisitor {
+    fn visit_compressed(&mut self, cell: compressed_pointer::CompressedPtr<dyn GcCell>) {
+        unsafe {
+            let mut cell = cell.get(self.heap);
+            cell.trace(self);
+        }
+    }
     fn visit_weak(&mut self, _slot: *const WeakSlot) {
         /* no-op */
     }
@@ -542,7 +549,7 @@ pub struct Heap {
     threadpool: Option<Pool>,
     n_workers: u32,
     max_heap_size: usize,
-    space: Space,
+    pub(crate) space: Space,
     verbose: bool,
     allocation_color: u8,
     pub(super) weak_refs: Vec<GcPointer<WeakSlot>>,

crates/starlight/src/gc/allocation.rs

@@ -1,7 +1,9 @@
 use std::{
     fmt::Debug,
+    intrinsics::unlikely,
+    marker::PhantomData,
     mem::size_of,
-    ptr::{drop_in_place, null_mut},
+    ptr::{drop_in_place, null_mut, NonNull},
 };
 
 use wtf_rs::round_up;
@@ -17,6 +19,8 @@ use crate::{
 use super::{
     block::FreeList,
     block_allocator::BlockAllocator,
+    cell::{GcCell, GcPointer},
+    compressed_pointer::HeapPage,
     large_object_space::{LargeObjectSpace, PreciseAllocation},
     space_bitmap::SpaceBitmap,
 };
@@ -170,6 +174,12 @@ impl Drop for Space {
 }
 
 impl Space {
+    pub fn heap_base(&self) -> usize {
+        unsafe { (*self.block_allocator).mmap.start() as _ }
+    }
+    pub fn heap_end(&self) -> usize {
+        unsafe { (*self.block_allocator).mmap.end() as _ }
+    }
     unsafe fn allocator_for_slow<'a>(&'a mut self, size: usize) -> Option<*mut LocalAllocator> {
         let index = size_class_to_index(size);
         let size_class = self.size_class_for_size_step[index];
@@ -308,6 +318,45 @@ impl Space {
             }
         }
     }
+
+    pub unsafe fn compress_pointer(&self, cell: GcPointer<dyn GcCell>) -> u32 {
+        let mut ptr = cell.base.as_ptr() as usize;
+        let compressed = if PreciseAllocation::is_precise(ptr as _) {
+            let mut ix = (*PreciseAllocation::from_cell(ptr as _)).index_in_space;
+            ix |= 1 << 0;
+            ix
+        } else {
+            debug_assert!(self.is_heap_pointer(cell.base.as_ptr() as _));
+            ptr -= (*self.block_allocator).mmap.start() as usize;
+            //ptr >>= 3;
+            ptr as u32
+        };
+        compressed
+    }
+    pub unsafe fn decompress_pointer<T: ?Sized + GcCell>(&self, cell: u32) -> GcPointer<T> {
+        if unlikely((cell & 1) != 0) {
+            let index = cell >> 1;
+            assert!(index < self.precise_allocations.allocations.len() as u32);
+            let precise_allocation = self.precise_allocations.allocations[index as usize];
+            unsafe {
+                GcPointer {
+                    base: NonNull::new_unchecked((*precise_allocation).cell()),
+                    marker: PhantomData,
+                }
+            }
+        } else {
+            let mut decompressed = cell as usize;
+            //decompressed <<= 3;
+            decompressed += (*self.block_allocator).mmap.start() as usize;
+            debug_assert!(self.is_heap_pointer(decompressed as _));
+            unsafe {
+                GcPointer {
+                    base: NonNull::new_unchecked(decompressed as _),
+                    marker: PhantomData,
+                }
+            }
+        }
+    }
 }
 #[derive(Clone, Copy)]
 pub struct LocalAllocator {
@@ -485,3 +534,12 @@ impl Debug for GCOOM {
         )
     }
 }
+
+impl HeapPage for Space {
+    fn decompress<T: ?Sized + GcCell>(&self, compressed: u32) -> GcPointer<T> {
+        unsafe { self.decompress_pointer::<T>(compressed) }
+    }
+    fn compress(&self, ptr: GcPointer<dyn GcCell>) -> u32 {
+        unsafe { self.compress_pointer(ptr) }
+    }
+}

crates/starlight/src/gc/cell.rs

@@ -17,13 +17,16 @@ use std::{
 };
 use wtf_rs::tagged_ptr::TaggedPointer;
 
+use super::compressed_pointer::CompressedPtr;
+
 pub trait Tracer {
     fn visit(&mut self, cell: &mut GcPointer<dyn GcCell>) -> GcPointer<dyn GcCell>;
     fn visit_raw(&mut self, cell: &mut *mut GcPointerBase) -> GcPointer<dyn GcCell>;
     /// Add memory range to search for conservative roots. Note that some collectors might scan this range multiple
     /// times if you supplied same range multiple times.
     fn add_conservative(&mut self, from: usize, to: usize);
     fn visit_weak(&mut self, at: *const WeakSlot);
+    fn visit_compressed(&mut self, cell: CompressedPtr<dyn GcCell>);
 }
 
 /// Indicates that a type can be traced by a garbage collector.
@@ -171,8 +174,8 @@ pub struct GcPointer<T: ?Sized> {
 }
 
 impl<T: GcCell + ?Sized> GcPointer<T> {
-    pub fn ptr_eq<U: GcCell + ?Sized>(this: &Self, other: &GcPointer<U>) -> bool {
-        this.base == other.base
+    pub fn ptr_eq<U: GcCell + ?Sized>(&self, other: &GcPointer<U>) -> bool {
+        self.base == other.base
     }
     #[inline]
     pub fn as_dyn(self) -> GcPointer<dyn GcCell> {

crates/starlight/src/gc/compressed_pointer.rs

@@ -0,0 +1,115 @@
+use crate::{
+    prelude::{Deserializable, Serializable},
+    vm::Runtime,
+};
+
+use super::cell::{GcCell, GcPointer, Trace};
+use std::marker::PhantomData;
+
+#[repr(transparent)]
+pub struct CompressedPtr<T: GcCell + ?Sized> {
+    marker: PhantomData<T>,
+    #[cfg(feature = "compressed-pointers")]
+    pointer: u32,
+    #[cfg(not(feature = "compressed-pointers"))]
+    pointer: GcPointer<T>,
+}
+#[allow(unused_variables)]
+impl<T: GcCell + ?Sized> CompressedPtr<T> {
+    #[inline(always)]
+    pub fn new<PAGE: HeapPage>(page: &PAGE, gcptr: GcPointer<T>) -> Self {
+        #[cfg(feature = "compressed-pointers")]
+        let compressed = page.compress(gcptr.as_dyn());
+        #[cfg(not(feature = "compressed-pointers"))]
+        let compressed = gcptr;
+        Self {
+            pointer: compressed,
+            marker: PhantomData,
+        }
+    }
+    #[inline(always)]
+    pub fn get<PAGE: HeapPage>(&self, page: &PAGE) -> GcPointer<T> {
+        #[cfg(feature = "compressed-pointers")]
+        {
+            unsafe { page.decompress(self.pointer) }
+        }
+        #[cfg(not(feature = "compressed-pointers"))]
+        {
+            self.pointer
+        }
+    }
+    #[inline(always)]
+    pub fn ptr_eq<U: GcCell + ?Sized>(&self, other: &CompressedPtr<U>) -> bool {
+        #[cfg(feature = "compressed-pointers")]
+        {
+            self.pointer == other.pointer
+        }
+        #[cfg(not(feature = "compressed-pointers"))]
+        {
+            GcPointer::ptr_eq(&self.pointer, &other.pointer)
+        }
+    }
+
+    pub fn as_dyn(&self) -> CompressedPtr<dyn GcCell> {
+        CompressedPtr {
+            #[cfg(feature = "compressed-pointers")]
+            pointer: self.pointer,
+            #[cfg(not(feature = "compressed-pointers"))]
+            pointer: self.pointer.as_dyn(),
+            marker: PhantomData,
+        }
+    }
+}
+pub trait HeapPage {
+    fn compress(&self, ptr: GcPointer<dyn GcCell>) -> u32;
+    fn decompress<T: ?Sized + GcCell>(&self, compressed: u32) -> GcPointer<T>;
+}
+
+impl<T: GcCell + ?Sized> Copy for CompressedPtr<T> {}
+impl<T: GcCell + ?Sized> Clone for CompressedPtr<T> {
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+impl<T: GcCell + ?Sized> Serializable for CompressedPtr<T> {
+    fn serialize(&self, serializer: &mut crate::prelude::SnapshotSerializer) {
+        self.get(&*serializer.rt()).serialize(serializer);
+    }
+}
+impl<T: GcCell + ?Sized> Deserializable for CompressedPtr<T> {
+    unsafe fn deserialize_inplace(deser: &mut crate::prelude::Deserializer) -> Self {
+        let rt = &mut *deser.rt;
+        let ref_ = GcPointer::<T>::deserialize_inplace(deser);
+        Self::new(rt, ref_)
+    }
+    unsafe fn deserialize(_at: *mut u8, _deser: &mut crate::prelude::Deserializer) {
+        unreachable!()
+    }
+    unsafe fn allocate(
+        _rt: &mut Runtime,
+        _deser: &mut crate::prelude::Deserializer,
+    ) -> *mut super::cell::GcPointerBase {
+        unreachable!()
+    }
+}
+
+unsafe impl<T: GcCell + ?Sized> Trace for CompressedPtr<T> {
+    fn trace(&mut self, visitor: &mut dyn super::cell::Tracer) {
+        visitor.visit_compressed(self.as_dyn());
+    }
+}
+
+impl<T: GcCell + ?Sized> GcCell for CompressedPtr<T> {
+    fn deser_pair(&self) -> (usize, usize) {
+        (Self::deserialize as _, Self::allocate as _)
+    }
+}
+
+impl<T: GcCell + ?Sized> PartialEq for CompressedPtr<T> {
+    fn eq(&self, other: &Self) -> bool {
+        self.ptr_eq(other)
+    }
+}
+
+impl<T: GcCell + ?Sized> Eq for CompressedPtr<T> {}

crates/starlight/src/gc/large_object_space.rs

@@ -210,6 +210,13 @@ impl LargeObjectSpace {
             retain
         });
 
+        for (index, alloc) in self.allocations.iter().enumerate() {
+            unsafe {
+                let alloc = &mut **alloc;
+                alloc.index_in_space = index as _;
+            }
+        }
+
         sweeped
     }
     #[allow(clippy::collapsible_if)]