From 864b9d40f773ae08d10ca581ddd802f141dce387 Mon Sep 17 00:00:00 2001 From: Brian Yahn Date: Sun, 7 Dec 2025 12:23:09 +0000 Subject: [PATCH 1/2] Migrate to O(1) deletion (w/ copy) --- spec/arena_spec.rb | 264 +++++++++++++++++++++----------- spec/memory_integration_spec.rb | 2 +- spec/types_spec.rb | 94 +----------- spec/vm_spec.rb | 128 ++-------------- src/arena.rb | 191 +++++++++++------------ src/memory_visualizer.rb | 83 +++++----- src/types.rb | 220 ++++++++++++++++++++++---- src/value.rb | 17 +- src/vm.rb | 164 +++++++++----------- 9 files changed, 587 insertions(+), 576 deletions(-) diff --git a/spec/arena_spec.rb b/spec/arena_spec.rb index 37f0c02b..2303e527 100644 --- a/spec/arena_spec.rb +++ b/spec/arena_spec.rb @@ -1,139 +1,219 @@ require 'rspec' require_relative '../src/arena' +require_relative '../src/value' require_relative '../src/types' # Needed for FluxObject -# A helper class to track poisoning status without complex logic -class TestObject < FluxObject - def initialize(register: true) - super(register: register) - @poisoned = false - end -end - RSpec.describe Arena do before(:each) do Arena.reset! end - describe "Registration & Lookup" do - it "registers an object and finds it by masked address" do - obj = TestObject.new + context "Nested Stack Frames" do + it "destroys all objects on a rewind with no promotion" do + # 1. Start Mark + start_mark = Arena.current.mark + expect(start_mark).to eq(0) + + # 2. Allocate Real Objects + str1 = FluxString.new("First", register: true) + str2 = FluxString.new("Second", register: true) - # Calculate expected address using the mask - expected_addr = obj.flux_id & Arena::ID_MASK + expect(str1.address).to eq(0) + expect(str2.address).to eq(1) + expect(Arena.current.mark).to eq(2) - found = Arena.current.find_object_by_address(expected_addr) - expect(found).to eq(obj) + # 3. Rewind (End of Scope) + Arena.current.rewind(start_mark) + + expect(Arena.current.mark).to eq(0) end - it "does not find objects that were never registered" do - obj = TestObject.new(register: false) - addr = obj.flux_id & Arena::ID_MASK + it "overwrites 'dead' objects when new ones are allocated" do + # 1. Allocate a "Ghost" + ghost_str = FluxString.new("I will die", register: true) + expect(ghost_str.address).to eq(0) + + # 2. Rewind (Ghost is now in the dead zone) + Arena.current.rewind(0) + + # 3. Allocate a New Object + new_str = FluxString.new("I am new", register: true) + + # 4. Verify Overwrite (Address Reuse) + expect(new_str.address).to eq(0) - found = Arena.current.find_object_by_address(addr) - expect(found).to be_nil + # Verify the slot in memory points to the new object + expect(Arena.current.memory[0]).to eq(new_str) + expect(Arena.current.memory[0].to_s).to eq("I am new") + end + + it "promotes an object correctly across stack frames" do + frame_start_mark = Arena.current.mark # 0 + + # 1. Allocate "Local" Object in Frame + local_list = FluxArray.new(10, nil, type: :int64) + local_list[0] = Value.box_number(100) + local_list[1] = Value.box_number(220) + local_list[2] = Value.box_number(333) + + expect(local_list.address).to eq(0) + expect(Arena.current.mark).to eq(1) # [local_list] + + # 2. PROMOTE + survivor = Arena.current.promote(local_list, frame_start_mark) + expect(survivor).to be_a(FluxArray) + expect(survivor.object_id).to_not eq(local_list.object_id) + + # 3. Verify Address Reuse + # Step A: Rewind moved cursor to 0. + # Step B: Registering the survivor moved cursor back to 1. + # Therefore, the survivor effectively "stole" the slot of the dead object. + expect(Arena.current.mark).to eq(1) + expect(survivor.address).to eq(0) + + # 6. Verify Data Integrity + # Check that the deep copy actually worked. + expect(survivor.read_at(0)).to eq(100) + expect(survivor.read_at(1)).to eq(220) + expect(survivor.read_at(2)).to eq(333) end - end - describe "Static Registration (Constants)" do - it "registers static objects that persist across resets" do - # 1. Register a static object - static_obj = TestObject.new(register: :static) - static_addr = static_obj.flux_id & Arena::ID_MASK + it "promotes a Nexted object (2D array) correctly across stack frames" do + frame_start_mark = Arena.current.mark # 0 - # 2. Reset the Arena (Simulate new VM run) - Arena.reset! + # INNER LIST: A packed NanBox array [10.5, 20] + # Allocated at Address 0 + inner_list = FluxArray.new(5, nil, type: :nanbox) + inner_list[0] = Value.box_number(10.5) + inner_list[1] = Value.box_number(20) + + expect(inner_list.address).to eq(0) + + # OUTER LIST: A standard Object array containing a pointer to inner_list + # Allocated at Address 1 + # Note: Outer list MUST be :obj type to store Heap Pointers + outer_list = FluxArray.new(1, nil, type: :obj) + outer_list[0] = Value.box_obj(inner_list) + + expect(outer_list.address).to eq(1) + expect(Arena.current.mark).to eq(2) # [inner, outer] - # 3. It should still be found - found = Arena.current.find_object_by_address(static_addr) - expect(found).to eq(static_obj) - expect(found.is_alive).to be true + # PROMOTE + survivor = Arena.current.promote(outer_list, frame_start_mark) + expect(survivor).to be_a(FluxArray) + + # Verify Address Reuse + expect(survivor.object_id).to_not eq(outer_list.object_id) + expect(Arena.current.mark).to eq(2) + expect(survivor.address).to eq(0) + + # Verify Child + boxed_child = survivor[0] + survivor_inner = Value.as_obj(boxed_child) + expect(survivor_inner.address).to eq(1) + expect(survivor_inner.object_id).to_not eq(inner_list.object_id) + + + # Verify Data Integrity - Check that the deep copy actually worked. + expect(survivor_inner.read_at(0)).to eq(10.5) + expect(survivor_inner.read_at(1)).to eq(20) end - end - describe "Stack Lifecycle (Mark & Rewind)" do - it "poisons objects allocated after the mark" do - # 1. Base level object - root_obj = TestObject.new - # 2. Mark the stack (Start of function) + it "promotes a FluxString correctly" do mark = Arena.current.mark - # 3. Allocate 'local' objects - local_obj = TestObject.new - - # 4. Rewind (End of function) - Arena.current.rewind(mark) + # 1. Allocate + local_str = FluxString.new("Hello World", register: true) + expect(local_str.address).to eq(0) - # Root should be alive - expect(root_obj.is_alive?).to be(true) - expect(Arena.current.find_object_by_address(root_obj.flux_id & Arena::ID_MASK)).to eq(root_obj) + # 2. Promote + survivor = Arena.current.promote(local_str, mark) - # Local should be dead and removed from registry - expect(local_obj.is_poisoned?).to be(true) - expect(Arena.current.find_object_by_address(local_obj.flux_id & Arena::ID_MASK)).to be_nil + # 3. Verify + expect(survivor.to_s).to eq("Hello World") + expect(survivor.address).to eq(0) + expect(survivor.object_id).to_not eq(local_str.object_id) end - it "handles nested stack frames correctly" do - mark1 = Arena.current.mark - obj1 = TestObject.new + it "discards objects that were NOT promoted" do + start_mark = Arena.current.mark + + # 1. Allocate Survivor (String) + survivor_source = FluxString.new("Survivor", register: true) + + # 2. Allocate Victim (Array) + victim = FluxArray.new(10, nil, type: :int64) - mark2 = Arena.current.mark - obj2 = TestObject.new + expect(Arena.current.mark).to eq(2) # [String, Array] - # Rewind inner frame - Arena.current.rewind(mark2) - expect(obj2.is_poisoned?).to be(true) - expect(obj1.is_alive?).to be(true) + # 3. Promote only the String + final_obj = Arena.current.promote(survivor_source, start_mark) - # Rewind outer frame - Arena.current.rewind(mark1) - expect(obj1.is_poisoned?).to be(true) + # 4. Assertions + # Cursor should be back at 1 (Only final_obj exists) + expect(Arena.current.mark).to eq(1) + + # Slot 0 should hold the survivor + expect(Arena.current.memory[0]).to eq(final_obj) + expect(final_obj.to_s).to eq("Survivor") + + # Slot 1 is now free/garbage. + # (If we allocate now, it should take slot 1) + next_obj = FluxString.new("Next", register: true) + expect(next_obj.address).to eq(1) end end - describe "Promotion (RVO)" do - it "protects promoted objects from being poisoned during rewind" do - mark = Arena.current.mark + describe "Static Registration" do + it "registers static strings with High Bit set" do + Arena.reset! - # Allocate object in this scope - result_obj = TestObject.new - garbage_obj = TestObject.new + # 1. Register Static String (e.g. Constant Pool) + static_str = FluxString.new("Global Constant", register: :static) - # Promote the result - Arena.current.promote(result_obj) + # Check address format (High bit 47 set) + expected_mask = (1 << 47) + expect(static_str.address & expected_mask).to eq(expected_mask) - # Rewind - Arena.current.rewind(mark) + # 2. Verify Lookup + found = Arena.current[static_str.address] + expect(found).to eq(static_str) + expect(found.to_s).to eq("Global Constant") + end + + it "persists static objects across Arena.reset!" do + static_str = FluxString.new("Persistent", register: :static) + addr = static_str.address - # Result should survive - expect(result_obj.is_alive?).to be(true) - expect(result_obj.is_alive?).to be(true) - # Registry lookup should still work - expect(Arena.current.find_object_by_address(result_obj.flux_id & Arena::ID_MASK)).to eq(result_obj) + # Wipe Heap + Arena.reset! + expect(Arena.current.mark).to eq(0) - # Garbage should be dead - expect(garbage_obj.is_poisoned?).to be(true) + # Static lookup still works + found = Arena.current[addr] + expect(found).to eq(static_str) end end - describe "Masking Logic" do - it "handles object_ids larger than the mask correctly" do - # This is harder to mock without stubbing object_id, - # but we can verify the mask constant exists and is applied. + describe "Views (MemorySlice)" do + it "Run-time Errors if attempted to promote" do + frame_mark = Arena.current.mark - obj = TestObject.new - raw_id = obj.flux_id - masked_id = raw_id & Arena::ID_MASK + # 1. Create a String on the Heap + str_obj = FluxString.new("Hello World", register: true) - # Verify internal storage uses masked key - internal_map = Arena.current.instance_variable_get(:@live_objects) - expect(internal_map.key?(masked_id)).to be true + # 2. Create a View (Slice) pointing to it + # Slices are usually registers (untracked), so we don't register this one + # (or we do, doesn't matter for this test logic). + slice = MemorySlice.new(str_obj, 0, 5) - # If the ID was small enough, masked == raw - # If large, masked != raw. Both should be consistent. - found = Arena.current.find_object_by_address(masked_id) - expect(found).to eq(obj) + # 3. Promote the Slice + # This forces the Arena to look at slice.container (str_obj) + # and Deep Copy it too. + expect { + Arena.current.promote(slice, frame_mark) + }.to raise_error(RuntimeError, /Dangling Pointer/) end end end diff --git a/spec/memory_integration_spec.rb b/spec/memory_integration_spec.rb index 35cf4c2c..2b5feb12 100644 --- a/spec/memory_integration_spec.rb +++ b/spec/memory_integration_spec.rb @@ -60,7 +60,7 @@ it "raises Segfault/MemoryError when unboxing a poisoned (dead) object" do # 1. Create object mark = Arena.current.mark - obj = FluxString.new("Temporary") + obj = FluxString.new("Temporary", register: true) boxed = Value.box_obj(obj) # 2. Kill it (Rewind) diff --git a/spec/types_spec.rb b/spec/types_spec.rb index be4b3264..c6acb25d 100644 --- a/spec/types_spec.rb +++ b/spec/types_spec.rb @@ -5,11 +5,6 @@ require_relative '../src/value' RSpec.describe "Flux Type System" do - # Reset the Arena before every test so allocations don't leak - before(:each) do - Arena.reset! - end - describe FluxByte do it "wraps around 255" do b = FluxByte.new(255) @@ -140,72 +135,13 @@ expect(view.read_at(0)).to eq(99) end end - - describe "Arena Memory Safety" do - it "poisons objects after a rewind (Use After Free)" do - # 1. Start a "Function Frame" - mark = Arena.current.mark - - # 2. Allocate object inside the frame - local_arr = FluxArray.new(nil, [1, 2, 3]) - - # Sanity check: it works - expect(local_arr[0]).to eq(1) - - # 3. End function (Rewind the stack) - Arena.current.rewind(mark) - - # 4. Try to access the object (Dangling Pointer) - expect { - local_arr[0] - }.to raise_error(/Memory Error: Use After Free/) - end - - it "poisons Views if the Owner dies" do - mark = Arena.current.mark - - owner = FluxArray.new(nil, [10, 20]) - - # Create View OUTSIDE the scope (simulate return or promotion) - # We manually promote it or create it in a way that survives rewind. - view = MemorySlice.new(owner, 0, 1) - Arena.current.promote(view) # This causes survival after rewind - - # Kill the stack frame - Arena.current.rewind(mark) - - # Accessing the view should fail because the Owner is dead - expect { - view.read_at(0) - }.to raise_error(/Memory Error: Dangling Pointer/) - end - - it "allows RVO via promote" do - mark = Arena.current.mark - - boxed_val = Value.box_number(1) - obj = FluxArray.new(nil, [boxed_val]) - - # Save 'obj' from the upcoming purge - survivors = Arena.current.promote(obj) - - # Rewind - Arena.current.rewind(mark) - - survivors.each { |s| Arena.current.register(s) } - - # It should still be alive! - expect(obj[0]).to eq(boxed_val) - end - end end RSpec.describe "VM Type Formatting" do - describe FluxArray do context "when it is a basic Object Array" do # VM Workflow: process_new_list usually inits with empty data - let(:arr) { FluxArray.new(10, [], type: :obj) } + let(:arr) { FluxArray.new(10, nil, type: :obj) } before do # VM Workflow: process_append or process_set_index fills it @@ -314,31 +250,3 @@ end end -RSpec.describe "Hypothesis Verification: Mixed-Key Recursion" do - it "successfully serializes the specific nested structure of Constant 4" do - # 1. SETUP: The exact structure retrieved from your debug output - # Outer key "chunks" is a String. - # Inner keys :name, :code, :constants are Symbols. - # Deepest key "number" is a String. - failing_structure = { - "chunks" => { - :name => "test", - :code => [ - [:LOADK, "R2", "K0"], - [:RETURN, "R2"], - [:JMP, 3], - [:RETURN, "R0"] - ], - :constants => [ - { "number" => nil } - ] - } - } - - # 2. ACT & ASSERT: Attempt to pack it - # If this fails with the same error, the structure/mixing of keys is the root cause. - # If this PASSES, I am wrong, and the issue is not the structure itself. - expect { failing_structure.to_msgpack }.not_to raise_error - end -end - diff --git a/spec/vm_spec.rb b/spec/vm_spec.rb index 4a73cb1f..88b48fdd 100644 --- a/spec/vm_spec.rb +++ b/spec/vm_spec.rb @@ -1112,19 +1112,9 @@ def make_chunk(name, instructions, constants=[]) context "Heap Object Survival (RVO)" do it "poison!s local objects that are NOT returned (Garbage Collection verification)" do - # 1. Setup: Create a list, but don't return it. Return something else (or nothing). - # We simulate a function that makes a list then discards it. - - # Since we need to inspect the "dead" object, we can't easily do it via VM return. - # We have to inspect the Arena directly after execution. - - # Code: - # R0 = NEW_LIST(0) - # RETURN R0 - chunk = make_chunk("LocalDeath", [ - [:NEW_LIST, 0, 0], # R0 = [] (This will be returned and survive) - [:NEW_LIST, 1, 0], # R1 = [] (This is waste, should die) + [:NEW_LIST, "R0", nil], # R0 = [] (This will be returned and survive) + [:NEW_LIST, "R1", nil], # R1 = [] (This is waste, should die) [:RETURN, "R0"] ]) @@ -1134,26 +1124,14 @@ def make_chunk(name, instructions, constants=[]) # The Survivor survivor = Value.as_obj(result_ref) expect(survivor).to be_a(FluxArray) - expect(survivor.is_alive?).to be(true) # The returned object must be alive - # The Victim - # We have to look inside the Arena's live_objects to ensure R1 is GONE. - # Or, if we held a reference (which we can't easily in the VM flow), check poison. - # Ideally, the Arena allocation count should only reflect the survivor. - - # Current Mark should be 1 (The survivor), not 2. - expect(Arena.current.mark).to eq(1) + expect(Arena.current.mark).to eq(survivor.address + 1) end it "successfully returns a HEAP LIST preventing Use-After-Free" do - # This explicitly tests pop_and_return logic: - # 1. promote(list) - # 2. rewind(stack) -> list is NOT in stack, so it isn't poisoned - # 3. register(list) -> list is back on top - chunk = make_chunk("HeapListReturn", [ - [:NEW_LIST, 0, 0], # R0 = [] - [:RETURN, "R0"] # Return the list + [:NEW_LIST, "R0", nil], # R0 = [] + [:RETURN, "R0"] # Return the list ]) vm = VM.new @@ -1165,16 +1143,10 @@ def make_chunk(name, instructions, constants=[]) end it "preserves data inside the returned HEAP LIST" do - # Code: - # R0 = [] - # R1 = "Hello" - # APPEND(R0, R1) - # RETURN R0 - chunk = make_chunk("ListContent", [ - [:NEW_LIST, "R0", nil], # R0 = [] + [:NEW_LIST, "R0", nil], # R0 = [] [:LOADK, "R1", "K0"], # R1 = "Hello" (Const index 0) - [:APPEND, "R0", "R1"],# R0 << R1 + [:APPEND, "R0", "R1"], # R0 << R1 [:RETURN, "R0"] ], ["Hello"]) @@ -1192,72 +1164,6 @@ def make_chunk(name, instructions, constants=[]) end end - context "Nested Stack Frames" do - it "promotes an object correctly across stack frames" do - # 1. Start State - Arena.reset! - frame_start_mark = Arena.current.mark # 0 - - # 2. Allocate "Local" Object in Frame - local_list = FluxArray.new(nil, []) - - expect(local_list.is_alive?).to be(true) - expect(Arena.current.mark).to eq(1) # [local_list] - - # 3. Simulate pop_and_return(local_list) - # We manually invoke the Arena logic that VM uses - - # A. PROMOTE - survivors = Arena.current.promote(local_list) - expect(survivors).to be_a(Array) - expect(survivors).to include(local_list) - - # At this exact moment, list is removed from allocations, so mark drops - expect(Arena.current.mark).to eq(0) - - # B. REWIND (POISON) - # Rewind to where the frame started. - Arena.current.rewind(frame_start_mark) - # Since list was promoted (removed), rewind catches nothing. - expect(local_list.is_alive?).to be(true) - - # C. REGISTER (Push back to caller scope) - survivors.each { |s| Arena.current.register(s) } - - expect(Arena.current.mark).to eq(1) - expect(local_list.is_alive?).to be(true) - end - - it "poisons objects that were NOT promoted during a frame pop" do - Arena.reset! - frame_start_mark = Arena.current.mark - - # 1. Allocate Survivor - survivor_list = FluxArray.new(nil, []) - - # 2. Allocate Victim (Local garbage) - victim_list = FluxArray.new(nil, []) - - expect(Arena.current.mark).to eq(2) - - # 3. Execute VM Logic: pop_and_return(survivor) - - # A. Promote Survivor - Arena.current.promote(survivor_list) # Removes survivor from list - - # B. Rewind (Should catch Victim) - Arena.current.rewind(frame_start_mark) - - # C. Register Survivor - Arena.current.register(survivor_list) - - # Assertions - expect(survivor_list.is_alive?).to be(true) - expect(victim_list.is_alive?).to be(false) - expect(Arena.current.mark).to eq(1) # Only survivor remains - end - end - context "Edge Cases" do it "handles returning nil/constants (Primitives) without crashing" do # Primitives usually don't need promotion logic, but the method shouldn't crash. @@ -1337,7 +1243,6 @@ def run(source) #end end - # THIS IS LIKELY WHERE YOUR CURRENT CODE FAILS describe "Level 3: Deep Containers (Nested Heap Objects)" do it "safely returns a List containing Heap Strings" do source = <<~FLUX @@ -1741,23 +1646,12 @@ def run(source) RETURN loop_waste(100); FLUX - # 1. Run the code + start_cursor = Arena.current.cursor run(source) + end_cursor = Arena.current.cursor - # 2. Check Memory - # The main chunk returns "Finished". - # The 100 "I am waste memory" strings should be GONE. - # The Arena should effectively be empty (or close to start_mark). - - # Current Allocations should be: - # 1. The Main Script Frame artifacts - # 2. The Final Result "Finished" - # (Depending on implementation details, maybe 1 or 2 other things, but NOT 100). - - live_count = Arena.current.instance_variable_get(:@allocations).count { |obj| obj.is_alive? } - - # We expect < 10 objects. If TCO failed to rewind, it would be > 100. - expect(live_count).to be < 10 + usage = end_cursor - start_cursor + expect(usage).to be < 10 end end diff --git a/src/arena.rb b/src/arena.rb index 2dba3aab..5f7cb67c 100644 --- a/src/arena.rb +++ b/src/arena.rb @@ -1,3 +1,8 @@ +# This class mocks a true Arena Memory manager +# It will eventually implement pages instead of OOM quickly +# It performs O(1) deletion and survival +# BUT!!! It cheats using Ruby's HEAP (for now). + # Static Registery # * Constants # * Struct Defs & Function Defs (named functions) @@ -7,127 +12,111 @@ # Regular Registry # * Everything else inside a function class Arena - # Define the mask locally or access it via Value::OBJ_PAYLOAD_MASK if required/available - # 48 bits = 0xFFFFFFFFFFFF - ID_MASK = 0xFFFFFFFFFFFF - - # This prevents Static (Compile Time) and Heap (Runtime) collisions. - @@next_id = 1 - def self.allocate_id - id = @@next_id - @@next_id = (@@next_id + 1) & ID_MASK - id - end + # 1MB Heap for v0.1 (Pre-allocated nil pointers) + SIZE = 128 * 1024 + STATIC_BIT = 1 << 47 - @@static_registry = {} + # Allow easy access to the singleton + def self.current; @instance ||= new; end + def self.reset!; @instance = new; end + + # Static/Const Registry (Maps ID -> Object) + @@static_registry = {} unless defined?(@@static_registry) def self.register_static(obj) - id = obj.flux_id & ID_MASK + id = @@static_registry.size @@static_registry[id] = obj - end - def self.current; @instance ||= new; end + # Return a "Virtual Address" with the High Bit set + # e.g. ID 5 becomes 0x800000000005 + id | STATIC_BIT + end - # Optional: Reset the arena (e.g. between tests) - def self.reset!; @instance = new; end + attr_reader :memory, :cursor def initialize - @allocations = [] - @live_objects = {} + @memory = Array.new(SIZE) + @cursor = 0 end - # 1. Register a new Heap Object - def register(obj) - @allocations << obj - id = obj.flux_id & ID_MASK - @live_objects[id] = obj + def mark + @cursor end - # 2. Get the current "Stack Pointer" - def mark - @allocations.size + # Alias for compatibility if you have code calling the old name + def find_object_by_address(addr) + self[addr] end - # 3. Rewind to a previous mark (Poisoning) - # Currently an O(N) operation. - # After implementing pages, this will be O(1). - def rewind(start_index) - dead_objects = @allocations[start_index..-1] || [] - @allocations = @allocations[0...start_index] - - dead_objects.each do |obj| - obj.poison! - id = obj.flux_id & ID_MASK - @live_objects.delete(id) + # O(1) Allocation + def register(obj) + if @cursor >= SIZE + raise "VM Out of Memory: Arena Full" end + + addr = @cursor + @memory[addr] = obj + @cursor += 1 + + return addr + end + + # O(1) Deletion + def rewind(saved_cursor) + @cursor = saved_cursor + # We do NOT clear the array. The garbage stays there until overwritten. + # This is standard low-level behavior. end - def find_object_by_address(address) - # The 'address' is the object_id we boxed earlier - @live_objects[address] || @@static_registry[address] + def [](addr) + # Check if the High Bit is set + if (addr & STATIC_BIT) != 0 + # It's Static! Strip the bit to get the array index + idx = addr ^ STATIC_BIT + @@static_registry[idx] + else + return nil if addr >= @cursor + @memory[addr] + end end - # Note: promote also needs to be updated to delete from @live_objects - # 4. Save an object (and descendents resursively) from the upcoming rewind - # (RVO) - def promote(obj_or_box) - survivors = [] - - # Queue for Breadth-First Search (Graph Traversal) - queue = [obj_or_box] - - # Keep track of visited IDs to handle cyclic references - visited = {} - - while queue.any? - curr = queue.shift - - # 1. Resolve Boxed Pointer -> Real Object - target_obj = curr - if curr.is_a?(Integer) - # Only unbox if it looks like an object pointer - if defined?(Value) && Value.get_tag(curr) == Value::TAG_OBJ - target_obj = Value.as_obj(curr) - else - next # Primitive (Number/Bool/Byte) - no need to promote - end - end - - # Skip if not a managed object or already processed - next unless target_obj.is_a?(FluxObject) - next if visited[target_obj.object_id] - - # 2. Check if currently managed by Arena - # If it's not in @allocations, it's either Static or already promoted. - if @allocations.include?(target_obj) - # REMOVE from the death zone (Stash) - @allocations.delete(target_obj) - - # Add to survivors list - survivors << target_obj - visited[target_obj.object_id] = true - - # 3. Enqueue Children (Recursion) - case target_obj - when FluxArray - (0...target_obj.size).each do |i| - queue << target_obj[i] - end - when FluxHash - # Promote values (keys are usually primitives/static strings) - target_obj.data.values.each do |val| - queue << val - end - when FluxClosure - # Promote captured variables - target_obj.captures.each do |val| - queue << val - end - end - end + # The Magic "Lift and Place" Promoter - save an object from DEATH + def promote(val_or_obj, to_mark) + # 1. Resolve: Is this a Heap Object or a Primitive? + obj_to_check = nil + is_boxed = false + + if val_or_obj.is_a?(FluxObject) + # It is already an unboxed object + obj_to_check = val_or_obj + elsif defined?(Value) && Value.get_tag(val_or_obj) == Value::TAG_OBJ + # It is a Boxed Pointer -> Unbox it + obj_to_check = Value.as_obj(val_or_obj) + is_boxed = true + else + # It is a Primitive (Float/Integer) -> Safe! + # Primitives don't live in the Arena, so they survive rewind automatically. + return val_or_obj end - return survivors + # 2. Optimization: If already safe, do nothing + return val_or_obj if obj_to_check.address < 0 || obj_to_check.address < to_mark || (obj_to_check.address & STATIC_BIT) != 0 + + # 2. Lift (Recursive Copy to Ruby Heap) + # This creates a graph of objects that don't have addresses yet + floating_survivor = obj_to_check.deep_copy + + # 3. Rewind (Clear the stack) + rewind(to_mark) + + # 4. Place (Recursive Register to Arena) + # This writes the survivor into the safe zone we just cleared + floating_survivor.register_recursively! + + return floating_survivor end -end + def inspect + "Arena{}" + end +end diff --git a/src/memory_visualizer.rb b/src/memory_visualizer.rb index 08e086dd..e03cb87a 100644 --- a/src/memory_visualizer.rb +++ b/src/memory_visualizer.rb @@ -14,21 +14,29 @@ def generate_mermaid lines << " direction TB" # We need a map of object_id -> node_id to draw arrows later - obj_map = {} + addr_map = {} - # Access private allocations for visualization - allocations = @arena.instance_variable_get(:@allocations) + statics = Arena.class_variable_get(:@@static_registry) + statics.each do |addr, obj| + node_id = "STATIC_#{addr.abs}" + addr_map[addr] = node_id - allocations.each_with_index do |obj, idx| - node_id = "HEAP_#{obj.object_id}" - obj_map[obj.object_id] = node_id + content = safe_inspect(obj) + lines << " #{node_id}{{ \"#{addr}: #{content}\" }}" + end + + memory = @arena.memory + cursor = @arena.cursor - # Style differently if dead - style = obj.is_alive ? "" : "style #{node_id} fill:#ffcccc,stroke:#ff0000" + (0...cursor).each do |addr| + obj = memory[addr] + next if obj.nil? # Should rarely happen in bump pointer unless explicitly nilled + + node_id = "HEAP_#{addr}" + addr_map[addr] = node_id content = safe_inspect(obj) - lines << " #{node_id}(\"#{idx}: #{content}\")" - lines << " #{style}" if !style.empty? + lines << " #{node_id}(\"#{addr}: #{content}\")" end lines << " end" @@ -50,17 +58,8 @@ def generate_mermaid # === CASE A: STACK DATA (Numbers, Bools, Bytes, Nils) === if tag != Value::TAG_OBJ - # Unbox to native (Int/Float/Bool) native_val = Formatter.to_native(val_boxed) - - # Format nicely (e.g. 0xAF for bytes) - display = if tag == Value::TAG_BYTE - "Byte(#{native_val})" - else - native_val.inspect - end - - # RENDER AS RECTANGLE [ ] + display = (tag == Value::TAG_BYTE) ? "Byte(#{native_val})" : native_val.inspect lines << " #{reg_node}[\"R#{r_idx}: #{display}\"]" # === CASE B: HEAP POINTERS (Objects, Strings, Arrays) === @@ -68,28 +67,28 @@ def generate_mermaid # It is a pointer. It gets a CIRCLE node. lines << " #{reg_node}((\"R#{r_idx}\"))" - # Now, where does it point? - # We need the actual Ruby object the pointer represents. - actual_obj = Value.as_obj(val_boxed) - - # 1. Is it in the Arena? (Dynamic Heap Object) - if obj_map.key?(actual_obj.object_id) - target_id = obj_map[actual_obj.object_id] - # Solid Arrow for Owners, Dotted for Views - arrow = (actual_obj.is_a?(MemorySlice)) ? "-.->" : "-->" - lines << " #{reg_node} #{arrow} #{target_id}" - - # 2. Is it a Constant/Static? (Not in Arena Map) - else - # Create a "floating" node for Constants so we see them clearly - const_id = "CONST_#{actual_obj.object_id}" - - # Format the content - content = safe_inspect(actual_obj) - - # Render a special Hexagon node for Constants/Statics - lines << " #{const_id}{{ #{content} }}" - lines << " #{reg_node} -.-> #{const_id}" + # DECODE ADDRESS (Payload) + # We do not unbox to the object yet; we want the address index. + address = val_boxed & Value::OBJ_PAYLOAD_MASK + + # Handle Negative Addresses (Static Objects) + # The bitmasking might obscure negative numbers depending on implementation. + # If your Value implementation masks negatives correctly to 48-bit, + # we might need to interpret the sign bit manually or trust `Value.as_obj` to look it up. + + # Safe fallback: Get the real object to check its address + actual_obj = Value.as_obj(val_boxed) rescue nil + + if actual_obj + real_addr = actual_obj.address + if addr_map.key?(real_addr) + target_id = addr_map[real_addr] + arrow = (actual_obj.is_a?(MemorySlice)) ? "-.->" : "-->" + lines << " #{reg_node} #{arrow} #{target_id}" + else + # Points to something not in our range (shouldn't happen) + lines << " #{reg_node} -.-> UNKNOWN[?]" + end end end end diff --git a/src/types.rb b/src/types.rb index a9233378..35871f16 100644 --- a/src/types.rb +++ b/src/types.rb @@ -5,10 +5,8 @@ module Formatter def self.to_native(val) - # FIX: Use bitwise mask to check for NanBox tags. - # Ruby handles 2's complement logic for negative integers automatically in bitwise ops. - if val.is_a?(Integer) && defined?(Value) && (val & Value::QNAN_MASK) == Value::QNAN_MASK - return Value.unbox(val) + if val.is_a?(Integer) && (val & Value::QNAN_MASK) == Value::QNAN_MASK + return Value.unbox(val) end return val if val.is_a?(Numeric) || val.is_a?(String) || val.nil? @@ -17,7 +15,7 @@ def self.to_native(val) return val.to_s end - return Value.to_native(val) if defined?(Value) && Value.respond_to?(:to_native) + return Value.to_native(val) if Value.respond_to?(:to_native) val end @@ -26,23 +24,46 @@ def self.to_native(val) ### # TYPES ## - class FluxObject - attr_reader :is_alive, :flux_id + attr_reader :address + attr_accessor :is_alive def initialize(register: true) @is_alive = true @is_frozen = false - @flux_id = Arena.allocate_id - if register == :static - Arena.register_static(self) + @address = Arena.register_static(self) elsif register - Arena.current.register(self) + # Normal allocation: Ask Arena for a slot + @address = Arena.current.register(self) + else + # Floating state (during deep_copy) + @address = nil end end + # Base implementation (Override in children) + def deep_copy + raise "Critical: #{self.class} did not implement deep_copy!" + end + + # The "Place" phase logic + def register_recursively! + # If we are already registered (visited in a cycle), return + return if @address + + # 1. Register Self + @address = Arena.current.register(self) + + # 2. Recurse (Override in children to register their contents) + on_register_children + end + + def on_register_children + # Default: Do nothing (for Int64, etc) + end + def ==(other) equal?(other) end @@ -108,6 +129,11 @@ def >=(other); @value >= other.value; end def !=(other); @value != other.value; end def ==(other); other.is_a?(FluxByte) && @value == other.value; end + def deep_copy + # Primitives are immutable in value, but the wrapper is a new object + FluxByte.new(@value, register: false) + end + def to_s; "0x#{@value.to_s(16).upcase.rjust(2, '0')}"; end def inspect; to_s; end end @@ -125,6 +151,11 @@ def +(other) FluxInt64.new(@value + other.value) end + def deep_copy + # Primitives are immutable in value, but the wrapper is a new object + FluxInt64.new(@value, register: false) + end + def to_s; "#{@value}_i64"; end def inspect; "#{@value}_i64"; end end @@ -142,6 +173,55 @@ def initialize(chunk, captures, register: true) @captures = captures # Array of Boxed Values end + def deep_copy + # 1. Deep Copy the Captures + # We must iterate through the captured variables. If any are Heap Objects, + # we must lift (deep_copy) them too. + new_captures = @captures.map do |boxed_val| + tag = Value.get_tag(boxed_val) + + if tag == Value::TAG_OBJ + # It is a pointer. Resolve to the object and deep_copy it. + # This returns a Floating Object (address = nil). + obj = Value.as_obj(boxed_val) + obj.deep_copy + else + # It is a Primitive (Int, Bool, etc). Copy the value as-is. + boxed_val + end + end + + # 2. Return Floating Closure + # Note: @function is usually a Static Object (FluxFunction), so we + # don't need to deep_copy it (it lives in the negative address space). + FluxClosure.new(@chunk, new_captures, register: false) + end + + def on_register_children + # The 'Place' phase. + # Our @captures currently contains a mix of: + # A) Floating FluxObjects (from the deep_copy above) + # B) Primitive Values (integers, etc.) + + @captures.map! do |item| + if item.is_a?(FluxObject) + # 1. Recurse: Give this floating child an address + item.register_recursively! + + # 2. Box: Now that it has an address, turn it back into a value pointer + Value.box_obj(item) + elsif item.is_a?(Integer) && Value.get_tag(item) == Value::TAG_OBJ + payload = item & Value::OBJ_PAYLOAD_MASK + if (payload & Arena::STATIC_BIT) == 0 + raise "Memory Error: Dangling Pointer detected in FluxArray during placement!" + end + else + # Already a primitive/boxed value + item + end + end + end + def to_s; "FN<#{@chunk.name}>"; end def inspect; to_s; end end @@ -152,37 +232,35 @@ class FluxArray < FluxObject # TYPE SCHEMA: # :obj -> Standard Array of Ruby native values - # :nanbox -> Packed NanBox String (8 bytes per item - can store Int52, Float64, Pointers to strings, etc) + # :nanbox -> Packed NanBox String (8 bytes per item - can store Int53, Float64, Pointers to strings, etc) # :int64 -> Packed Binary String (8 bytes per item) # :byte -> Packed Binary String (1 byte per item) def initialize(max_size, initial_data, register: true, type: :obj, struct_type: nil) - super(register: register) # Register with Arena - @struct_type = struct_type @max_size = max_size @data = initial_data @type = type - if @type == :obj - @data = initial_data || [] - # Ensure size matches max_size if strict - if @max_size && @data.size < @max_size - @data.fill(nil, @data.size...@max_size) - end + if initial_data + @data = initial_data else - # BINARY MODE: @data is a Byte String - # Initialize string of NULL bytes - bytes_per_item = (@type == :byte) ? 1 : 8 - size = max_size || 0 - - # 1. Allocate Binary Blob - @data = ("\x00" * (size * bytes_per_item)).force_encoding("BINARY") - - # 2. Fill if data provided - if initial_data && !initial_data.empty? - initial_data.each_with_index { |v, i| self[i] = v } + # Default Empty Allocation + if type == :obj + @data = Array.new(max_size || 0, Value.box_nil) + else + element_size = (type == :byte) ? 1 : 8 + @data = "\x00" * ((max_size || 0) * element_size) end end + + if type == :obj && !@data.is_a?(Array) + raise "Runtime Error: FluxArray :obj must use a Ruby Array storage." + end + if type != :obj && !@data.is_a?(String) + raise "Runtime Error: FluxArray binary type must use a Ruby String storage. Got #{@data.class}" + end + + super(register: register) # Register with Arena end # Need to allow dynamic resize. @@ -251,11 +329,71 @@ def []=(idx, val_boxed) end end + def truncate!(new_limit) + return if !@max_size.nil? && new_limit >= @max_size + + @max_size = new_limit + + if @type == :obj + @data = @data[0...new_limit] + else + element_size = (@type == :byte) ? 1 : 8 + byte_limit = new_limit * element_size + @data = @data.byteslice(0, byte_limit) + end + end + + # --- SURVIVOR INTERFACE --- def deep_copy check_alive! - FluxArray.new(@max_size, @data.dup) + + new_data = nil + + if @type == :obj + # Iterate the existing Ruby Array and lift safe objects. + new_data = @data.map do |boxed_val| + tag = Value.get_tag(boxed_val) + if tag == Value::TAG_OBJ + payload = boxed_val & Value::OBJ_PAYLOAD_MASK + + # Optimization: Don't copy Globals/Statics + if (payload & Arena::STATIC_BIT) != 0 + boxed_val + else + # Lift the Child (Returns Floating Object) + Value.as_obj(boxed_val).deep_copy + end + else + # Primitive (Int/Bool/Nil) - Copy as-is + boxed_val + end + end + else + # BINARY MODE: Simple String Duplication + # We trust @data is already a packed String. + new_data = @data.dup + end + + # Pass the prepared data directly to initialize + FluxArray.new(@max_size, new_data, register: false, type: @type, struct_type: @struct_type) end + # RECURSIVE REGISTER ("Place") + def on_register_children + return unless @type == :obj + + @data.map! do |item| + if item.is_a?(FluxObject) + item.register_recursively! + Value.box_obj(item) + else + item + end + end + end + + # --- END SURVIVOR INTERFACE --- + def size; check_alive!; @type == :obj ? @data.size : (@data.size / ((@type == :byte) ? 1 : 8)); end def each(&block); check_alive!; @data.each(&block); end @@ -404,8 +542,8 @@ class FluxString < FluxObject attr_reader :data def initialize(val, register: false) - super(register: register) @data = val.to_s + super(register: register) end def +(other) @@ -414,6 +552,11 @@ def +(other) FluxString.new(@data + other_str, register: true) end + def deep_copy + # Duplicate the string so the new object owns its own memory + FluxString.new(@data.dup, register: false) + end + def read_at(index) @data[index] end @@ -460,6 +603,17 @@ def deref @container end + # --- SURVIVOR INTERFACE --- + + def deep_copy + # TODO: This *MUST* be a compiler error + raise "Memory Error: Dangling Pointer - Cannot return a View of local memory" + end + + def on_register_children + # No-op, slices are *BORROWED* memory and cannot be returned. + end + # --- THE UNIFIED INTERFACE --- def read_at(index) diff --git a/src/value.rb b/src/value.rb index c0e92896..d0d1be0b 100644 --- a/src/value.rb +++ b/src/value.rb @@ -75,10 +75,11 @@ def self.box_nil def self.box_obj(obj) raise "NaNBox Error: Expected FluxObject" unless obj.is_a?(FluxObject) + if obj.address.nil? + raise "Memory Error: Attempted to box a 'Floating' object. It must be registered in the Arena first." + end - # We use the object_id as the "Pointer Address" - # We ensure it fits in 48 bits (Ruby object_ids usually do) - address = obj.flux_id & OBJ_PAYLOAD_MASK + address = obj.address & OBJ_PAYLOAD_MASK QNAN_MASK | (ID_OBJ << TAG_SHIFT) | address end @@ -153,9 +154,9 @@ def self.get_tag(val) case tag_id when ID_BYTE then return TAG_BYTE - when ID_OBJ then return TAG_OBJ + when ID_OBJ then return TAG_OBJ when ID_BOOL then return TAG_BOOL - when ID_NIL then return TAG_NIL + when ID_NIL then return TAG_NIL end end @@ -165,9 +166,9 @@ def self.get_tag(val) def self.unbox(val) case get_tag(val) when TAG_NUMBER then as_number(val) - when TAG_BYTE then as_byte(val) - when TAG_BOOL then as_bool(val) - when TAG_NIL then nil + when TAG_BYTE then as_byte(val) + when TAG_BOOL then as_bool(val) + when TAG_NIL then nil when TAG_OBJ obj = as_obj(val) obj.is_a?(FluxString) ? obj.to_s : obj diff --git a/src/vm.rb b/src/vm.rb index 3b10790f..164832af 100644 --- a/src/vm.rb +++ b/src/vm.rb @@ -267,7 +267,7 @@ def process_new_list(target_reg, args, frame) type = :nanbox size = args.first end - Value.box_obj(FluxArray.new(size, [], type: type)) + Value.box_obj(FluxArray.new(size, nil, type: type)) end def process_append(target_reg, args, frame) @@ -354,12 +354,15 @@ def process_array_cast(val_boxed, type_name, tag) if list_obj.size > limit raise "Runtime Error: Array too large for fixed size #{limit}" end - new_arr = FluxArray.new(limit, list_obj.data, type: storage_type) - return Value.box_obj(new_arr) elsif constraint == "*" - new_arr = FluxArray.new(list_obj.size, list_obj.data, type: storage_type) - return Value.box_obj(new_arr) + limit = list_obj.size end + + new_arr = list_obj.deep_copy + new_arr.truncate!(limit) + new_arr.register_recursively! + + return Value.box_obj(new_arr) end def process_def_global(target_reg, args, frame) @@ -630,14 +633,14 @@ def process_print(target_reg, args, frame) def process_return(target_reg, args, frame) return_val = args[0] - pop_and_return(return_val) + survivor_val = pop_and_return(return_val) if @frames.empty? # This is the final program result. - throw EXIT_SIGNAL, return_val + throw EXIT_SIGNAL, survivor_val end - return ReturnSignal.new(return_val) + return ReturnSignal.new(survivor_val) end def process_jmp(target_reg, args, frame) @@ -887,13 +890,8 @@ def process_mem_copy(target_reg, args, frame) nil end - def process_tail_call(target_reg, args, frame) - func_name = args.shift - arity = args.shift - - # 2. Resolve the Function Object + def get_func_and_captures(func_name) boxed_func = @globals[func_name] - func_obj = if boxed_func.is_a?(Chunk) boxed_func elsif boxed_func @@ -908,85 +906,70 @@ def process_tail_call(target_reg, args, frame) # 3. Unwrap Closure vs Chunk if func_obj.is_a?(FluxClosure) - next_chunk = func_obj.chunk - captures = func_obj.captures + return [func_obj.chunk, func_obj.captures] else - next_chunk = func_obj - captures = [] + return [func_obj, []] end + end - # ========================================================= - # 4. GARBAGE COLLECTION (Arena Rewind) - # ========================================================= - # We are about to wipe the memory of the current frame. - # We must ensure the things we need for the NEXT frame survive. - - # What must survive? - # A. The Arguments for the next function - # B. The Captures for the next function - # C. The Closure Object itself (if it was created in this scope) - roots = args + captures - roots << func_obj if func_obj.is_a?(FluxClosure) - - # A. Promote: Move roots to safety (Caller's heap space) - survivors = [] - roots.each do |root| - # promote returns the survivor (or nil if it wasn't in the arena) - s = Arena.current.promote(root) - survivors << s if s - end - survivors.flatten! - - # B. Rewind: Kill all local variables allocated in this frame - Arena.current.rewind(frame.arena_mark) - - # C. Register: Add survivors back to the live set - # This ensures the new function doesn't overwrite our args - # when it starts allocating its own objects. - survivors.each { |s| Arena.current.register(s) } - - # ========================================================= - # 5. MUTATE THE CURRENT FRAME (TCO) - # ========================================================= - # Instead of pushing a new frame, we repurpose the current one. - - # A. Overwrite Arguments - # We write the new argument values into the registers starting at 0. - # Since 'args' contains values already read from the old registers, - # we don't have to worry about overwriting a source register before reading it. - args.each_with_index do |arg_val, i| + def process_tail_call(target_reg, args, frame) + func_name = args.shift + arity = args.shift + next_chunk, captures = get_func_and_captures(func_name) + + # GARBAGE COLLECTION (Arena Rewind) + mark = frame.arena_mark + + lifted_args = args.map { |val| lift_if_unsafe(val, mark) } + lifted_captures = captures.map { |val| lift_if_unsafe(val, mark) } + + Arena.current.rewind(mark) + + final_args = lifted_args.map { |val| place_safe(val) } + final_captures = lifted_captures.map { |val| place_safe(val) } + + # MUTATE THE CURRENT FRAME (TCO) + final_args.each_with_index do |arg_val, i| frame.registers[i] = arg_val end - # B. Overwrite Captures - # If the new function is a closure, it expects captures after the args. offset = args.size - captures.each_with_index do |cap_val, i| + final_captures.each_with_index do |cap_val, i| frame.registers[offset + i] = cap_val end - # May need to do this, in Crystal / Zig, this would just be memset. - #start_wipe = offset + captures.size - #(start_wipe...256).each do |i| - # frame.registers[i] = nil - #end - - # C. Reset Stack Scratchpad - # CRITICAL: We are restarting the frame, so we wipe the scratchpad allocation. - # The new function gets the full 1KB blob to work with. - frame.stack_pointer = 0 - - # C. Swap the Code - frame.chunk = next_chunk - - # D. Reset Instruction Pointer - # The run_loop does `ins = chunk[ip]; ip += 1`. - # By setting it to 0 here, the NEXT iteration of the loop will - # read instruction 0 of the NEW chunk. - frame.ip = 0 + # TODO: May need to do memset erase. + + frame.stack_pointer = 0 # Reset Stack Scratchpad + frame.chunk = next_chunk # Swap the Code + frame.ip = 0 # Reset Instruction Pointer + nil end + # Helper: If the value points to an object in the 'Danger Zone', + # returns a Deep Copy (Floating). Otherwise returns the value as-is. + def lift_if_unsafe(boxed_val, mark) + tag = Value.get_tag(boxed_val) + if tag == Value::TAG_OBJ + obj = Value.as_obj(boxed_val) + # Check if object lives in the region we are about to delete + if obj.address && obj.address >= mark + return obj.deep_copy + end + end + return boxed_val + end + + # Helper: If the value is a Floating Object, register it and return the box. + def place_safe(val_or_obj) + if val_or_obj.is_a?(FluxObject) && val_or_obj.address.nil? + val_or_obj.register_recursively! + return Value.box_obj(val_or_obj) + end + return val_or_obj + end + # Helper to run a function and handle the signal/unwind logic safely def invoke_function(func_obj, args, caller_frame) result = execute_function(func_obj, args) @@ -1107,17 +1090,20 @@ def pop_and_return(keep_obj) frame = @frames.last return nil unless frame - # 1. RVO: Save the object from the upcoming purge - # (If it's already promoted/safe, this is a no-op) - survivors = Arena.current.promote(keep_obj) + caller_mark = frame.arena_mark # Where this frame started + + tag = Value.get_tag(keep_obj) - # 2. POISON: Kill everything allocated in this specific frame - Arena.current.rewind(frame.arena_mark) + # If Object, promote + if tag == Value::TAG_OBJ + obj = Value.as_obj(keep_obj) + survivor = Arena.current.promote(obj, caller_mark) + keep_obj = Value.box_obj(survivor) - # 3. RE-STACK: Place the survivor back on the top of the stack (Caller's scope) - # Arena.register adds it to the END of @allocations. - # Since we just rewound, the "End" is now the top of the Caller's frame - survivors.each { |s| Arena.current.register(s) } + # Else, primative, copied to stack with SRVO, rewind completely + else + Arena.current.rewind(caller_mark) + end # 4. POP: Remove the execution context @frames.pop From 5e13b999efb965fc42e5dc41af50de4c135ede87 Mon Sep 17 00:00:00 2001 From: syzygysolstice <31633243+syzygysolstice@users.noreply.github.com> Date: Wed, 17 Dec 2025 15:53:05 -0800 Subject: [PATCH 2/2] Update WALKTHROUGH.md