diff --git a/vm/builtin/fiber.cpp b/vm/builtin/fiber.cpp index 7f48e5a96b..71f8893fb2 100644 --- a/vm/builtin/fiber.cpp +++ b/vm/builtin/fiber.cpp @@ -20,85 +20,6 @@ #include "on_stack.hpp" -namespace rubinius { - -#ifdef FIBER_ENABLED - -#if defined(FIBER_ASM_X8664) - -static void fiber_wrap_main(void) { - __asm__ __volatile__ ("\tmovq %r13, %rdi\n\tjmpq *%r12\n"); -} - -static inline void fiber_switch(fiber_context_t* from, fiber_context_t* to) { - __asm__ __volatile__ ( - "leaq 1f(%%rip), %%rax\n\t" - "movq %%rax, (%0)\n\t" "movq %%rsp, 8(%0)\n\t" "movq %%rbp, 16(%0)\n\t" - "movq %%rbx, 24(%0)\n\t" "movq %%r12, 32(%0)\n\t" "movq %%r13, 40(%0)\n\t" - "movq %%r14, 48(%0)\n\t" "movq %%r15, 56(%0)\n\t" - "movq 56(%1), %%r15\n\t" "movq 48(%1), %%r14\n\t" "movq 40(%1), %%r13\n\t" - "movq 32(%1), %%r12\n\t" "movq 24(%1), %%rbx\n\t" "movq 16(%1), %%rbp\n\t" - "movq 8(%1), %%rsp\n\t" "jmpq *(%1)\n" "1:\n" - : "+S" (from), "+D" (to) : - : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "memory", "cc"); -} - -static void fiber_makectx(fiber_context_t* ctx, void* func, void** stack_bottom, - int stack_size) -{ - // Get a pointer to the highest address as stack that is properly aligned - // with room for the fake return value. - uintptr_t s = ((uintptr_t)stack_bottom) + stack_size; - uintptr_t diff = s % 16; - - void** stack = (void**)(s - diff) - 1; - - *--stack = (void*)0xdeadcafedeadcafe; /* Dummy return address. */ - ctx->rip = (void*)fiber_wrap_main; - ctx->rsp = stack; - ctx->rbp = 0; - ctx->rbx = 0; - ctx->r12 = func; - ctx->r13 = 0; - ctx->r14 = 0; - ctx->r15 = 0; - -} - -#elif defined(FIBER_ASM_X8632) - -static inline void fiber_switch(fiber_context_t* from, fiber_context_t* to) { - __asm__ __volatile__ ( - "call 1f\n" "1:\tpopl %%eax\n\t" "addl $(2f-1b),%%eax\n\t" - "movl %%eax, (%0)\n\t" "movl %%esp, 4(%0)\n\t" - "movl %%ebp, 8(%0)\n\t" "movl %%ebx, 12(%0)\n\t" - "movl 12(%1), %%ebx\n\t" "movl 8(%1), %%ebp\n\t" - "movl 4(%1), %%esp\n\t" "jmp *(%1)\n" "2:\n" - : "+S" (from), "+D" (to) : : "eax", "ecx", "edx", "memory", "cc"); -} - -static void fiber_makectx(fiber_context_t* ctx, void* func, void** stack_bottom, - int stack_size) -{ - // Get a pointer to the highest address as stack that is properly aligned - // with room for the fake return value. - uintptr_t s = ((uintptr_t)stack_bottom) + stack_size; - uintptr_t diff = s % 16; - - void** stack = (void**)(s - diff) - 1; - - *--stack = (void*)0xdeadcafe; - - ctx->eip = func; - ctx->esp = stack; - ctx->ebp = 0; -} -#endif - -#endif - -} - namespace rubinius { void Fiber::init(STATE) { @@ -128,7 +49,7 @@ namespace rubinius { fib->status_ = Fiber::eRunning; fib->vm_ = state->vm(); - fib->data_ = new FiberData; + fib->data_ = new FiberData(true); state->memory()->needs_finalization(fib, (FinalizerFunction)&Fiber::finalize); @@ -178,14 +99,11 @@ namespace rubinius { } } - fib->data_->orphan(&state); - dest->run(); dest->value(&state, result); state.vm()->set_current_fiber(dest); - fiber_context_t dummy; - fiber_switch(&dummy, dest->ucontext()); + dest->data_->switch_and_orphan(&state, fib->data_); assert(0 && "fatal start_on_stack error"); #else @@ -238,24 +156,12 @@ namespace rubinius { Fiber* cur = Fiber::current(state); prev(state, cur); - if(data_->uninitialized_p()) { - FiberStack* stack = data_->allocate_stack(state); - - data_->take_stack(state); - - fiber_makectx(ucontext(), (void*)start_on_stack, - (void**)stack->address(), - stack->size()); - } else { - data_->take_stack(state); - } - cur->sleep(calling_environment); run(); state->vm()->set_current_fiber(this); - fiber_switch(cur->ucontext(), ucontext()); + data_->switch_to(state, cur->data_); // Back here when someone yields back to us! // Beware here, because the GC has probably run so GC pointers on the C++ stack @@ -303,7 +209,7 @@ namespace rubinius { run(); state->vm()->set_current_fiber(this); - fiber_switch(cur->ucontext(), ucontext()); + data_->switch_to(state, cur->data_); // Back here when someone transfers back to us! // Beware here, because the GC has probably run so GC pointers on the C++ stack @@ -352,7 +258,7 @@ namespace rubinius { dest_fib->run(); state->vm()->set_current_fiber(dest_fib); - fiber_switch(cur->ucontext(), dest_fib->ucontext()); + dest_fib->data_->switch_to(state, cur->data_); // Back here when someone yields back to us! // Beware here, because the GC has probably run so GC pointers on the C++ stack @@ -377,8 +283,8 @@ namespace rubinius { void Fiber::finalize(STATE, Fiber* fib) { #ifdef FIBER_ENABLED + fib->data_->orphan(state); delete fib->data_; - // if(fib->stack_ && !fib->root_) free(fib->stack_); #endif } @@ -388,9 +294,16 @@ namespace rubinius { mark.remember_object(obj); Fiber* fib = (Fiber*)obj; + + AddressDisplacement dis(fib->data_->data_offset(), + fib->data_->data_lower_bound(), + fib->data_->data_upper_bound()); + if(CallFrame* cf = fib->call_frame()) { - mark.gc->walk_call_frame(cf, fib->data_->data_offset()); + mark.gc->walk_call_frame(cf, &dis); } + + mark.gc->scan(fib->data_->variable_root_buffers(), false, &dis); } } diff --git a/vm/builtin/fiber.hpp b/vm/builtin/fiber.hpp index a60be778ef..8aa105f7ae 100644 --- a/vm/builtin/fiber.hpp +++ b/vm/builtin/fiber.hpp @@ -31,11 +31,16 @@ namespace rubinius { bool root_; FiberData* data_; + public: attr_accessor(starter, Object); attr_accessor(value, Array); attr_accessor(prev, Fiber); - attr_accessor(exception, Exception) + attr_accessor(exception, Exception); + + bool root_p() { + return root_; + } CallFrame* call_frame() { return top_; @@ -59,6 +64,10 @@ namespace rubinius { return vm_; } + FiberData* data() { + return data_; + } + void* stack() { return data_->stack_address(); } @@ -67,6 +76,10 @@ namespace rubinius { return data_->stack_size(); } + VariableRootBuffers& variable_root_buffers() { + return data_->variable_root_buffers(); + } + public: static void init(STATE); diff --git a/vm/fiber_data.cpp b/vm/fiber_data.cpp index bb01f3822b..cc5a50941e 100644 --- a/vm/fiber_data.cpp +++ b/vm/fiber_data.cpp @@ -1,5 +1,96 @@ #include "vm/vm.hpp" #include "vm/fiber_data.hpp" +#include "builtin/fiber.hpp" + +namespace rubinius { + +#ifdef FIBER_ENABLED + +#if defined(FIBER_ASM_X8664) + +static void fiber_wrap_main(void) { + __asm__ __volatile__ ("\tmovq %r13, %rdi\n\tjmpq *%r12\n"); +} + +static inline void fiber_switch(fiber_context_t* from, fiber_context_t* to) { + __asm__ __volatile__ ( + "leaq 1f(%%rip), %%rax\n\t" + "movq %%rax, (%0)\n\t" + "movq %%rsp, 8(%0)\n\t" + "movq %%rbp, 16(%0)\n\t" + "movq %%rbx, 24(%0)\n\t" + "movq %%r12, 32(%0)\n\t" + "movq %%r13, 40(%0)\n\t" + "movq %%r14, 48(%0)\n\t" + "movq %%r15, 56(%0)\n\t" + "movq 56(%1), %%r15\n\t" + "movq 48(%1), %%r14\n\t" + "movq 40(%1), %%r13\n\t" + "movq 32(%1), %%r12\n\t" + "movq 24(%1), %%rbx\n\t" + "movq 16(%1), %%rbp\n\t" + "movq 8(%1), %%rsp\n\t" + "jmpq *(%1)\n" "1:\n" + + : "+S" (from), "+D" (to) : + : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "memory", "cc"); +} + +static void fiber_makectx(fiber_context_t* ctx, void* func, void** stack_bottom, + int stack_size) +{ + // Get a pointer to the highest address as stack that is properly aligned + // with room for the fake return value. + uintptr_t s = ((uintptr_t)stack_bottom) + stack_size; + uintptr_t diff = s % 16; + + void** stack = (void**)(s - diff) - 1; + + *--stack = (void*)0xdeadcafedeadcafe; /* Dummy return address. */ + ctx->rip = (void*)fiber_wrap_main; + ctx->rsp = stack; + ctx->rbp = 0; + ctx->rbx = 0; + ctx->r12 = func; + ctx->r13 = 0; + ctx->r14 = 0; + ctx->r15 = 0; + +} + +#elif defined(FIBER_ASM_X8632) + +static inline void fiber_switch(fiber_context_t* from, fiber_context_t* to) { + __asm__ __volatile__ ( + "call 1f\n" "1:\tpopl %%eax\n\t" "addl $(2f-1b),%%eax\n\t" + "movl %%eax, (%0)\n\t" "movl %%esp, 4(%0)\n\t" + "movl %%ebp, 8(%0)\n\t" "movl %%ebx, 12(%0)\n\t" + "movl 12(%1), %%ebx\n\t" "movl 8(%1), %%ebp\n\t" + "movl 4(%1), %%esp\n\t" "jmp *(%1)\n" "2:\n" + : "+S" (from), "+D" (to) : : "eax", "ecx", "edx", "memory", "cc"); +} + +static void fiber_makectx(fiber_context_t* ctx, void* func, void** stack_bottom, + int stack_size) +{ + // Get a pointer to the highest address as stack that is properly aligned + // with room for the fake return value. + uintptr_t s = ((uintptr_t)stack_bottom) + stack_size; + uintptr_t diff = s % 16; + + void** stack = (void**)(s - diff) - 1; + + *--stack = (void*)0xdeadcafe; + + ctx->eip = func; + ctx->esp = stack; + ctx->ebp = 0; +} +#endif + +#endif + +} namespace rubinius { void FiberData::take_stack(STATE) { @@ -20,6 +111,9 @@ namespace rubinius { } void FiberData::copy_to_heap(STATE) { + assert(status_ != eDead); + assert(stack_); + heap_size_ = (uintptr_t)stack_->top_address() - (uintptr_t)stack_bottom(); if(heap_capacity_ < heap_size_) { // Round to nearest 1k @@ -42,7 +136,64 @@ namespace rubinius { void FiberData::orphan(STATE) { status_ = eDead; + if(!stack_) return; + stack_->orphan(state, this); stack_ = 0; } + + static void save_and_switch() { + VM* vm = VM::current(); + State state(vm); + + Fiber* fib = Fiber::current(&state); + + fib->data()->take_stack(&state); + + fiber_context_t tmp; + fiber_switch(&tmp, fib->ucontext()); + } + + void FiberData::switch_to(STATE, FiberData* from) { + if(uninitialized_p()) { + FiberStack* stack = allocate_stack(state); + + fiber_makectx(machine(), (void*)Fiber::start_on_stack, + (void**)stack->address(), + stack->size()); + } + + if(currently_on_stack_p(from)) { + fiber_context_t dest; + + fiber_makectx(&dest, (void*)save_and_switch, + (void**)state->vm()->fiber_trampoline(), + 4096); + + fiber_switch(from->machine(), &dest); + } else { + take_stack(state); + fiber_switch(from->machine(), machine()); + } + } + + void FiberData::switch_and_orphan(STATE, FiberData* from) { + fiber_context_t dummy; + + if(currently_on_stack_p(from)) { + from->orphan(state); + + fiber_context_t dest; + + fiber_makectx(&dest, (void*)save_and_switch, + (void**)state->vm()->fiber_trampoline(), + 4096); + + fiber_switch(&dummy, &dest); + } else { + from->orphan(state); + take_stack(state); + fiber_switch(&dummy, machine()); + } + } } diff --git a/vm/fiber_data.hpp b/vm/fiber_data.hpp index 4d8839e6d4..77a828eef1 100644 --- a/vm/fiber_data.hpp +++ b/vm/fiber_data.hpp @@ -4,6 +4,8 @@ #include "vm/config.h" #include "prelude.hpp" +#include "gc/variable_buffer.hpp" + namespace rubinius { #if defined(IS_X86) @@ -55,18 +57,24 @@ namespace rubinius { size_t heap_size_; size_t heap_capacity_; + VariableRootBuffers variable_root_buffers_; + public: - FiberData() - : status_(eInitial) + FiberData(bool root=false) + : status_(root ? eOnStack : eInitial) , stack_(0) , heap_(0) , heap_size_(0) , heap_capacity_(0) {} + VariableRootBuffers& variable_root_buffers() { + return variable_root_buffers_; + } + bool uninitialized_p() { - return stack_ == 0; + return status_ == eInitial && stack_ == 0; } fiber_context_t* machine() { @@ -96,12 +104,29 @@ namespace rubinius { return (intptr_t)heap_ - (intptr_t)stack_bottom(); } + intptr_t data_lower_bound() { + if(status_ != eOnHeap) return 0; + return (intptr_t)stack_->address(); + } + + intptr_t data_upper_bound() { + if(status_ != eOnHeap) return 0; + return (intptr_t)stack_->address() + stack_->size(); + } + + bool currently_on_stack_p(FiberData* cur) { + return stack_ && stack_->user() == cur; + } + FiberStack* allocate_stack(STATE); void take_stack(STATE); void copy_stack(STATE, void* addr, size_t size); void copy_to_heap(STATE); void orphan(STATE); + + void switch_to(STATE, FiberData* from); + void switch_and_orphan(STATE, FiberData* from); }; } diff --git a/vm/fiber_stack.cpp b/vm/fiber_stack.cpp index 752ecbc028..111d8cc9c0 100644 --- a/vm/fiber_stack.cpp +++ b/vm/fiber_stack.cpp @@ -34,7 +34,10 @@ namespace rubinius { } void FiberStack::orphan(STATE, FiberData* user) { - assert(user_ == user); + if(user == user_) { + user_ = 0; + } + dec_ref(); } @@ -73,4 +76,12 @@ namespace rubinius { return stack; } + + void* FiberStacks::trampoline() { + if(trampoline_ == 0) { + trampoline_ = malloc(cTrampolineSize); + } + + return trampoline_; + } } diff --git a/vm/fiber_stack.hpp b/vm/fiber_stack.hpp index 8c3ad79913..30b7303ec2 100644 --- a/vm/fiber_stack.hpp +++ b/vm/fiber_stack.hpp @@ -45,6 +45,10 @@ namespace rubinius { return user_ && refs_ > 1; } + FiberData* user() { + return user_; + } + void set_user(FiberData* d) { user_ = d; } @@ -59,11 +63,19 @@ namespace rubinius { typedef std::list Stacks; const static size_t cStackSize = 128 * 1024; const static size_t cMaxStacks = 1; + const static size_t cTrampolineSize = 4096; Stacks stacks_; + void* trampoline_; public: + FiberStacks() + : trampoline_(0) + {} + FiberStack* allocate(); + + void* trampoline(); }; } diff --git a/vm/gc/gc.cpp b/vm/gc/gc.cpp index 166360adc0..6b19984a2b 100644 --- a/vm/gc/gc.cpp +++ b/vm/gc/gc.cpp @@ -149,15 +149,16 @@ namespace rubinius { } template - T displace(T ptr, intptr_t by) { - return (T)((char*)ptr + by); + T displace(T ptr, AddressDisplacement* dis) { + if(!dis) return ptr; + return dis->displace(ptr); } /** * Walks the chain of objects accessible from the specified CallFrame. */ void GarbageCollector::walk_call_frame(CallFrame* top_call_frame, - intptr_t offset) + AddressDisplacement* offset) { CallFrame* call_frame = top_call_frame; @@ -199,7 +200,8 @@ namespace rubinius { call_frame->set_block_env((BlockEnvironment*)mark_object(env)); } - Arguments* args = call_frame->arguments; + Arguments* args = displace(call_frame->arguments, offset); + if(!call_frame->inline_method_p() && args) { args->set_recv(mark_object(args->recv())); args->set_block(mark_object(args->block())); @@ -207,7 +209,7 @@ namespace rubinius { if(Tuple* tup = args->argument_container()) { args->update_argument_container((Tuple*)mark_object(tup)); } else { - Object** ary = args->arguments(); + Object** ary = displace(args->arguments(), offset); for(uint32_t i = 0; i < args->total(); i++) { ary[i] = mark_object(ary[i]); } @@ -262,14 +264,16 @@ namespace rubinius { } } - void GarbageCollector::scan(VariableRootBuffers& buffers, bool young_only) { + void GarbageCollector::scan(VariableRootBuffers& buffers, + bool young_only, AddressDisplacement* offset) + { for(VariableRootBuffers::Iterator vi(buffers); vi.more(); vi.advance()) { - Object*** buffer = vi->buffer(); + Object*** buffer = displace(vi->buffer(), offset); for(int idx = 0; idx < vi->size(); idx++) { - Object** var = buffer[idx]; + Object** var = displace(buffer[idx], offset); Object* tmp = *var; if(tmp->reference_p() && (!young_only || tmp->young_object_p())) { diff --git a/vm/gc/gc.hpp b/vm/gc/gc.hpp index 58a9f34ac0..9dbdc8b18f 100644 --- a/vm/gc/gc.hpp +++ b/vm/gc/gc.hpp @@ -89,6 +89,28 @@ namespace rubinius { } }; + class AddressDisplacement { + intptr_t offset_; + intptr_t lower_bound_; + intptr_t upper_bound_; + + public: + AddressDisplacement(intptr_t o, intptr_t l, intptr_t u) + : offset_(o) + , lower_bound_(l) + , upper_bound_(u) + {} + + template + T displace(T ptr) { + intptr_t addr = (intptr_t)ptr; + if(addr < lower_bound_) return ptr; + if(addr >= upper_bound_) return ptr; + + return (T)((char*)ptr + offset_); + } + }; + /** * Abstract base class for the various garbage collector implementations. @@ -124,7 +146,7 @@ namespace rubinius { // Scans the specified Object for references to other Objects. void scan_object(Object* obj); void delete_object(Object* obj); - void walk_call_frame(CallFrame* top_call_frame, intptr_t offset=0); + void walk_call_frame(CallFrame* top_call_frame, AddressDisplacement* offset=0); void saw_variable_scope(CallFrame* call_frame, StackVariables* scope); /** @@ -140,7 +162,7 @@ namespace rubinius { void clean_weakrefs(bool check_forwards=false); // Scans the thread for object references void scan(ManagedThread* thr, bool young_only); - void scan(VariableRootBuffers& buffers, bool young_only); + void scan(VariableRootBuffers& buffers, bool young_only, AddressDisplacement* offset=0); void scan(RootBuffers& rb, bool young_only); VM* state(); diff --git a/vm/on_stack.hpp b/vm/on_stack.hpp index 39089fa1d7..a184cc2d9d 100644 --- a/vm/on_stack.hpp +++ b/vm/on_stack.hpp @@ -16,7 +16,7 @@ namespace rubinius { template OnStack(STATE, T& o1) - : buffer_(state->vm()->variable_root_buffers(), objects_, size) + : buffer_(state->vm()->current_root_buffers(), objects_, size) { // Ok, let me explain. // We want to get the address of o1 to put into the buffer, so that @@ -51,7 +51,7 @@ namespace rubinius { template OnStack(STATE, T1& o1, T2& o2) - : buffer_(state->vm()->variable_root_buffers(), objects_, size) + : buffer_(state->vm()->current_root_buffers(), objects_, size) { (void)static_cast(o1); (void)static_cast(o2); @@ -62,7 +62,7 @@ namespace rubinius { template OnStack(STATE, T1& o1, T2& o2, T3& o3) - : buffer_(state->vm()->variable_root_buffers(), objects_, size) + : buffer_(state->vm()->current_root_buffers(), objects_, size) { (void)static_cast(o1); (void)static_cast(o2); @@ -75,7 +75,7 @@ namespace rubinius { template OnStack(STATE, T1& o1, T2& o2, T3& o3, T4& o4) - : buffer_(state->vm()->variable_root_buffers(), objects_, size) + : buffer_(state->vm()->current_root_buffers(), objects_, size) { (void)static_cast(o1); (void)static_cast(o2); diff --git a/vm/vm.cpp b/vm/vm.cpp index ad5b0cb761..26e719d13e 100644 --- a/vm/vm.cpp +++ b/vm/vm.cpp @@ -427,6 +427,14 @@ namespace rubinius { current_fiber.set(fib); } + VariableRootBuffers& VM::current_root_buffers() { + if(current_fiber->nil_p() || current_fiber->root_p()) { + return variable_root_buffers(); + } + + return current_fiber->variable_root_buffers(); + } + GCIndependent::GCIndependent(NativeMethodEnvironment* env) : state_(env->state()) { diff --git a/vm/vm.hpp b/vm/vm.hpp index f2229765ee..e5975084d4 100644 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -265,6 +265,12 @@ namespace rubinius { return fiber_stacks_.allocate(); } + void* fiber_trampoline() { + return fiber_stacks_.trampoline(); + } + + VariableRootBuffers& current_root_buffers(); + public: static void init_stack_size();