diff --git a/mk/rt.mk b/mk/rt.mk index 81ccbf4b7ad1b..eff16f510f9ae 100644 --- a/mk/rt.mk +++ b/mk/rt.mk @@ -63,6 +63,7 @@ RUNTIME_CXXS_$(1) := \ rt/rust_log.cpp \ rt/rust_gc_metadata.cpp \ rt/rust_util.cpp \ + rt/rust_exchange_alloc.cpp \ rt/isaac/randport.cpp \ rt/miniz.cpp \ rt/rust_kernel.cpp \ diff --git a/src/libcore/private.rs b/src/libcore/private.rs index d4cf39ad2625e..56e3325edba3e 100644 --- a/src/libcore/private.rs +++ b/src/libcore/private.rs @@ -30,6 +30,8 @@ pub mod global; pub mod finally; #[path = "private/weak_task.rs"] pub mod weak_task; +#[path = "private/exchange_alloc.rs"] +pub mod exchange_alloc; extern mod rustrt { pub unsafe fn rust_create_little_lock() -> rust_little_lock; @@ -86,6 +88,17 @@ fn test_run_in_bare_thread() { } } +#[test] +fn test_run_in_bare_thread_exchange() { + unsafe { + // Does the exchange heap work without the runtime? + let i = ~100; + do run_in_bare_thread { + assert i == ~100; + } + } +} + fn compare_and_swap(address: &mut int, oldval: int, newval: int) -> bool { unsafe { let old = rusti::atomic_cxchg(address, oldval, newval); diff --git a/src/libcore/private/exchange_alloc.rs b/src/libcore/private/exchange_alloc.rs new file mode 100644 index 0000000000000..4a3c8d59af59b --- /dev/null +++ b/src/libcore/private/exchange_alloc.rs @@ -0,0 +1,79 @@ +// Copyright 2012 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use sys::{TypeDesc, size_of}; +use libc::{c_void, size_t, uintptr_t}; +use c_malloc = libc::malloc; +use c_free = libc::free; +use managed::raw::{BoxHeaderRepr, BoxRepr}; +use cast::transmute; +use ptr::{set_memory, null}; +use intrinsic::TyDesc; + +pub unsafe fn malloc(td: *TypeDesc, size: uint) -> *c_void { + unsafe { + assert td.is_not_null(); + + let total_size = get_box_size(size, (*td).align); + let p = c_malloc(total_size as size_t); + assert p.is_not_null(); + + // FIXME #4761: Would be very nice to not memset all allocations + let p: *mut u8 = transmute(p); + set_memory(p, 0, total_size); + + // FIXME #3475: Converting between our two different tydesc types + let td: *TyDesc = transmute(td); + + let box: &mut BoxRepr = transmute(p); + box.header.ref_count = -1; // Exchange values not ref counted + box.header.type_desc = td; + box.header.prev = null(); + box.header.next = null(); + + let exchange_count = &mut *rust_get_exchange_count_ptr(); + rusti::atomic_xadd(exchange_count, 1); + + return transmute(box); + } +} + +pub unsafe fn free(ptr: *c_void) { + let exchange_count = &mut *rust_get_exchange_count_ptr(); + rusti::atomic_xsub(exchange_count, 1); + + assert ptr.is_not_null(); + c_free(ptr); +} + +fn get_box_size(body_size: uint, body_align: uint) -> uint { + let header_size = size_of::(); + // FIXME (#2699): This alignment calculation is suspicious. Is it right? + let total_size = align_to(header_size, body_align) + body_size; + return total_size; +} + +// Rounds |size| to the nearest |alignment|. Invariant: |alignment| is a power +// of two. +fn align_to(size: uint, align: uint) -> uint { + assert align != 0; + (size + align - 1) & !(align - 1) +} + +extern { + #[rust_stack] + fn rust_get_exchange_count_ptr() -> *mut int; +} + +#[abi = "rust-intrinsic"] +extern mod rusti { + fn atomic_xadd(dst: &mut int, src: int) -> int; + fn atomic_xsub(dst: &mut int, src: int) -> int; +} diff --git a/src/libcore/rt.rs b/src/libcore/rt.rs index 33d76cb3c6859..769c0b3c70703 100644 --- a/src/libcore/rt.rs +++ b/src/libcore/rt.rs @@ -15,6 +15,8 @@ use libc::{c_char, c_uchar, c_void, size_t, uintptr_t}; use managed::raw::BoxRepr; use str; use sys; +use private::exchange_alloc; +use cast::transmute; use gc::{cleanup_stack_for_failure, gc, Word}; @@ -27,13 +29,6 @@ pub const FROZEN_BIT: uint = 0x80000000; pub const FROZEN_BIT: uint = 0x8000000000000000; pub extern mod rustrt { - #[rust_stack] - unsafe fn rust_upcall_exchange_malloc(td: *c_char, size: uintptr_t) - -> *c_char; - - #[rust_stack] - unsafe fn rust_upcall_exchange_free(ptr: *c_char); - #[rust_stack] unsafe fn rust_upcall_malloc(td: *c_char, size: uintptr_t) -> *c_char; @@ -67,10 +62,11 @@ pub unsafe fn rt_fail_borrowed() { } } +// XXX: Make these signatures agree with exchange_alloc's signatures #[rt(exchange_malloc)] #[lang="exchange_malloc"] pub unsafe fn rt_exchange_malloc(td: *c_char, size: uintptr_t) -> *c_char { - return rustrt::rust_upcall_exchange_malloc(td, size); + transmute(exchange_alloc::malloc(transmute(td), transmute(size))) } // NB: Calls to free CANNOT be allowed to fail, as throwing an exception from @@ -79,7 +75,7 @@ pub unsafe fn rt_exchange_malloc(td: *c_char, size: uintptr_t) -> *c_char { #[rt(exchange_free)] #[lang="exchange_free"] pub unsafe fn rt_exchange_free(ptr: *c_char) { - rustrt::rust_upcall_exchange_free(ptr); + exchange_alloc::free(transmute(ptr)) } #[rt(malloc)] diff --git a/src/rt/rust_exchange_alloc.cpp b/src/rt/rust_exchange_alloc.cpp new file mode 100644 index 0000000000000..6c0204ca73611 --- /dev/null +++ b/src/rt/rust_exchange_alloc.cpp @@ -0,0 +1,63 @@ +// Copyright 2013 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#include "rust_exchange_alloc.h" +#include "sync/sync.h" +#include +#include +#include +#include + +uintptr_t exchange_count = 0; + +void * +rust_exchange_alloc::malloc(size_t size, bool zero) { + void *value = ::malloc(size); + assert(value); + if (zero) { + memset(value, 0, size); + } + + sync::increment(exchange_count); + + return value; +} + +void * +rust_exchange_alloc::calloc(size_t size) { + return this->malloc(size); +} + +void * +rust_exchange_alloc::realloc(void *ptr, size_t size) { + void *new_ptr = ::realloc(ptr, size); + assert(new_ptr); + return new_ptr; +} + +void +rust_exchange_alloc::free(void *ptr) { + sync::decrement(exchange_count); + ::free(ptr); +} + +extern "C" uintptr_t * +rust_get_exchange_count_ptr() { + return &exchange_count; +} + +void +rust_check_exchange_count_on_exit() { + if (exchange_count != 0) { + printf("exchange heap not empty on on exit"); + printf("%d dangling allocations", (int)exchange_count); + abort(); + } +} diff --git a/src/rt/rust_exchange_alloc.h b/src/rt/rust_exchange_alloc.h new file mode 100644 index 0000000000000..1b52929acf1b7 --- /dev/null +++ b/src/rt/rust_exchange_alloc.h @@ -0,0 +1,31 @@ +// Copyright 2013 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#ifndef RUST_EXCHANGE_ALLOC_H +#define RUST_EXCHANGE_ALLOC_H + +#include +#include + +class rust_exchange_alloc { + public: + void *malloc(size_t size, bool zero = true); + void *calloc(size_t size); + void *realloc(void *mem, size_t size); + void free(void *mem); +}; + +extern "C" uintptr_t * +rust_get_exchange_count_ptr(); + +void +rust_check_exchange_count_on_exit(); + +#endif diff --git a/src/rt/rust_kernel.cpp b/src/rt/rust_kernel.cpp index 99a6ed9494470..e0494c9300bc3 100644 --- a/src/rt/rust_kernel.cpp +++ b/src/rt/rust_kernel.cpp @@ -22,7 +22,6 @@ KLOG_LVL(this, field, log_err, __VA_ARGS__) rust_kernel::rust_kernel(rust_env *env) : - _region(env, true), _log(NULL), max_task_id(INIT_TASK_ID-1), // sync_add_and_fetch increments first rval(0), @@ -77,21 +76,21 @@ rust_kernel::fatal(char const *fmt, ...) { void * rust_kernel::malloc(size_t size, const char *tag) { - return _region.malloc(size, tag); + return exchange_alloc.malloc(size); } void * rust_kernel::calloc(size_t size, const char *tag) { - return _region.calloc(size, tag); + return exchange_alloc.calloc(size); } void * rust_kernel::realloc(void *mem, size_t size) { - return _region.realloc(mem, size); + return exchange_alloc.realloc(mem, size); } void rust_kernel::free(void *mem) { - _region.free(mem); + exchange_alloc.free(mem); } rust_sched_id @@ -217,6 +216,7 @@ rust_kernel::run() { assert(osmain_driver != NULL); osmain_driver->start_main_loop(); sched_reaper.join(); + rust_check_exchange_count_on_exit(); return rval; } diff --git a/src/rt/rust_kernel.h b/src/rt/rust_kernel.h index c3d5a5a19bb4f..11af02dace4a0 100644 --- a/src/rt/rust_kernel.h +++ b/src/rt/rust_kernel.h @@ -45,11 +45,12 @@ #include #include -#include "memory_region.h" +#include "rust_exchange_alloc.h" #include "rust_log.h" #include "rust_sched_reaper.h" #include "rust_type.h" #include "util/hash_map.h" +#include "sync/lock_and_signal.h" class rust_scheduler; class rust_sched_driver; @@ -71,7 +72,7 @@ struct exit_functions { }; class rust_kernel { - memory_region _region; + rust_exchange_alloc exchange_alloc; rust_log _log; // The next task id @@ -135,7 +136,7 @@ class rust_kernel { void *calloc(size_t size, const char *tag); void *realloc(void *mem, size_t size); void free(void *mem); - memory_region *region() { return &_region; } + rust_exchange_alloc *region() { return &exchange_alloc; } void fail(); diff --git a/src/rt/rust_sched_loop.cpp b/src/rt/rust_sched_loop.cpp index 5ddfd88d4b4a7..0d0eaaee9628e 100644 --- a/src/rt/rust_sched_loop.cpp +++ b/src/rt/rust_sched_loop.cpp @@ -260,7 +260,7 @@ rust_sched_loop::run_single_turn() { assert(!extra_c_stack); if (cached_c_stack) { - destroy_stack(kernel->region(), cached_c_stack); + destroy_exchange_stack(kernel->region(), cached_c_stack); cached_c_stack = NULL; } @@ -389,14 +389,15 @@ void rust_sched_loop::prepare_c_stack(rust_task *task) { assert(!extra_c_stack); if (!cached_c_stack && !task->have_c_stack()) { - cached_c_stack = create_stack(kernel->region(), C_STACK_SIZE); + cached_c_stack = create_exchange_stack(kernel->region(), + C_STACK_SIZE); } } void rust_sched_loop::unprepare_c_stack() { if (extra_c_stack) { - destroy_stack(kernel->region(), extra_c_stack); + destroy_exchange_stack(kernel->region(), extra_c_stack); extra_c_stack = NULL; } } diff --git a/src/rt/rust_sched_loop.h b/src/rt/rust_sched_loop.h index a5e6bc231e696..0105b83e28b45 100644 --- a/src/rt/rust_sched_loop.h +++ b/src/rt/rust_sched_loop.h @@ -135,6 +135,7 @@ struct rust_sched_loop void place_task_in_tls(rust_task *task); static rust_task *get_task_tls(); + static rust_task *try_get_task_tls(); // Called by each task when they are ready to be destroyed void release_task(rust_task *task); @@ -154,7 +155,7 @@ rust_sched_loop::get_log() { return _log; } -inline rust_task* rust_sched_loop::get_task_tls() +inline rust_task* rust_sched_loop::try_get_task_tls() { if (!tls_initialized) return NULL; @@ -165,6 +166,12 @@ inline rust_task* rust_sched_loop::get_task_tls() rust_task *task = reinterpret_cast (pthread_getspecific(task_key)); #endif + return task; +} + +inline rust_task* rust_sched_loop::get_task_tls() +{ + rust_task *task = try_get_task_tls(); assert(task && "Couldn't get the task from TLS!"); return task; } diff --git a/src/rt/rust_stack.cpp b/src/rt/rust_stack.cpp index 466399bd5b5df..3bcda8adf4024 100644 --- a/src/rt/rust_stack.cpp +++ b/src/rt/rust_stack.cpp @@ -53,6 +53,8 @@ check_stack_canary(stk_seg *stk) { assert(stk->canary == canary_value && "Somebody killed the canary"); } +// XXX: Duplication here between the local and exchange heap constructors + stk_seg * create_stack(memory_region *region, size_t sz) { size_t total_sz = sizeof(stk_seg) + sz; @@ -69,3 +71,20 @@ destroy_stack(memory_region *region, stk_seg *stk) { deregister_valgrind_stack(stk); region->free(stk); } + +stk_seg * +create_exchange_stack(rust_exchange_alloc *exchange, size_t sz) { + size_t total_sz = sizeof(stk_seg) + sz; + stk_seg *stk = (stk_seg *)exchange->malloc(total_sz, false); + memset(stk, 0, sizeof(stk_seg)); + stk->end = (uintptr_t) &stk->data[sz]; + add_stack_canary(stk); + register_valgrind_stack(stk); + return stk; +} + +void +destroy_exchange_stack(rust_exchange_alloc *exchange, stk_seg *stk) { + deregister_valgrind_stack(stk); + exchange->free(stk); +} diff --git a/src/rt/rust_stack.h b/src/rt/rust_stack.h index 91a6f8b256abf..51b884e47b1e7 100644 --- a/src/rt/rust_stack.h +++ b/src/rt/rust_stack.h @@ -12,6 +12,7 @@ #define RUST_STACK_H #include "rust_globals.h" +#include "rust_exchange_alloc.h" #include "memory_region.h" struct rust_task; @@ -37,6 +38,12 @@ create_stack(memory_region *region, size_t sz); void destroy_stack(memory_region *region, stk_seg *stk); +stk_seg * +create_exchange_stack(rust_exchange_alloc *exchange, size_t sz); + +void +destroy_exchange_stack(rust_exchange_alloc *exchange, stk_seg *stk); + // Must be called before each time a stack is reused to tell valgrind // that the stack is accessible. void diff --git a/src/rt/rust_task.cpp b/src/rt/rust_task.cpp index a9246963ca462..e51af464e488a 100644 --- a/src/rt/rust_task.cpp +++ b/src/rt/rust_task.cpp @@ -181,7 +181,7 @@ void task_start_wrapper(spawn_args *a) // free the environment (which should be a unique closure). const type_desc *td = env->td; td->drop_glue(NULL, NULL, NULL, box_body(env)); - upcall_exchange_free(env); + task->kernel->region()->free(env); } // The cleanup work needs lots of stack diff --git a/src/rt/rust_task.h b/src/rt/rust_task.h index bff4af09b32f4..cbde863fa2319 100644 --- a/src/rt/rust_task.h +++ b/src/rt/rust_task.h @@ -619,14 +619,14 @@ rust_task::record_stack_limit() { record_sp_limit(stk->data + LIMIT_OFFSET + RED_ZONE_SIZE); } -inline rust_task* rust_get_current_task() { +inline rust_task* rust_try_get_current_task() { uintptr_t sp_limit = get_sp_limit(); // FIXME (#1226) - Because of a hack in upcall_call_shim_on_c_stack this // value is sometimes inconveniently set to 0, so we can't use this // method of retreiving the task pointer and need to fall back to TLS. if (sp_limit == 0) - return rust_sched_loop::get_task_tls(); + return rust_sched_loop::try_get_task_tls(); // The stack pointer boundary is stored in a quickly-accessible location // in the TCB. From that we can calculate the address of the stack segment @@ -642,6 +642,12 @@ inline rust_task* rust_get_current_task() { return stk->task; } +inline rust_task* rust_get_current_task() { + rust_task* task = rust_try_get_current_task(); + assert(task != NULL && "no current task"); + return task; +} + // // Local Variables: // mode: C++ diff --git a/src/rt/rust_upcall.cpp b/src/rt/rust_upcall.cpp index 0e6df8e14a451..008b470fede76 100644 --- a/src/rt/rust_upcall.cpp +++ b/src/rt/rust_upcall.cpp @@ -45,6 +45,8 @@ call_upcall_on_c_stack(rust_task *task, void *args, void *fn_ptr) { task->call_on_c_stack(args, fn_ptr); } +typedef void (*CDECL stack_switch_shim)(void*); + /********************************************************************** * Switches to the C-stack and invokes |fn_ptr|, passing |args| as argument. * This is used by the C compiler to call foreign functions and by other @@ -54,13 +56,20 @@ call_upcall_on_c_stack(rust_task *task, void *args, void *fn_ptr) { */ extern "C" CDECL void upcall_call_shim_on_c_stack(void *args, void *fn_ptr) { - rust_task *task = rust_get_current_task(); - - try { - task->call_on_c_stack(args, fn_ptr); - } catch (...) { - // Logging here is not reliable - assert(false && "Foreign code threw an exception"); + rust_task *task = rust_try_get_current_task(); + + if (task) { + // We're running in task context, do a stack switch + try { + task->call_on_c_stack(args, fn_ptr); + } catch (...) { + // Logging here is not reliable + assert(false && "Foreign code threw an exception"); + } + } else { + // There's no task. Call the function and hope for the best + stack_switch_shim f = (stack_switch_shim)fn_ptr; + f(args); } } @@ -70,15 +79,22 @@ upcall_call_shim_on_c_stack(void *args, void *fn_ptr) { */ extern "C" CDECL void upcall_call_shim_on_rust_stack(void *args, void *fn_ptr) { - rust_task *task = rust_get_current_task(); - - try { - task->call_on_rust_stack(args, fn_ptr); - } catch (...) { - // We can't count on being able to unwind through arbitrary - // code. Our best option is to just fail hard. - // Logging here is not reliable - assert(false && "Rust task failed after reentering the Rust stack"); + rust_task *task = rust_try_get_current_task(); + + if (task) { + try { + task->call_on_rust_stack(args, fn_ptr); + } catch (...) { + // We can't count on being able to unwind through arbitrary + // code. Our best option is to just fail hard. + // Logging here is not reliable + assert(false + && "Rust task failed after reentering the Rust stack"); + } + } else { + // There's no task. Call the function and hope for the best + stack_switch_shim f = (stack_switch_shim)fn_ptr; + f(args); } } @@ -124,81 +140,6 @@ struct s_trace_args { size_t line; }; -/********************************************************************** - * Allocate an object in the exchange heap - */ - -struct s_exchange_malloc_args { - rust_task *task; - uintptr_t retval; - type_desc *td; - uintptr_t size; -}; - -extern "C" CDECL void -upcall_s_exchange_malloc(s_exchange_malloc_args *args) { - rust_task *task = args->task; - LOG_UPCALL_ENTRY(task); - - size_t total_size = get_box_size(args->size, args->td->align); - void *p = task->kernel->malloc(total_size, "exchange malloc"); - - rust_opaque_box *header = static_cast(p); - header->ref_count = -1; // This is not ref counted - header->td = args->td; - header->prev = 0; - header->next = 0; - - LOG(task, mem, "exchange malloced %p of size %" PRIuPTR, - header, args->size); - - args->retval = (uintptr_t)header; -} - -extern "C" CDECL uintptr_t -upcall_exchange_malloc(type_desc *td, uintptr_t size) { - rust_task *task = rust_get_current_task(); - s_exchange_malloc_args args = {task, 0, td, size}; - UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_malloc); - return args.retval; -} - -// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with -// autogenerated wrappers for upcall_exchange_malloc. Remove this when we -// fully move away away from the C upcall path. -extern "C" CDECL uintptr_t -rust_upcall_exchange_malloc(type_desc *td, uintptr_t size) { - return upcall_exchange_malloc(td, size); -} - -struct s_exchange_free_args { - rust_task *task; - void *ptr; -}; - -extern "C" CDECL void -upcall_s_exchange_free(s_exchange_free_args *args) { - rust_task *task = args->task; - LOG_UPCALL_ENTRY(task); - LOG(task, mem, "exchange freed %p", args->ptr); - task->kernel->free(args->ptr); -} - -extern "C" CDECL void -upcall_exchange_free(void *ptr) { - rust_task *task = rust_get_current_task(); - s_exchange_free_args args = {task,ptr}; - UPCALL_SWITCH_STACK(task, &args, upcall_s_exchange_free); -} - -// FIXME (#2861): Alias used by libcore/rt.rs to avoid naming conflicts with -// autogenerated wrappers for upcall_exchange_free. Remove this when we fully -// move away away from the C upcall path. -extern "C" CDECL void -rust_upcall_exchange_free(void *ptr) { - return upcall_exchange_free(ptr); -} - /********************************************************************** * Allocate an object in the task-local heap. */ diff --git a/src/rt/rustrt.def.in b/src/rt/rustrt.def.in index dcc02341e76de..9076670392a85 100644 --- a/src/rt/rustrt.def.in +++ b/src/rt/rustrt.def.in @@ -68,10 +68,6 @@ upcall_call_shim_on_rust_stack upcall_new_stack upcall_del_stack upcall_reset_stack_limit -upcall_exchange_malloc -upcall_exchange_free -rust_upcall_exchange_free -rust_upcall_exchange_malloc rust_upcall_fail rust_upcall_free rust_upcall_malloc @@ -194,4 +190,5 @@ rust_raw_thread_join_delete rust_register_exit_function rust_get_global_data_ptr rust_inc_weak_task_count -rust_dec_weak_task_count \ No newline at end of file +rust_dec_weak_task_count +rust_get_exchange_count_ptr diff --git a/src/test/run-pass/foreign-call-no-runtime.rs b/src/test/run-pass/foreign-call-no-runtime.rs new file mode 100644 index 0000000000000..865aa77536284 --- /dev/null +++ b/src/test/run-pass/foreign-call-no-runtime.rs @@ -0,0 +1,24 @@ +use core::private::run_in_bare_thread; + +extern { + pub fn rust_dbg_call(cb: *u8, + data: libc::uintptr_t) -> libc::uintptr_t; +} + +pub fn main() { + unsafe { + do run_in_bare_thread() { + unsafe { + let i = &100; + rust_dbg_call(callback, cast::transmute(i)); + } + } + } +} + +extern fn callback(data: libc::uintptr_t) { + unsafe { + let data: *int = cast::transmute(data); + assert *data == 100; + } +} \ No newline at end of file