diff --git a/hpx/runtime.hpp b/hpx/runtime.hpp index 35cab3f74b7f..7a0600de8703 100644 --- a/hpx/runtime.hpp +++ b/hpx/runtime.hpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include diff --git a/hpx/runtime/components/server/destroy_component.hpp b/hpx/runtime/components/server/destroy_component.hpp index d9aa626eec8b..75eb584b6b0e 100644 --- a/hpx/runtime/components/server/destroy_component.hpp +++ b/hpx/runtime/components/server/destroy_component.hpp @@ -14,7 +14,6 @@ #include #include #include -#include #include diff --git a/hpx/runtime/components/server/runtime_support.hpp b/hpx/runtime/components/server/runtime_support.hpp index 0e31d27b0ed0..f450041d408d 100644 --- a/hpx/runtime/components/server/runtime_support.hpp +++ b/hpx/runtime/components/server/runtime_support.hpp @@ -32,7 +32,6 @@ #include #include #include -#include #include #include #include diff --git a/hpx/runtime/components/server/wrapper_heap.hpp b/hpx/runtime/components/server/wrapper_heap.hpp index 73d98b6b97d8..986faaa19f33 100644 --- a/hpx/runtime/components/server/wrapper_heap.hpp +++ b/hpx/runtime/components/server/wrapper_heap.hpp @@ -9,18 +9,12 @@ #include #include -#include -#include #include #include #include #include #include -#include -#include - -#include -#include +#include #include #include @@ -28,380 +22,106 @@ #include #include #include +#include + +#include + +#define HPX_DEBUG_WRAPPER_HEAP 1 /////////////////////////////////////////////////////////////////////////////// namespace hpx { namespace components { namespace detail { #if HPX_DEBUG_WRAPPER_HEAP != 0 -#define HPX_WRAPPER_HEAP_INITIALIZED_MEMORY 1 +# define HPX_WRAPPER_HEAP_INITIALIZED_MEMORY 1 +#else +# define HPX_WRAPPER_HEAP_INITIALIZED_MEMORY 0 +#endif - namespace debug + /////////////////////////////////////////////////////////////////////////// + namespace one_size_heap_allocators { /////////////////////////////////////////////////////////////////////// - // Test memory area for being filled as expected - inline - bool test_fill_bytes (void *p, unsigned char c, std::size_t cnt) + // TODO: this interface should conform to the Boost.Pool allocator + // interface, to maximize code reuse and consistency - wash. + // + // simple allocator which gets the memory from the default malloc, + // but which does not reallocate the heap (it doesn't grow) + struct fixed_mallocator { - unsigned char* uc = (unsigned char*)p; - for (std::size_t i = 0; i < cnt; ++i) { - if (*uc++ != c) - return false; + static void* alloc(std::size_t size) + { + return ::malloc(size); } - return true; - } - - /////////////////////////////////////////////////////////////////////// - // Fill memory area - inline - void fill_bytes (void *p, unsigned char c, int cnt) - { - using namespace std; // some systems have memset in namespace std - memset (p, c, cnt); - } - } // namespace debug -#else -# define HPX_WRAPPER_HEAP_INITIALIZED_MEMORY 0 -#endif + static void free(void* p) + { + ::free(p); + } + static void* realloc(std::size_t &, void *) + { + // normally this should return ::realloc(p, size), but we are + // not interested in growing the allocated heaps, so we just + // return nullptr + return nullptr; + } + }; + } - /////////////////////////////////////////////////////////////////////////////// - template - class wrapper_heap + /////////////////////////////////////////////////////////////////////////// + class HPX_EXPORT wrapper_heap : public util::wrapper_heap_base { public: HPX_NON_COPYABLE(wrapper_heap); public: - typedef T value_type; - typedef Allocator allocator_type; + typedef one_size_heap_allocators::fixed_mallocator allocator_type; + typedef hpx::lcos::local::spinlock mutex_type; + typedef std::unique_lock scoped_lock; #if HPX_DEBUG_WRAPPER_HEAP != 0 - enum guard_value { + enum guard_value + { initial_value = 0xcc, // memory has been initialized freed_value = 0xdd, // memory has been freed }; #endif - typedef Mutex mutex_type; - - typedef std::unique_lock scoped_lock; - - typedef boost::aligned_storage::value> storage_type; -// storage_type data; - - enum { - heap_step = 0xFFFF, // default initial number of elements - heap_size = sizeof(storage_type) // size of one element in the heap - }; - public: - explicit wrapper_heap( - char const* class_name, -#if defined(HPX_DEBUG) - std::size_t count, -#else - std::size_t, -#endif - std::size_t step = static_cast(-1) - ) - : pool_(nullptr), first_free_(nullptr), step_(step), size_(0), free_size_(0), - base_gid_(naming::invalid_gid), - class_name_(class_name), -#if defined(HPX_DEBUG) - alloc_count_(0), free_count_(0), heap_count_(count), -#endif - heap_alloc_function_("wrapper_heap::alloc", class_name), - heap_free_function_("wrapper_heap::free", class_name) - { - util::itt::heap_internal_access hia; - - HPX_ASSERT(sizeof(storage_type) == heap_size); - - // adjust step to reasonable value - if (static_cast(-1) == step_ || step_ < heap_step) //-V104 - step_ = heap_step; //-V101 - else - step_ = ((step_ + heap_step - 1)/heap_step)*heap_step; //-V104 - - if (!init_pool()) - throw std::bad_alloc(); - } - - wrapper_heap() - : pool_(nullptr), first_free_(nullptr), - step_(heap_step), size_(0), free_size_(0), - base_gid_(naming::invalid_gid), -#if defined(HPX_DEBUG) - alloc_count_(0), free_count_(0), heap_count_(0), -#endif - heap_alloc_function_("wrapper_heap::alloc", ""), - heap_free_function_("wrapper_heap::free", "") - { - util::itt::heap_internal_access hia; HPX_UNUSED(hia); - - HPX_ASSERT(sizeof(storage_type) == heap_size); - if (!init_pool()) - throw std::bad_alloc(); - } - - ~wrapper_heap() - { - util::itt::heap_internal_access hia; HPX_UNUSED(hia); - tidy(); - } - - std::size_t size() const - { - util::itt::heap_internal_access hia; HPX_UNUSED(hia); - return size_ - free_size_; - } - std::size_t free_size() const - { - util::itt::heap_internal_access hia; HPX_UNUSED(hia); - return free_size_; - } - bool is_empty() const - { - util::itt::heap_internal_access hia; HPX_UNUSED(hia); - return nullptr == pool_; - } - bool has_allocatable_slots() const - { - util::itt::heap_internal_access hia; HPX_UNUSED(hia); - return first_free_ < pool_+size_; - } + explicit wrapper_heap(char const* class_name, std::size_t count, + std::size_t heap_size, std::size_t step); - bool alloc(T** result, std::size_t count = 1) - { - util::itt::heap_allocate heap_allocate( - heap_alloc_function_, result, count*sizeof(storage_type), - HPX_WRAPPER_HEAP_INITIALIZED_MEMORY); + wrapper_heap(); + ~wrapper_heap() override; - scoped_lock l(mtx_); + std::size_t size() const override; + std::size_t free_size() const override; - if (!ensure_pool(count)) - return false; - -#if defined(HPX_DEBUG) - alloc_count_ += count; -#endif - - value_type* p = static_cast(first_free_->address()); //-V707 - HPX_ASSERT(p != nullptr); - - first_free_ += count; - - HPX_ASSERT(free_size_ >= count); - free_size_ -= count; - -#if HPX_DEBUG_WRAPPER_HEAP != 0 - // init memory blocks - debug::fill_bytes(p, initial_value, count*sizeof(storage_type)); -#endif + bool is_empty() const; + bool has_allocatable_slots() const; - *result = p; - return true; - } + bool alloc(void** result, std::size_t count = 1) override; + void free(void *p, std::size_t count = 1) override; + bool did_alloc (void *p) const override; - void free(void *p, std::size_t count = 1) - { - util::itt::heap_free heap_free(heap_free_function_, p); - -#if HPX_DEBUG_WRAPPER_HEAP != 0 - HPX_ASSERT(did_alloc(p)); -#endif - scoped_lock l(mtx_); - -#if HPX_DEBUG_WRAPPER_HEAP != 0 - storage_type* p1 = static_cast(p); - - HPX_ASSERT(nullptr != pool_ && p1 >= pool_); - HPX_ASSERT(nullptr != pool_ && p1 + count <= pool_ + size_); - HPX_ASSERT(first_free_ == nullptr || p1 != first_free_); - HPX_ASSERT(free_size_ + count <= size_); - // make sure this has not been freed yet - HPX_ASSERT(!debug::test_fill_bytes(p1->address(), freed_value, - count*sizeof(storage_type))); - - // give memory back to pool - debug::fill_bytes(p1->address(), freed_value, sizeof(storage_type)); -#else - HPX_UNUSED(p); -#endif - -#if defined(HPX_DEBUG) - free_count_ += count; -#endif - free_size_ += count; - - // release the pool if this one was the last allocated item - test_release(l); - } - bool did_alloc (void *p) const - { - // no lock is necessary here as all involved variables are immutable - util::itt::heap_internal_access hia; HPX_UNUSED(hia); - return nullptr != pool_ && nullptr != p && pool_ <= p && p < pool_ + size_; - } - - /// \brief Get the global id of the managed_component instance - /// given by the parameter \a p. - /// - /// - /// \note The pointer given by the parameter \a p must have been - /// allocated by this instance of a \a wrapper_heap + // Get the global id of the managed_component instance given by the + // parameter p. + // + // The pointer given by the parameter p must have been allocated by + // this instance of a wrapper_heap naming::gid_type get_gid(util::unique_id_ranges& ids, void* p, - components::component_type type) - { - util::itt::heap_internal_access hia; HPX_UNUSED(hia); - - HPX_ASSERT(did_alloc(p)); - - scoped_lock l(mtx_); - value_type* addr = static_cast(pool_->address()); - - if (!base_gid_) { - naming::gid_type base_gid; - - { - // this is the first call to get_gid() for this heap - allocate - // a sufficiently large range of global ids - util::unlock_guard ul(l); - base_gid = ids.get_id(step_); - - // register the global ids and the base address of this heap - // with the AGAS - if (!applier::bind_range_local(base_gid, step_, - naming::address(hpx::get_locality(), type, addr), - sizeof(value_type))) - { - return naming::invalid_gid; - } - } - - // if some other thread has already set the base GID for this - // heap, we ignore the result - if (!base_gid_) - { - // this is the first thread succeeding in binding the new gid range - base_gid_ = base_gid; - } - else - { - // unbind the range which is not needed anymore - util::unlock_guard ul(l); - applier::unbind_range_local(base_gid, step_); - } - } - - return base_gid_ + static_cast( - static_cast(p) - addr); - } - - void set_gid(naming::gid_type const& g) - { - util::itt::heap_internal_access hia; HPX_UNUSED(hia); + components::component_type type) override; - scoped_lock l(mtx_); - base_gid_ = g; - } - - naming::address get_address() - { - util::itt::heap_internal_access hia; HPX_UNUSED(hia); - - value_type* addr = static_cast(pool_->address()); - return naming::address(get_locality(), - components::get_component_type(), - addr); - } + void set_gid(naming::gid_type const& g); protected: - bool test_release(scoped_lock& lk) - { - if (pool_ == nullptr || free_size_ < size_ || first_free_ < pool_+size_) - return false; - HPX_ASSERT(free_size_ == size_); - - // unbind in AGAS service - if (base_gid_) { - naming::gid_type base_gid = base_gid_; - base_gid_ = naming::invalid_gid; + bool test_release(scoped_lock& lk); + bool ensure_pool(std::size_t count); - util::unlock_guard ull(lk); - applier::unbind_range_local(base_gid, step_); - } - - tidy(); - return true; - } - - bool ensure_pool(std::size_t count) - { - if (nullptr == pool_) - return false; - if (first_free_ + count > pool_+size_) - return false; - return true; - } - - bool init_pool() - { - HPX_ASSERT(size_ == 0); - HPX_ASSERT(first_free_ == nullptr); - - std::size_t s = step_ * heap_size; //-V104 //-V707 - pool_ = static_cast(Allocator::alloc(s)); - if (nullptr == pool_) - return false; - - first_free_ = pool_; - size_ = s / heap_size; //-V104 - free_size_ = size_; - - LOSH_(info) //-V128 - << "wrapper_heap (" - << (!class_name_.empty() ? class_name_.c_str() : "") - << "): init_pool (" << std::hex << pool_ << ")" - << " size: " << s << "."; - - return true; - } - - void tidy() - { - if (pool_ != nullptr) { - LOSH_(debug) //-V128 - << "wrapper_heap (" - << (!class_name_.empty() ? class_name_.c_str() : "") - << ")" -#if defined(HPX_DEBUG) - << ": releasing heap: alloc count: " << alloc_count_ - << ", free count: " << free_count_ -#endif - << "."; - if (free_size_ != size_ -#if defined(HPX_DEBUG) - || alloc_count_ != free_count_ -#endif - ) - { - LOSH_(warning) //-V128 - << "wrapper_heap (" - << (!class_name_.empty() ? class_name_.c_str() : "") - << "): releasing heap (" << std::hex << pool_ << ")" - << " with " << size_-free_size_ << " allocated object(s)!"; - } - - Allocator::free(pool_); - pool_ = first_free_ = nullptr; - size_ = free_size_ = 0; - } - } + bool init_pool(); + void tidy(); - private: - storage_type* pool_; - storage_type* first_free_; + protected: + void* pool_; + void* first_free_; std::size_t step_; std::size_t size_; std::size_t free_size_; @@ -420,59 +140,35 @@ namespace hpx { namespace components { namespace detail std::size_t heap_count_; #endif + std::size_t const element_size_; // size of one element in the heap + + std::size_t heap_count() const override { return heap_count_; } + private: util::itt::heap_function heap_alloc_function_; util::itt::heap_function heap_free_function_; }; - /////////////////////////////////////////////////////////////////////////// - namespace one_size_heap_allocators - { - /////////////////////////////////////////////////////////////////////// - // TODO: this interface should conform to the Boost.Pool allocator - // interface, to maximize code reuse and consistency - wash. - // - // simple allocator which gets the memory from the default malloc, - // but which does not reallocate the heap (it doesn't grow) - struct fixed_mallocator - { - static void* alloc(std::size_t size) - { - return ::malloc(size); - } - static void free(void* p) - { - ::free(p); - } - static void* realloc(std::size_t &, void *) - { - // normally this should return ::realloc(p, size), but we are - // not interested in growing the allocated heaps, so we just - // return nullptr - return nullptr; - } - }; - } - /////////////////////////////////////////////////////////////////////////// // heap using malloc and friends - template - class fixed_wrapper_heap : - public wrapper_heap + template + class fixed_wrapper_heap : public wrapper_heap { private: - typedef - wrapper_heap - base_type; + typedef wrapper_heap base_type; public: - explicit fixed_wrapper_heap(char const* class_name = "", - std::size_t count = 0, std::size_t step = std::size_t(-1)) - : base_type(class_name, count, step) + typedef T value_type; + + explicit fixed_wrapper_heap(char const* class_name, + std::size_t count, std::size_t step, std::size_t element_size) + : base_type(class_name, count, step, element_size) {} }; }}} // namespace hpx::components::detail +#include + #undef HPX_WRAPPER_HEAP_INITIALIZED_MEMORY #endif diff --git a/hpx/runtime/components/server/wrapper_heap_list.hpp b/hpx/runtime/components/server/wrapper_heap_list.hpp index d2a856d879a3..617bf27c45d4 100644 --- a/hpx/runtime/components/server/wrapper_heap_list.hpp +++ b/hpx/runtime/components/server/wrapper_heap_list.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 1998-2013 Hartmut Kaiser +// Copyright (c) 1998-2017 Hartmut Kaiser // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -12,21 +12,34 @@ #include #include +#include + /////////////////////////////////////////////////////////////////////////////// namespace hpx { namespace components { namespace detail { /////////////////////////////////////////////////////////////////////////// // list of managed_component heaps template - class wrapper_heap_list - : public util::one_size_heap_list + class wrapper_heap_list : public util::one_size_heap_list { - typedef util::one_size_heap_list base_type; + typedef util::one_size_heap_list base_type; + typedef typename Heap::value_type value_ype; + + typedef typename std::aligned_storage< + sizeof(value_ype), std::alignment_of::value + >::type storage_type; + + enum + { + heap_step = 0xFFF, // default initial number of elements + heap_size = sizeof(storage_type) // size of one element in the heap + }; public: wrapper_heap_list(component_type type) - : base_type(get_component_type_name(type)), - type_(type) + : base_type(get_component_type_name(type), heap_step, heap_size, + (Heap*) nullptr) + , type_(type) {} /// diff --git a/hpx/util/one_size_heap_list.hpp b/hpx/util/one_size_heap_list.hpp index d00b0da30273..f207d8ce639b 100644 --- a/hpx/util/one_size_heap_list.hpp +++ b/hpx/util/one_size_heap_list.hpp @@ -9,17 +9,8 @@ #include #include -#include -#include -#include #include -#include -#include -#if defined(HPX_DEBUG) -#include -#endif -#include -#include +#include #include #include @@ -27,33 +18,54 @@ #include #include +#include + /////////////////////////////////////////////////////////////////////////////// namespace hpx { namespace util { - template - class one_size_heap_list : public one_size_heap_list_base + class HPX_EXPORT one_size_heap_list { public: - typedef Heap heap_type; - - typedef typename heap_type::allocator_type allocator_type; - typedef typename heap_type::value_type value_type; - - typedef std::list > list_type; + typedef std::list > list_type; typedef typename list_type::iterator iterator; typedef typename list_type::const_iterator const_iterator; - enum - { - heap_step = Heap::heap_step, // default grow step - heap_size = Heap::heap_size // size of the object - }; - - typedef Mutex mutex_type; + typedef lcos::local::spinlock mutex_type; typedef std::unique_lock unique_lock_type; - explicit one_size_heap_list(char const* class_name = "") + private: + template + static std::shared_ptr create_heap( + char const* name, std::size_t counter, std::size_t step, + std::size_t heap_size) + { +#if defined(HPX_DEBUG) + return std::make_shared(name, counter, step, heap_size); +#else + return std::make_shared(name, 0, step, heap_size); +#endif + } + + public: + one_size_heap_list() + : class_name_() +#if defined(HPX_DEBUG) + , alloc_count_(0) + , free_count_(0) + , heap_count_(0) + , max_alloc_count_(0) +#endif + , create_heap_(nullptr) + , heap_step_(0) + , heap_size_(0) + { + HPX_ASSERT(false); // shouldn't ever be called + } + + template + explicit one_size_heap_list(char const* class_name, + std::size_t heap_step, std::size_t heap_size, Heap* = nullptr) : class_name_(class_name) #if defined(HPX_DEBUG) , alloc_count_(0L) @@ -61,11 +73,14 @@ namespace hpx { namespace util , heap_count_(0L) , max_alloc_count_(0L) #endif - { - HPX_ASSERT(sizeof(typename heap_type::storage_type) == uint64_t(heap_size)); - } - - explicit one_size_heap_list(std::string const& class_name) + , create_heap_(&one_size_heap_list::create_heap) + , heap_step_(heap_step) + , heap_size_(heap_size) + {} + + template + explicit one_size_heap_list(std::string const& class_name, + std::size_t heap_step, std::size_t heap_size, Heap* = nullptr) : class_name_(class_name) #if defined(HPX_DEBUG) , alloc_count_(0L) @@ -73,250 +88,24 @@ namespace hpx { namespace util , heap_count_(0L) , max_alloc_count_(0L) #endif - { - HPX_ASSERT(sizeof(typename heap_type::storage_type) == uint64_t(heap_size)); - } + , create_heap_(&one_size_heap_list::create_heap) + , heap_step_(heap_step) + , heap_size_(heap_size) + {} - ~one_size_heap_list() noexcept - { -#if defined(HPX_DEBUG) - LOSH_(info) << hpx::util::format( - "%1%::~%1%: size(%2%), max_count(%3%), alloc_count(%4%), " - "free_count(%5%)", - name(), - heap_count_, - max_alloc_count_, - alloc_count_, - free_count_); - - if (alloc_count_ > free_count_) - { - LOSH_(warning) << hpx::util::format( - "%1%::~%1%: releasing with %2% allocated objects", - name(), - alloc_count_ - free_count_); - } -#endif - } + ~one_size_heap_list() noexcept; // operations - void* alloc(std::size_t count = 1) - { - unique_lock_type guard(mtx_); - - if (HPX_UNLIKELY(0 == count)) - { - HPX_THROW_EXCEPTION(bad_parameter, - name() + "::alloc", - "cannot allocate 0 objects"); - } - - //std::size_t size = 0; - value_type* p = nullptr; - { - if (!heap_list_.empty()) - { - //size = heap_list_.size(); - for (iterator it = heap_list_.begin(); it != heap_list_.end(); ++it) - { - typename list_type::value_type heap = *it; - bool allocated = false; - - { - util::unlock_guard ul(guard); - allocated = heap->alloc(&p, count); - } - - if (allocated) - { -#if defined(HPX_DEBUG) - // Allocation succeeded, update statistics. - alloc_count_ += count; - if (alloc_count_ - free_count_ > max_alloc_count_) - max_alloc_count_ = alloc_count_- free_count_; -#endif - return p; - } - -#if defined(HPX_DEBUG) - LOSH_(info) << hpx::util::format( - "%1%::alloc: failed to allocate from heap[%2%] " - "(heap[%2%] has allocated %3% objects and has " - "space for %4% more objects)", - name(), - (*it)->heap_count_, - (*it)->size(), - (*it)->free_size()); -#endif - } - } - } - - // Create new heap. - bool did_create = false; - { -#if defined(HPX_DEBUG) - heap_list_.push_front(typename list_type::value_type( - new heap_type(class_name_.c_str(), heap_count_ + 1, heap_step))); -#else - heap_list_.push_front(typename list_type::value_type( - new heap_type(class_name_.c_str(), 0, heap_step))); -#endif - - iterator itnew = heap_list_.begin(); - typename list_type::value_type heap = *itnew; - bool result = false; - - { - util::unlock_guard ul(guard); - result = heap->alloc(&p, count); - } - - if (HPX_UNLIKELY(!result || nullptr == p)) - { - // out of memory - HPX_THROW_EXCEPTION(out_of_memory, - name() + "::alloc", - hpx::util::format( - "new heap failed to allocate %1% objects", - count)); - } - -#if defined(HPX_DEBUG) - alloc_count_ += count; - ++heap_count_; - - LOSH_(info) << hpx::util::format( - "%1%::alloc: creating new heap[%2%], size is now %3%", - name(), - heap_count_, - heap_list_.size()); -#endif - did_create = true; - } - - if (did_create) - return p; - - guard.unlock(); - - // Try again, we just got a new heap, so we should be good. - return alloc(count); - } - - heap_type* alloc_heap() - { - return new heap_type(class_name_.c_str(), 0, heap_step); - } - - void add_heap(heap_type* p) - { - if (HPX_UNLIKELY(!p)) - { - HPX_THROW_EXCEPTION(bad_parameter, - name() + "::add_heap", "encountered nullptr heap"); - } - - unique_lock_type ul(mtx_); -#if defined(HPX_DEBUG) - p->heap_count_ = heap_count_; -#endif - - iterator it = heap_list_.insert(heap_list_.begin(), - typename list_type::value_type(p)); - - // Check for insertion failure. - if (HPX_UNLIKELY(it == heap_list_.end())) - { - HPX_THROW_EXCEPTION(out_of_memory, - name() + "::add_heap", - hpx::util::format("heap %1% could not be added", p)); - } - -#if defined(HPX_DEBUG) - ++heap_count_; -#endif - } + void* alloc(std::size_t count = 1); // need to reschedule if not using boost::mutex - bool reschedule(void* p, std::size_t count) - { - if (nullptr == threads::get_self_ptr()) - { - hpx::applier::register_work( - util::bind(&one_size_heap_list::free, this, p, count), - "one_size_heap_list::free"); - return true; - } - return false; - } - - void free(void* p, std::size_t count = 1) - { - unique_lock_type ul(mtx_); - - if (nullptr == p || !threads::threadmanager_is(state_running)) - return; - - // if this is called from outside a HPX thread we need to - // re-schedule the request - if (reschedule(p, count)) - return; - - // Find the heap which allocated this pointer. - for (iterator it = heap_list_.begin(); it != heap_list_.end(); ++it) - { - typename list_type::value_type heap = *it; - bool did_allocate = false; - - { - util::unlock_guard ull(ul); - did_allocate = heap->did_alloc(p); - if (did_allocate) - heap->free(p, count); - } - - if (did_allocate) - { -#if defined(HPX_DEBUG) - free_count_ += count; -#endif - return; - } - } - - HPX_THROW_EXCEPTION(bad_parameter, - name() + "::free", - hpx::util::format( - "pointer %1% was not allocated by this %2%", - p, name())); - } + bool reschedule(void* p, std::size_t count); - bool did_alloc(void* p) const - { - unique_lock_type ul(mtx_); - for (const_iterator it = heap_list_.begin(); it != heap_list_.end(); ++it) - { - typename list_type::value_type heap = *it; - bool did_allocate = false; + void free(void* p, std::size_t count = 1); - { - util::unlock_guard ull(ul); - did_allocate = heap->did_alloc(p); - } + bool did_alloc(void* p) const; - if (did_allocate) - return true; - } - return false; - } - - std::string name() const - { - if (class_name_.empty()) - return std::string("one_size_heap_list(unknown)"); - return std::string("one_size_heap_list(") + class_name_ + ")"; - } + std::string name() const; protected: mutable mutex_type mtx_; @@ -332,7 +121,14 @@ namespace hpx { namespace util std::size_t heap_count_; std::size_t max_alloc_count_; #endif + std::shared_ptr (*create_heap_)( + char const*, std::size_t, std::size_t, std::size_t); + + std::size_t const heap_step_; // default grow step + std::size_t const heap_size_; // size of the object }; -}} // namespace hpx::util +}} + +#include #endif /*HPX_UTIL_ONE_SIZE_HEAP_LIST_HPP*/ diff --git a/hpx/util/one_size_heap_list_base.hpp b/hpx/util/one_size_heap_list_base.hpp deleted file mode 100644 index dbb127ba708d..000000000000 --- a/hpx/util/one_size_heap_list_base.hpp +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 1998-2013 Hartmut Kaiser -// -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#ifndef HPX_UTIL_ONE_SIZE_HEAP_LIST_BASE_HPP -#define HPX_UTIL_ONE_SIZE_HEAP_LIST_BASE_HPP - -#include -#include - -#include - -namespace hpx { namespace util -{ - struct one_size_heap_list_base - { - virtual ~one_size_heap_list_base() {} - - virtual void* alloc(std::size_t count = 1) = 0; - virtual bool did_alloc(void* p) const = 0; - virtual void free(void* p, std::size_t count = 1) = 0; - - virtual naming::gid_type get_gid(void* p) = 0; - }; -}} - -#endif /*HPX_UTIL_ONE_SIZE_HEAP_LIST_BASE_HPP*/ diff --git a/hpx/util/wrapper_heap_base.hpp b/hpx/util/wrapper_heap_base.hpp new file mode 100644 index 000000000000..e66f9f060e8d --- /dev/null +++ b/hpx/util/wrapper_heap_base.hpp @@ -0,0 +1,35 @@ +// Copyright (c) 1998-2013 Hartmut Kaiser +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#ifndef HPX_UTIL_WRAPPER_HEAP_BASE_HPP +#define HPX_UTIL_WRAPPER_HEAP_BASE_HPP + +#include +#include +#include +#include + +#include + +namespace hpx { namespace util +{ + struct wrapper_heap_base + { + virtual ~wrapper_heap_base() {} + + virtual bool alloc(void** result, std::size_t count = 1) = 0; + virtual bool did_alloc (void *p) const = 0; + virtual void free(void *p, std::size_t count = 1) = 0; + + virtual naming::gid_type get_gid(util::unique_id_ranges& ids, void* p, + components::component_type type) = 0; + + virtual std::size_t heap_count() const = 0; + virtual std::size_t size() const = 0; + virtual std::size_t free_size() const = 0; + }; +}} + +#endif diff --git a/src/runtime/components/server/wrapper_heap.cpp b/src/runtime/components/server/wrapper_heap.cpp new file mode 100644 index 000000000000..d68caa2d3f44 --- /dev/null +++ b/src/runtime/components/server/wrapper_heap.cpp @@ -0,0 +1,377 @@ +// Copyright (c) 1998-2017 Hartmut Kaiser +// Copyright (c) 2011 Bryce Lelbach +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#if HPX_DEBUG_WRAPPER_HEAP != 0 +#include +#endif +#include +#include +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +namespace hpx { namespace components { namespace detail +{ +#if HPX_DEBUG_WRAPPER_HEAP != 0 +#define HPX_WRAPPER_HEAP_INITIALIZED_MEMORY 1 + + namespace debug + { + /////////////////////////////////////////////////////////////////////// + // Test memory area for being filled as expected + inline + bool test_fill_bytes (void *p, unsigned char c, std::size_t cnt) + { + unsigned char* uc = (unsigned char*)p; + for (std::size_t i = 0; i < cnt; ++i) + { + if (*uc++ != c) + return false; + } + return true; + } + + // Fill memory area + inline + void fill_bytes (void *p, unsigned char c, std::size_t cnt) + { + using namespace std; // some systems have memset in namespace std + memset (p, c, cnt); + } + } // namespace debug +#else +# define HPX_WRAPPER_HEAP_INITIALIZED_MEMORY 0 +#endif + + /////////////////////////////////////////////////////////////////////////// + wrapper_heap::wrapper_heap(char const* class_name +#if defined(HPX_DEBUG) + , std::size_t count +#else + , std::size_t +#endif + , std::size_t heap_size + , std::size_t step) + : pool_(nullptr) + , first_free_(nullptr) + , step_(step) + , size_(0) + , free_size_(0) + , base_gid_(naming::invalid_gid) + , class_name_(class_name) +#if defined(HPX_DEBUG) + , alloc_count_(0) + , free_count_(0) + , heap_count_(count) +#endif + , element_size_(heap_size) + , heap_alloc_function_("wrapper_heap::alloc", class_name) + , heap_free_function_("wrapper_heap::free", class_name) + { + util::itt::heap_internal_access hia; + if (!init_pool()) + throw std::bad_alloc(); + } + + wrapper_heap::wrapper_heap() + : pool_(nullptr) + , first_free_(nullptr) + , step_(0) + , size_(0) + , free_size_(0) + , base_gid_(naming::invalid_gid) +#if defined(HPX_DEBUG) + , alloc_count_(0) + , free_count_(0) + , heap_count_(0) +#endif + , element_size_(0) + , heap_alloc_function_("wrapper_heap::alloc", "") + , heap_free_function_("wrapper_heap::free", "") + { + HPX_ASSERT(false); // shouldn't ever be called + } + + wrapper_heap::~wrapper_heap() + { + util::itt::heap_internal_access hia; HPX_UNUSED(hia); + tidy(); + } + + std::size_t wrapper_heap::size() const + { + util::itt::heap_internal_access hia; HPX_UNUSED(hia); + return size_ - free_size_; + } + + std::size_t wrapper_heap::free_size() const + { + util::itt::heap_internal_access hia; HPX_UNUSED(hia); + return free_size_; + } + + bool wrapper_heap::is_empty() const + { + util::itt::heap_internal_access hia; HPX_UNUSED(hia); + return nullptr == pool_; + } + + bool wrapper_heap::has_allocatable_slots() const + { + util::itt::heap_internal_access hia; HPX_UNUSED(hia); + return first_free_ < static_cast(pool_) + size_ * element_size_; + } + + bool wrapper_heap::alloc(void** result, std::size_t count) + { + util::itt::heap_allocate heap_allocate( + heap_alloc_function_, result, count * element_size_, + HPX_WRAPPER_HEAP_INITIALIZED_MEMORY); + + scoped_lock l(mtx_); + + if (!ensure_pool(count)) + return false; + +#if defined(HPX_DEBUG) + alloc_count_ += count; +#endif + + void* p = first_free_; + HPX_ASSERT(p != nullptr); + + first_free_ = static_cast(first_free_) + count * element_size_; + + HPX_ASSERT(free_size_ >= count); + free_size_ -= count; + +#if HPX_DEBUG_WRAPPER_HEAP != 0 + // init memory blocks + debug::fill_bytes(p, initial_value, count * element_size_); +#endif + + *result = p; + return true; + } + + void wrapper_heap::free(void *p, std::size_t count) + { + util::itt::heap_free heap_free(heap_free_function_, p); + +#if HPX_DEBUG_WRAPPER_HEAP != 0 + HPX_ASSERT(did_alloc(p)); +#endif + scoped_lock l(mtx_); + +#if HPX_DEBUG_WRAPPER_HEAP != 0 + char* p1 = static_cast(p); + + HPX_ASSERT(nullptr != pool_ && p1 >= pool_); + HPX_ASSERT(nullptr != pool_ && + p1 + count * element_size_ <= + static_cast(pool_) + size_ * element_size_); + HPX_ASSERT(first_free_ == nullptr || p1 != first_free_); + HPX_ASSERT(free_size_ + count <= size_); + // make sure this has not been freed yet + HPX_ASSERT(!debug::test_fill_bytes(p1, freed_value, + count * element_size_)); + + // give memory back to pool + debug::fill_bytes(p1, freed_value, element_size_); +#else + HPX_UNUSED(p); +#endif + +#if defined(HPX_DEBUG) + free_count_ += count; +#endif + free_size_ += count; + + // release the pool if this one was the last allocated item + test_release(l); + } + + bool wrapper_heap::did_alloc (void *p) const + { + // no lock is necessary here as all involved variables are immutable + util::itt::heap_internal_access hia; HPX_UNUSED(hia); + return nullptr != pool_ && nullptr != p && pool_ <= p && + static_cast(p) < + static_cast(pool_) + size_ * element_size_; + } + + naming::gid_type wrapper_heap::get_gid( + util::unique_id_ranges& ids, void* p, components::component_type type) + { + util::itt::heap_internal_access hia; HPX_UNUSED(hia); + + HPX_ASSERT(did_alloc(p)); + + scoped_lock l(mtx_); + void* addr = pool_; + + if (!base_gid_) + { + naming::gid_type base_gid; + + { + // this is the first call to get_gid() for this heap - allocate + // a sufficiently large range of global ids + util::unlock_guard ul(l); + base_gid = ids.get_id(step_); + + // register the global ids and the base address of this heap + // with the AGAS + if (!applier::bind_range_local(base_gid, step_, + naming::address(hpx::get_locality(), type, addr), + element_size_)) + { + return naming::invalid_gid; + } + } + + // if some other thread has already set the base GID for this + // heap, we ignore the result + if (!base_gid_) + { + // this is the first thread succeeding in binding the new gid + // range + base_gid_ = base_gid; + } + else + { + // unbind the range which is not needed anymore + util::unlock_guard ul(l); + applier::unbind_range_local(base_gid, step_); + } + } + + return base_gid_ + static_cast( + (static_cast(p) - static_cast(addr)) / element_size_); + } + + void wrapper_heap::set_gid(naming::gid_type const& g) + { + util::itt::heap_internal_access hia; HPX_UNUSED(hia); + + scoped_lock l(mtx_); + base_gid_ = g; + } + + bool wrapper_heap::test_release(scoped_lock& lk) + { + if (pool_ == nullptr || free_size_ < size_ || + static_cast(first_free_) < + static_cast(pool_) + size_ * element_size_) + { + return false; + } + + HPX_ASSERT(free_size_ == size_); + + // unbind in AGAS service + if (base_gid_) + { + naming::gid_type base_gid = base_gid_; + base_gid_ = naming::invalid_gid; + + util::unlock_guard ull(lk); + applier::unbind_range_local(base_gid, step_); + } + + tidy(); + return true; + } + + bool wrapper_heap::ensure_pool(std::size_t count) + { + if (nullptr == pool_) + { + return false; + } + + if (static_cast(first_free_) + count * element_size_ > + static_cast(pool_) + size_ * element_size_) + { + return false; + } + return true; + } + + bool wrapper_heap::init_pool() + { + HPX_ASSERT(size_ == 0); + HPX_ASSERT(first_free_ == nullptr); + + std::size_t s = step_ * element_size_; //-V104 //-V707 + pool_ = allocator_type::alloc(s); + if (nullptr == pool_) + { + return false; + } + + first_free_ = pool_; + size_ = s / element_size_; //-V104 + free_size_ = size_; + + LOSH_(info) //-V128 + << "wrapper_heap (" + << (!class_name_.empty() ? class_name_.c_str() : "") + << "): init_pool (" << std::hex << pool_ << ")" + << " size: " << s << "."; + + return true; + } + + void wrapper_heap::tidy() + { + if (pool_ != nullptr) + { + LOSH_(debug) //-V128 + << "wrapper_heap (" + << (!class_name_.empty() ? class_name_.c_str() : "") + << ")" +#if defined(HPX_DEBUG) + << ": releasing heap: alloc count: " << alloc_count_ + << ", free count: " << free_count_ +#endif + << "."; + + if (free_size_ != size_ +#if defined(HPX_DEBUG) + || alloc_count_ != free_count_ +#endif + ) + { + LOSH_(warning) //-V128 + << "wrapper_heap (" + << (!class_name_.empty() ? class_name_.c_str() : "") + << "): releasing heap (" << std::hex << pool_ << ")" + << " with " << size_-free_size_ << " allocated object(s)!"; + } + + allocator_type::free(pool_); + pool_ = first_free_ = nullptr; + size_ = free_size_ = 0; + } + } +}}} diff --git a/src/util/one_size_heap_list.cpp b/src/util/one_size_heap_list.cpp new file mode 100644 index 000000000000..3ede622a4369 --- /dev/null +++ b/src/util/one_size_heap_list.cpp @@ -0,0 +1,224 @@ +// Copyright (c) 1998-2017 Hartmut Kaiser +// Copyright (c) 2011 Bryce Lelbach +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include +#include +#include +#if defined(HPX_DEBUG) +#include +#endif +#include +#include + +#include +#include +#include +#include +#include + +namespace hpx { namespace util +{ + one_size_heap_list::~one_size_heap_list() noexcept + { +#if defined(HPX_DEBUG) + LOSH_(info) << hpx::util::format( + "%1%::~%1%: size(%2%), max_count(%3%), alloc_count(%4%), " + "free_count(%5%)", + name(), + heap_count_, + max_alloc_count_, + alloc_count_, + free_count_); + + if (alloc_count_ > free_count_) + { + LOSH_(warning) << hpx::util::format( + "%1%::~%1%: releasing with %2% allocated objects", + name(), + alloc_count_ - free_count_); + } +#endif + } + + void* one_size_heap_list::alloc(std::size_t count) + { + unique_lock_type guard(mtx_); + + if (HPX_UNLIKELY(0 == count)) + { + HPX_THROW_EXCEPTION(bad_parameter, + name() + "::alloc", + "cannot allocate 0 objects"); + } + + //std::size_t size = 0; + void* p = nullptr; + { + if (!heap_list_.empty()) + { + //size = heap_list_.size(); + for (auto & heap : heap_list_) + { + bool allocated = false; + + { + util::unlock_guard ul(guard); + allocated = heap->alloc(&p, count); + } + + if (allocated) + { +#if defined(HPX_DEBUG) + // Allocation succeeded, update statistics. + alloc_count_ += count; + if (alloc_count_ - free_count_ > max_alloc_count_) + max_alloc_count_ = alloc_count_- free_count_; +#endif + return p; + } + +#if defined(HPX_DEBUG) + LOSH_(info) << hpx::util::format( + "%1%::alloc: failed to allocate from heap[%2%] " + "(heap[%2%] has allocated %3% objects and has " + "space for %4% more objects)", + name(), + heap->heap_count(), + heap->size(), + heap->free_size()); +#endif + } + } + } + + // Create new heap. + bool did_create = false; + { + heap_list_.push_front(create_heap_( + class_name_.c_str(), heap_count_ + 1, heap_step_, heap_size_)); + + iterator itnew = heap_list_.begin(); + typename list_type::value_type heap = *itnew; + bool result = false; + + { + util::unlock_guard ul(guard); + result = heap->alloc((void**)&p, count); + } + + if (HPX_UNLIKELY(!result || nullptr == p)) + { + // out of memory + HPX_THROW_EXCEPTION(out_of_memory, + name() + "::alloc", + hpx::util::format( + "new heap failed to allocate %1% objects", + count)); + } + +#if defined(HPX_DEBUG) + alloc_count_ += count; + ++heap_count_; + + LOSH_(info) << hpx::util::format( + "%1%::alloc: creating new heap[%2%], size is now %3%", + name(), + heap_count_, + heap_list_.size()); +#endif + did_create = true; + } + + if (did_create) + return p; + + guard.unlock(); + + // Try again, we just got a new heap, so we should be good. + return alloc(count); + } + + bool one_size_heap_list::reschedule(void* p, std::size_t count) + { + if (nullptr == threads::get_self_ptr()) + { + hpx::applier::register_work( + util::bind(&one_size_heap_list::free, this, p, count), + "one_size_heap_list::free"); + return true; + } + return false; + } + + void one_size_heap_list::free(void* p, std::size_t count) + { + unique_lock_type ul(mtx_); + + if (nullptr == p || !threads::threadmanager_is(state_running)) + return; + + // if this is called from outside a HPX thread we need to + // re-schedule the request + if (reschedule(p, count)) + return; + + // Find the heap which allocated this pointer. + for (auto & heap : heap_list_) + { + bool did_allocate = false; + + { + util::unlock_guard ull(ul); + did_allocate = heap->did_alloc(p); + if (did_allocate) + heap->free(p, count); + } + + if (did_allocate) + { +#if defined(HPX_DEBUG) + free_count_ += count; +#endif + return; + } + } + + HPX_THROW_EXCEPTION(bad_parameter, + name() + "::free", + hpx::util::format( + "pointer %1% was not allocated by this %2%", + p, name())); + } + + bool one_size_heap_list::did_alloc(void* p) const + { + unique_lock_type ul(mtx_); + for (typename list_type::value_type const& heap : heap_list_) + { + bool did_allocate = false; + + { + util::unlock_guard ull(ul); + did_allocate = heap->did_alloc(p); + } + + if (did_allocate) + return true; + } + return false; + } + + std::string one_size_heap_list::name() const + { + if (class_name_.empty()) + return std::string("one_size_heap_list(unknown)"); + return std::string("one_size_heap_list(") + class_name_ + ")"; + } +}}