From 02fa0219dd0cec8503cad03e71e144b466cc7c3f Mon Sep 17 00:00:00 2001 From: "d.levin256@gmail.com" Date: Mon, 2 Dec 2019 18:24:49 +0000 Subject: [PATCH] Move memory.hpp to cometa --- CMakeLists.txt | 8 +- include/kfr/base/memory.hpp | 257 +----------------------------- include/kfr/base/pointer.hpp | 2 +- include/kfr/cometa.hpp | 1 + include/kfr/cometa/function.hpp | 26 ++- include/kfr/cometa/memory.hpp | 269 ++++++++++++++++++++++++++++++++ sources.cmake | 1 + 7 files changed, 299 insertions(+), 265 deletions(-) create mode 100644 include/kfr/cometa/memory.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0351d57e..5cd72bdd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,9 +159,11 @@ endif() message(STATUS CPU_ARCH=${CPU_ARCH}) -add_executable(detect_cpu ${CMAKE_CURRENT_SOURCE_DIR}/cmake/detect_cpu.cpp) -target_link_libraries(detect_cpu PRIVATE kfr) -target_set_arch(detect_cpu PRIVATE generic) +if (X86) + add_executable(detect_cpu ${CMAKE_CURRENT_SOURCE_DIR}/cmake/detect_cpu.cpp) + target_link_libraries(detect_cpu PRIVATE kfr) + target_set_arch(detect_cpu PRIVATE generic) +endif () if (ENABLE_TESTS) diff --git a/include/kfr/base/memory.hpp b/include/kfr/base/memory.hpp index 141af0f3..5228311a 100644 --- a/include/kfr/base/memory.hpp +++ b/include/kfr/base/memory.hpp @@ -25,259 +25,4 @@ */ #pragma once -#include "../simd/read_write.hpp" -#include "../simd/types.hpp" -#include -#include -#include - -namespace kfr -{ - -namespace internal_generic -{ - -struct memory_statistics -{ - std::atomic_uintptr_t allocation_count = ATOMIC_VAR_INIT(0); - std::atomic_uintptr_t allocation_size = ATOMIC_VAR_INIT(0); - std::atomic_uintptr_t deallocation_count = ATOMIC_VAR_INIT(0); - std::atomic_uintptr_t deallocation_size = ATOMIC_VAR_INIT(0); -}; - -inline memory_statistics& get_memory_statistics() -{ - static memory_statistics ms; - return ms; -} - -#pragma pack(push, 1) - -struct mem_header -{ - u8 offset; - u8 alignment; - u8 reserved1; - u8 reserved2; - unsigned int references_uint; - size_t size; - - KFR_MEM_INTRINSIC std::atomic_uint& references() - { - return reinterpret_cast(references_uint); - } -} -#ifdef CMT_GNU_ATTRIBUTES -__attribute__((__packed__)) -#endif -; - -#pragma pack(pop) - -inline mem_header* aligned_header(void* ptr) { return ptr_cast(ptr) - 1; } - -inline size_t aligned_size(void* ptr) { return aligned_header(ptr)->size; } - -inline void* aligned_malloc(size_t size, size_t alignment) -{ - get_memory_statistics().allocation_count++; - get_memory_statistics().allocation_size += size; - void* ptr = malloc(size + (alignment - 1) + sizeof(mem_header)); - if (ptr == nullptr) - return nullptr; - void* aligned_ptr = advance(ptr, sizeof(mem_header)); - aligned_ptr = align_up(aligned_ptr, alignment); - aligned_header(aligned_ptr)->alignment = static_cast(alignment > 255 ? 255 : alignment); - aligned_header(aligned_ptr)->offset = static_cast(distance(aligned_ptr, ptr)); - aligned_header(aligned_ptr)->references() = 1; - aligned_header(aligned_ptr)->size = size; - return aligned_ptr; -} - -inline void aligned_force_free(void* ptr) -{ - get_memory_statistics().deallocation_count++; - get_memory_statistics().deallocation_size += aligned_size(ptr); - free(advance(ptr, -static_cast(aligned_header(ptr)->offset))); -} - -inline void aligned_add_ref(void* ptr) { aligned_header(ptr)->references()++; } - -inline void aligned_free(void* ptr) -{ - if (--aligned_header(ptr)->references() == 0) - aligned_force_free(ptr); -} - -inline void aligned_release(void* ptr) { aligned_free(ptr); } - -inline void* aligned_reallocate(void* ptr, size_t new_size, size_t alignment) -{ - if (ptr) - { - if (new_size) - { - void* new_ptr = aligned_malloc(new_size, alignment); - size_t old_size = aligned_size(ptr); - memcpy(new_ptr, ptr, std::min(old_size, new_size)); - aligned_release(ptr); - return new_ptr; - } - else - { - aligned_release(ptr); - return nullptr; - } - } - else - { - if (new_size) - { - return internal_generic::aligned_malloc(new_size, alignment); - } - else - { - return nullptr; // do nothing - } - } -} -} // namespace internal_generic - -/// @brief Allocates aligned memory -template ::native_cache_alignment> -KFR_INTRINSIC T* aligned_allocate(size_t size = 1) -{ - T* ptr = static_cast(CMT_ASSUME_ALIGNED( - internal_generic::aligned_malloc(std::max(alignment, size * details::elementsize()), alignment), - alignment)); - return ptr; -} - -/// @brief Deallocates aligned memory -template -KFR_INTRINSIC void aligned_deallocate(T* ptr) -{ - return internal_generic::aligned_free(ptr); -} - -namespace internal_generic -{ -template -struct aligned_deleter -{ - KFR_MEM_INTRINSIC void operator()(T* ptr) const { aligned_deallocate(ptr); } -}; -} // namespace internal_generic - -template -struct autofree -{ - KFR_MEM_INTRINSIC autofree() {} - explicit KFR_MEM_INTRINSIC autofree(size_t size) : ptr(aligned_allocate(size)) {} - autofree(const autofree&) = delete; - autofree& operator=(const autofree&) = delete; - autofree(autofree&&) CMT_NOEXCEPT = default; - autofree& operator=(autofree&&) CMT_NOEXCEPT = default; - KFR_MEM_INTRINSIC T& operator[](size_t index) CMT_NOEXCEPT { return ptr[index]; } - KFR_MEM_INTRINSIC const T& operator[](size_t index) const CMT_NOEXCEPT { return ptr[index]; } - - template - KFR_MEM_INTRINSIC U* data() CMT_NOEXCEPT - { - return ptr_cast(ptr.get()); - } - template - KFR_MEM_INTRINSIC const U* data() const CMT_NOEXCEPT - { - return ptr_cast(ptr.get()); - } - - std::unique_ptr> ptr; -}; - -#ifdef KFR_USE_STD_ALLOCATION - -template -using allocator = std::allocator; - -#else - -/// @brief Aligned allocator -template -struct allocator -{ - using value_type = T; - using pointer = T*; - using const_pointer = const T*; - using reference = T&; - using const_reference = const T&; - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - - template - struct rebind - { - using other = allocator; - }; - constexpr allocator() CMT_NOEXCEPT = default; - constexpr allocator(const allocator&) CMT_NOEXCEPT = default; - template - constexpr allocator(const allocator&) CMT_NOEXCEPT - { - } - pointer allocate(size_type n) const - { - pointer result = aligned_allocate(n); - if (!result) - CMT_THROW(std::bad_alloc()); - return result; - } - void deallocate(pointer p, size_type) { aligned_deallocate(p); } -}; - -template -constexpr inline bool operator==(const allocator&, const allocator&) CMT_NOEXCEPT -{ - return true; -} -template -constexpr inline bool operator!=(const allocator&, const allocator&) CMT_NOEXCEPT -{ - return false; -} - -#endif - -struct aligned_new -{ - inline static void* operator new(size_t size) noexcept { return aligned_allocate(size); } - inline static void operator delete(void* ptr) noexcept { return aligned_deallocate(ptr); } - -#ifdef __cpp_aligned_new - inline static void* operator new(size_t size, std::align_val_t al) noexcept - { - return internal_generic::aligned_malloc(size, std::max(size_t(32), static_cast(al))); - } - inline static void operator delete(void* ptr, std::align_val_t al) noexcept - { - return internal_generic::aligned_free(ptr); - } -#endif -}; - -#define KFR_CLASS_REFCOUNT(cl) \ - \ -public: \ - void addref() const { m_refcount++; } \ - void release() const \ - { \ - if (--m_refcount == 0) \ - { \ - delete this; \ - } \ - } \ - \ -private: \ - mutable std::atomic_uintptr_t m_refcount = ATOMIC_VAR_INIT(0); - -} // namespace kfr +#include "../cometa/memory.hpp" diff --git a/include/kfr/base/pointer.hpp b/include/kfr/base/pointer.hpp index d4ea7e2e..964a1f00 100644 --- a/include/kfr/base/pointer.hpp +++ b/include/kfr/base/pointer.hpp @@ -129,7 +129,7 @@ struct expression_resource_impl : expression_resource #ifdef __cpp_aligned_new static void operator delete (void* p, std::align_val_t al) noexcept { - internal_generic::aligned_release(p); + details::aligned_release(p); } #endif diff --git a/include/kfr/cometa.hpp b/include/kfr/cometa.hpp index 026486da..b4f4ec88 100644 --- a/include/kfr/cometa.hpp +++ b/include/kfr/cometa.hpp @@ -2128,3 +2128,4 @@ CMT_PRAGMA_GNU(GCC diagnostic pop) CMT_PRAGMA_GNU(GCC diagnostic pop) CMT_PRAGMA_MSVC(warning(pop)) + diff --git a/include/kfr/cometa/function.hpp b/include/kfr/cometa/function.hpp index 396713b7..7df7a149 100644 --- a/include/kfr/cometa/function.hpp +++ b/include/kfr/cometa/function.hpp @@ -4,6 +4,14 @@ #pragma once #include "../cometa.hpp" +#include "memory.hpp" +#include +#include +#include +#include +#if CMT_HAS_EXCEPTIONS +#include +#endif namespace cometa { @@ -55,13 +63,16 @@ struct function_abstract template struct function_impl : public function_abstract { - inline static void* operator new(size_t size) noexcept { return std::aligned_alloc(size, alignof(Fn)); } - inline static void operator delete(void* ptr) noexcept { return std::free(ptr); } + inline static void* operator new(size_t size) noexcept { return aligned_allocate(size, alignof(Fn)); } + inline static void operator delete(void* ptr) noexcept { return aligned_deallocate(ptr); } inline static void* operator new(size_t size, std::align_val_t al) noexcept { - return std::aligned_alloc(size, static_cast(al)); + return aligned_allocate(size, static_cast(al)); + } + inline static void operator delete(void* ptr, std::align_val_t al) noexcept + { + return aligned_deallocate(ptr); } - inline static void operator delete(void* ptr, std::align_val_t al) noexcept { return std::free(ptr); } template function_impl(Fn_ fn) : fn(std::forward(fn)) @@ -82,7 +93,7 @@ struct function template && !std::is_same_v, function>>> - function(Fn fn) : impl(new internal::function_impl, R, Args...>(std::move(fn))) + function(Fn fn) : impl(new details::function_impl, R, Args...>(std::move(fn))) { } @@ -96,11 +107,16 @@ struct function R operator()(Args... args) const { +#if CMT_HAS_EXCEPTIONS if (impl) { return impl->operator()(std::forward(args)...); } throw std::bad_function_call(); +#else + // With exceptions disabled let it crash. To prevent this, check first + return impl->operator()(std::forward(args)...); +#endif } [[nodiscard]] explicit operator bool() const { return !!impl; } diff --git a/include/kfr/cometa/memory.hpp b/include/kfr/cometa/memory.hpp new file mode 100644 index 00000000..0e82a0f1 --- /dev/null +++ b/include/kfr/cometa/memory.hpp @@ -0,0 +1,269 @@ +/** @addtogroup cometa + * @{ + */ +#pragma once + +#include "numeric.hpp" +#include +#include +#include +#include + +namespace cometa +{ + +namespace details +{ + +struct memory_statistics +{ + std::atomic_uintptr_t allocation_count = ATOMIC_VAR_INIT(0); + std::atomic_uintptr_t allocation_size = ATOMIC_VAR_INIT(0); + std::atomic_uintptr_t deallocation_count = ATOMIC_VAR_INIT(0); + std::atomic_uintptr_t deallocation_size = ATOMIC_VAR_INIT(0); +}; + +inline memory_statistics& get_memory_statistics() +{ + static memory_statistics ms; + return ms; +} + +#pragma pack(push, 1) + +struct mem_header +{ + u8 offset; + u8 alignment; + u8 reserved1; + u8 reserved2; + unsigned int references_uint; + size_t size; + + CMT_MEM_INTRINSIC std::atomic_uint& references() + { + return reinterpret_cast(references_uint); + } +} +#ifdef CMT_GNU_ATTRIBUTES +__attribute__((__packed__)) +#endif +; + +#pragma pack(pop) + +inline mem_header* aligned_header(void* ptr) { return ptr_cast(ptr) - 1; } + +inline size_t aligned_size(void* ptr) { return aligned_header(ptr)->size; } + +inline void* aligned_malloc(size_t size, size_t alignment) +{ + get_memory_statistics().allocation_count++; + get_memory_statistics().allocation_size += size; + void* ptr = malloc(size + (alignment - 1) + sizeof(mem_header)); + if (ptr == nullptr) + return nullptr; + void* aligned_ptr = advance(ptr, sizeof(mem_header)); + aligned_ptr = align_up(aligned_ptr, alignment); + aligned_header(aligned_ptr)->alignment = static_cast(alignment > 255 ? 255 : alignment); + aligned_header(aligned_ptr)->offset = static_cast(distance(aligned_ptr, ptr)); + aligned_header(aligned_ptr)->references() = 1; + aligned_header(aligned_ptr)->size = size; + return aligned_ptr; +} + +inline void aligned_force_free(void* ptr) +{ + get_memory_statistics().deallocation_count++; + get_memory_statistics().deallocation_size += aligned_size(ptr); + free(advance(ptr, -static_cast(aligned_header(ptr)->offset))); +} + +inline void aligned_add_ref(void* ptr) { aligned_header(ptr)->references()++; } + +inline void aligned_free(void* ptr) +{ + if (--aligned_header(ptr)->references() == 0) + aligned_force_free(ptr); +} + +inline void aligned_release(void* ptr) { aligned_free(ptr); } + +inline void* aligned_reallocate(void* ptr, size_t new_size, size_t alignment) +{ + if (ptr) + { + if (new_size) + { + void* new_ptr = aligned_malloc(new_size, alignment); + size_t old_size = aligned_size(ptr); + memcpy(new_ptr, ptr, std::min(old_size, new_size)); + aligned_release(ptr); + return new_ptr; + } + else + { + aligned_release(ptr); + return nullptr; + } + } + else + { + if (new_size) + { + return details::aligned_malloc(new_size, alignment); + } + else + { + return nullptr; // do nothing + } + } +} +} // namespace details + +/// @brief Allocates aligned memory +template +CMT_INTRINSIC T* aligned_allocate(size_t size = 1) +{ + T* ptr = static_cast(CMT_ASSUME_ALIGNED( + details::aligned_malloc(std::max(alignment, size * details::elementsize()), alignment), + alignment)); + return ptr; +} +/// @brief Allocates aligned memory +template +CMT_INTRINSIC T* aligned_allocate(size_t size, size_t alignment) +{ + T* ptr = static_cast(CMT_ASSUME_ALIGNED( + details::aligned_malloc(std::max(alignment, size * details::elementsize()), alignment), + alignment)); + return ptr; +} + +/// @brief Deallocates aligned memory +template +CMT_INTRINSIC void aligned_deallocate(T* ptr) +{ + return details::aligned_free(ptr); +} + +namespace details +{ +template +struct aligned_deleter +{ + CMT_MEM_INTRINSIC void operator()(T* ptr) const { aligned_deallocate(ptr); } +}; +} // namespace details + +template +struct autofree +{ + CMT_MEM_INTRINSIC autofree() {} + explicit CMT_MEM_INTRINSIC autofree(size_t size) : ptr(aligned_allocate(size)) {} + autofree(const autofree&) = delete; + autofree& operator=(const autofree&) = delete; + autofree(autofree&&) CMT_NOEXCEPT = default; + autofree& operator=(autofree&&) CMT_NOEXCEPT = default; + CMT_MEM_INTRINSIC T& operator[](size_t index) CMT_NOEXCEPT { return ptr[index]; } + CMT_MEM_INTRINSIC const T& operator[](size_t index) const CMT_NOEXCEPT { return ptr[index]; } + + template + CMT_MEM_INTRINSIC U* data() CMT_NOEXCEPT + { + return ptr_cast(ptr.get()); + } + template + CMT_MEM_INTRINSIC const U* data() const CMT_NOEXCEPT + { + return ptr_cast(ptr.get()); + } + + std::unique_ptr> ptr; +}; + +#ifdef KFR_USE_STD_ALLOCATION + +template +using allocator = std::allocator; + +#else + +/// @brief Aligned allocator +template +struct allocator +{ + using value_type = T; + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + template + struct rebind + { + using other = allocator; + }; + constexpr allocator() CMT_NOEXCEPT = default; + constexpr allocator(const allocator&) CMT_NOEXCEPT = default; + template + constexpr allocator(const allocator&) CMT_NOEXCEPT + { + } + pointer allocate(size_type n) const + { + pointer result = aligned_allocate(n); + if (!result) + CMT_THROW(std::bad_alloc()); + return result; + } + void deallocate(pointer p, size_type) { aligned_deallocate(p); } +}; + +template +constexpr inline bool operator==(const allocator&, const allocator&) CMT_NOEXCEPT +{ + return true; +} +template +constexpr inline bool operator!=(const allocator&, const allocator&) CMT_NOEXCEPT +{ + return false; +} + +#endif + +struct aligned_new +{ + inline static void* operator new(size_t size) noexcept { return aligned_allocate(size); } + inline static void operator delete(void* ptr) noexcept { return aligned_deallocate(ptr); } + +#ifdef __cpp_aligned_new + inline static void* operator new(size_t size, std::align_val_t al) noexcept + { + return details::aligned_malloc(size, std::max(size_t(64), static_cast(al))); + } + inline static void operator delete(void* ptr, std::align_val_t al) noexcept + { + return details::aligned_free(ptr); + } +#endif +}; + +#define KFR_CLASS_REFCOUNT(cl) \ + \ +public: \ + void addref() const { m_refcount++; } \ + void release() const \ + { \ + if (--m_refcount == 0) \ + { \ + delete this; \ + } \ + } \ + \ +private: \ + mutable std::atomic_uintptr_t m_refcount = ATOMIC_VAR_INIT(0); +} // namespace cometa \ No newline at end of file diff --git a/sources.cmake b/sources.cmake index bccb72b7..23cc80ef 100644 --- a/sources.cmake +++ b/sources.cmake @@ -36,6 +36,7 @@ set( ${PROJECT_SOURCE_DIR}/include/kfr/cometa/cstring.hpp ${PROJECT_SOURCE_DIR}/include/kfr/cometa/ctti.hpp ${PROJECT_SOURCE_DIR}/include/kfr/cometa/function.hpp + ${PROJECT_SOURCE_DIR}/include/kfr/cometa/memory.hpp ${PROJECT_SOURCE_DIR}/include/kfr/cometa/named_arg.hpp ${PROJECT_SOURCE_DIR}/include/kfr/cometa/numeric.hpp ${PROJECT_SOURCE_DIR}/include/kfr/cometa/range.hpp