diff --git a/ddtrace/profiling/collector/CMakeLists.txt b/ddtrace/profiling/collector/CMakeLists.txt new file mode 100644 index 00000000000..39c3616d351 --- /dev/null +++ b/ddtrace/profiling/collector/CMakeLists.txt @@ -0,0 +1,147 @@ +cmake_minimum_required(VERSION 3.19) +include(FetchContent) + +project(_memalloc) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# Add compile options +add_compile_options(-fPIC -fvisibility=hidden -pthread -Wall -Wextra) + +# Platform-specific compile definitions +if(APPLE) + # Fix for newer macOS SDKs that don't define BSD-style types These are needed by Abseil on macOS + add_compile_definitions(_DARWIN_C_SOURCE) +endif() + +add_compile_definitions(_POSIX_C_SOURCE=200809L) + +# Check the DD_COMPILE_ABSEIL environment variable and build type +if(DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}" STREQUAL + "false")) + message("==============================================================") + message("WARNING: DD_COMPILE_ABSEIL set to 0 or false: not using abseil") + message("==============================================================") + add_definitions(-DDONT_COMPILE_ABSEIL) +elseif(CMAKE_BUILD_TYPE STREQUAL "Debug") + message("=====================================") + message("WARNING: Debug mode: not using abseil") + message("=====================================") + add_definitions(-DDONT_COMPILE_ABSEIL) +else() + message("Release, RelWithDebInfo, or MinSizeRel mode: using abseil (DD_COMPILE_ABSEIL unset or not 0/false)") + FetchContent_Declare(absl URL "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.zip") + FetchContent_MakeAvailable(absl) +endif() + +# Find Python (be flexible about what's available in build environments) +find_package(Python3 COMPONENTS Interpreter Development) + +# Make sure we have necessary Python variables +if(NOT Python3_INCLUDE_DIRS) + # Fallback to PYTHON_INCLUDE_DIRS if Python3_INCLUDE_DIRS not found + if(PYTHON_INCLUDE_DIRS) + set(Python3_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS}) + else() + message(FATAL_ERROR "Python3_INCLUDE_DIRS not found") + endif() +endif() + +# Python3::Python target might not exist in all build environments so we'll link using include dirs and let the linker +# find Python dynamically +if(NOT TARGET Python3::Python) + message(STATUS "Python3::Python target not found, using include dirs only") +endif() + +# Source files for the extension +set(SOURCE_FILES _memalloc.cpp _memalloc_tb.cpp _memalloc_heap.cpp _memalloc_reentrant.cpp _memalloc_heap_map.cpp) + +# Get the extension name from setup.py or use default Note: EXTENSION_NAME from setup.py already includes the full +# suffix +if(DEFINED EXTENSION_NAME) + set(FULL_EXTENSION_NAME "${EXTENSION_NAME}") +else() + set(FULL_EXTENSION_NAME "_memalloc.so") +endif() + +# Create the shared library with the full name +add_library(${FULL_EXTENSION_NAME} SHARED ${SOURCE_FILES}) + +# Set properties to prevent CMake from adding any prefix or suffix +set_target_properties(${FULL_EXTENSION_NAME} PROPERTIES PREFIX "" SUFFIX "") + +# Set output directory if specified +if(DEFINED LIB_INSTALL_DIR) + set_target_properties(${FULL_EXTENSION_NAME} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${LIB_INSTALL_DIR}) +endif() + +# Include directories +target_include_directories( + ${FULL_EXTENSION_NAME} PRIVATE ${Python3_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../../internal/datadog/profiling/dd_wrapper/include) + +# Link libraries Python3::Python target might not exist in all build environments (e.g., manylinux) Python modules +# should use -undefined dynamic_lookup on macOS and not link to libpython on Linux +if(TARGET Python3::Python AND NOT APPLE) + target_link_libraries(${FULL_EXTENSION_NAME} PRIVATE Python3::Python) +endif() + +# Link Abseil if available +if(NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" + OR (DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}" + STREQUAL "false")))) + target_link_libraries(${FULL_EXTENSION_NAME} PRIVATE absl::flat_hash_map) +endif() + +# Platform-specific settings +if(APPLE) + # macOS specific - set rpath for libdd_wrapper and use dynamic lookup for Python symbols + set_target_properties( + ${FULL_EXTENSION_NAME} + PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "@loader_path/../../internal/datadog/profiling" + LINK_FLAGS "-undefined dynamic_lookup") +elseif(UNIX) + # Linux specific + target_link_libraries(${FULL_EXTENSION_NAME} PRIVATE atomic) + set_target_properties(${FULL_EXTENSION_NAME} PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "$ORIGIN/../../internal/datadog/profiling") +endif() + +# Link with libdd_wrapper if NATIVE_EXTENSION_LOCATION is defined +if(DEFINED NATIVE_EXTENSION_LOCATION) + # Find the libdd_wrapper shared library + find_library( + DD_WRAPPER_LIB + NAMES libdd_wrapper${EXTENSION_SUFFIX} + PATHS ${CMAKE_CURRENT_SOURCE_DIR}/../../internal/datadog/profiling + ${NATIVE_EXTENSION_LOCATION}/../../datadog/profiling + NO_DEFAULT_PATH) + + if(DD_WRAPPER_LIB) + message(STATUS "Found libdd_wrapper: ${DD_WRAPPER_LIB}") + target_link_libraries(${FULL_EXTENSION_NAME} PRIVATE ${DD_WRAPPER_LIB}) + else() + message(WARNING "libdd_wrapper not found, extension may not link correctly") + endif() +endif() + +# Add NDEBUG flag for release builds +if(CMAKE_BUILD_TYPE STREQUAL "Release" + OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" + OR CMAKE_BUILD_TYPE STREQUAL "MinSizeRel") + target_compile_definitions(${FULL_EXTENSION_NAME} PRIVATE NDEBUG) +else() + target_compile_definitions(${FULL_EXTENSION_NAME} PRIVATE UNDEBUG) +endif() + +# Install the extension +install(TARGETS ${FULL_EXTENSION_NAME} LIBRARY DESTINATION ${LIB_INSTALL_DIR}) + +# Optional: Build tests if BUILD_TESTING is ON +option(BUILD_TESTING "Build tests" OFF) +if(BUILD_TESTING) + enable_testing() + add_subdirectory(test) +endif() diff --git a/ddtrace/profiling/collector/_memalloc_heap_map.cpp b/ddtrace/profiling/collector/_memalloc_heap_map.cpp index 3be6819b1a7..1945708325f 100644 --- a/ddtrace/profiling/collector/_memalloc_heap_map.cpp +++ b/ddtrace/profiling/collector/_memalloc_heap_map.cpp @@ -1,93 +1,71 @@ #include "_memalloc_heap_map.hpp" #include "_memalloc_debug.h" -/* Note that the HeapSample tables will, in general, never free their backing - * memory unless we completely clear them. The table takes 17 bytes per entry: 8 - * for the void* keys, 8 for the traceback* values, and 1 byte per entry for - * control metadata. Assuming a load factor target of ~50%, meaning our table - * has roughly twice as many slots as actual entries, then for our default - * maximum of 2^16 entries the table will be about 2MiB. A table this large - * would correspond to a program with a ~65GiB live heap with a 1MiB default - * sampling interval. Most of the memory usage of the profiler will come from - * the tracebacks themselves, which we _do_ free when we're done with them. +/* Note that the heap tracking tables will, in general, never free their backing + * memory unless we completely clear them. Abseil's flat_hash_map uses approximately + * 8 bytes for the key (void*), 8 bytes for the value (traceback_t*), plus metadata. + * With a load factor of ~87.5% (Abseil's default), the table is quite efficient. + * For our default maximum of 2^16 entries, the table will be roughly 2-3 MiB. + * A table this large would correspond to a program with a ~65GiB live heap with + * a 1MiB default sampling interval. Most of the memory usage of the profiler will + * come from the tracebacks themselves, which we _do_ free when we're done with them. */ // memalloc_heap_map implementation -memalloc_heap_map::memalloc_heap_map() - : map(HeapSamples_new(0)) -{ -} - memalloc_heap_map::~memalloc_heap_map() { - HeapSamples_CIter it = HeapSamples_citer(&map); - for (const HeapSamples_Entry* e = HeapSamples_CIter_get(&it); e != nullptr; e = HeapSamples_CIter_next(&it)) { - delete e->val; + // Delete all traceback objects before map is destroyed + for (auto& [key, tb] : map) { + delete tb; } - HeapSamples_destroy(&map); -} - -size_t -memalloc_heap_map::size() const -{ - return HeapSamples_size(&map); } traceback_t* memalloc_heap_map::insert(void* key, traceback_t* value) { - HeapSamples_Entry k = { .key = key, .val = value }; - HeapSamples_Insert res = HeapSamples_insert(&map, &k); - traceback_t* prev = nullptr; - if (!res.inserted) { + // Try to insert the new value + auto [it, inserted] = map.insert({ key, value }); + + if (!inserted) { + // Key already existed, replace the value /* This should not happen. It means we did not properly remove a previously-tracked * allocation from the map. This should probably be an assertion. Return the previous * entry as it is for an allocation that has been freed. */ - HeapSamples_Entry* e = HeapSamples_Iter_get(&res.iter); - prev = e->val; - e->val = value; + traceback_t* prev = it->second; + it->second = value; + return prev; } - return prev; -} -bool -memalloc_heap_map::contains(void* key) const -{ - return HeapSamples_contains(&map, &key); + return nullptr; } traceback_t* memalloc_heap_map::remove(void* key) { - traceback_t* res = nullptr; - HeapSamples_Iter it = HeapSamples_find(&map, &key); - HeapSamples_Entry* e = HeapSamples_Iter_get(&it); - if (e != nullptr) { - res = e->val; - /* This erases the entry but won't shrink the table. */ - HeapSamples_erase_at(it); + auto it = map.find(key); + if (it == map.end()) { + return nullptr; } - return res; + + traceback_t* result = it->second; + map.erase(it); + return result; } PyObject* memalloc_heap_map::export_to_python() const { - PyObject* heap_list = PyList_New(HeapSamples_size(&map)); + PyObject* heap_list = PyList_New(map.size()); if (heap_list == nullptr) { return nullptr; } - int i = 0; - HeapSamples_CIter it = HeapSamples_citer(&map); - for (const HeapSamples_Entry* e = HeapSamples_CIter_get(&it); e != nullptr; e = HeapSamples_CIter_next(&it)) { - traceback_t* tb = e->val; - + size_t i = 0; + for (const auto& [key, tb] : map) { PyObject* tb_and_size = PyTuple_New(2); PyTuple_SET_ITEM(tb_and_size, 0, tb->to_tuple()); PyTuple_SET_ITEM(tb_and_size, 1, PyLong_FromSize_t(tb->size)); - PyList_SET_ITEM(heap_list, i, tb_and_size); - i++; + PyList_SET_ITEM(heap_list, i++, tb_and_size); memalloc_debug_gil_release(); } @@ -97,79 +75,10 @@ memalloc_heap_map::export_to_python() const void memalloc_heap_map::destructive_copy_from(memalloc_heap_map& src) { - HeapSamples_Iter it = HeapSamples_iter(&src.map); - for (const HeapSamples_Entry* e = HeapSamples_Iter_get(&it); e != nullptr; e = HeapSamples_Iter_next(&it)) { - HeapSamples_insert(&map, e); - } - /* Can't erase inside the loop or the iterator is invalidated */ - HeapSamples_clear(&src.map); -} - -// Iterator implementation -memalloc_heap_map::iterator::iterator() - : iter{} -{ -} + // Move all entries from src to this map using merge (C++17) + // This efficiently transfers ownership without copying + map.merge(src.map); -memalloc_heap_map::iterator::iterator(const memalloc_heap_map& map) - : iter(HeapSamples_citer(&map.map)) -{ -} - -memalloc_heap_map::iterator& -memalloc_heap_map::iterator::operator++() -{ - const HeapSamples_Entry* e = HeapSamples_CIter_get(&iter); - if (!e) { - return *this; - } - HeapSamples_CIter_next(&iter); - return *this; -} - -memalloc_heap_map::iterator -memalloc_heap_map::iterator::operator++(int) -{ - iterator tmp = *this; - ++(*this); - return tmp; -} - -memalloc_heap_map::iterator::value_type -memalloc_heap_map::iterator::operator*() const -{ - const HeapSamples_Entry* e = HeapSamples_CIter_get(&iter); - if (!e) { - return { nullptr, nullptr }; - } - return { e->key, e->val }; -} - -bool -memalloc_heap_map::iterator::operator==(const iterator& other) const -{ - // Compare underlying iterators by their current entry pointers - // Note: HeapSamples_CIter doesn't have equality comparison, so we compare - // the current entry pointers. Both end iterators will have nullptr entries. - const HeapSamples_Entry* e1 = HeapSamples_CIter_get(&iter); - const HeapSamples_Entry* e2 = HeapSamples_CIter_get(&other.iter); - return e1 == e2; -} - -bool -memalloc_heap_map::iterator::operator!=(const iterator& other) const -{ - return !(*this == other); -} - -memalloc_heap_map::iterator -memalloc_heap_map::begin() const -{ - return iterator(*this); -} - -memalloc_heap_map::iterator -memalloc_heap_map::end() const -{ - return iterator(); + // Clear any remaining entries in src (shouldn't be any after merge) + src.map.clear(); } diff --git a/ddtrace/profiling/collector/_memalloc_heap_map.hpp b/ddtrace/profiling/collector/_memalloc_heap_map.hpp index 9bd1c60a7e5..7836c83ea30 100644 --- a/ddtrace/profiling/collector/_memalloc_heap_map.hpp +++ b/ddtrace/profiling/collector/_memalloc_heap_map.hpp @@ -7,22 +7,22 @@ #include "_memalloc_tb.h" #include -/* cwisstable.h provides a C implementation of SwissTables hash maps, originally - * implemented in the Abseil C++ library. +/* Use Abseil's flat_hash_map for tracking sampled allocations. + * flat_hash_map provides excellent performance with low memory overhead, + * using the Swiss Tables algorithm from Abseil. * - * This header is was generated from https://github.com/google/cwisstable - * at commit 6de0e5f2e55f90017534a3366198ce7d3e3b7fef - * and lightly modified for our use. - * See "BEGIN MODIFICATION" and "END MODIFICATION" in the header. - * - * The following macro will expand to a type-safe implementation with void* keys - * and traceback_t* values for use by the heap profiler. We encapsulate this - * implementation in a wrapper specialized for use by the heap profiler, both to - * keep compilation fast (the cwisstables header is big) and to allow us to swap - * out the implementation if we want. + * We use a conditional compilation to fall back to std::unordered_map + * when Abseil is not available (e.g., in Debug builds). */ -#include "vendor/cwisstable.h" -CWISS_DECLARE_FLAT_HASHMAP(HeapSamples, void*, traceback_t*); +#if defined(NDEBUG) && !defined(DONT_COMPILE_ABSEIL) +#include "absl/container/flat_hash_map.h" +template +using HeapMapType = absl::flat_hash_map; +#else +#include +template +using HeapMapType = std::unordered_map; +#endif /* memalloc_heap_map tracks sampled allocations by their address. * C++ interface - implementation is in _memalloc_heap_map.cpp @@ -30,60 +30,42 @@ CWISS_DECLARE_FLAT_HASHMAP(HeapSamples, void*, traceback_t*); class memalloc_heap_map { public: - memalloc_heap_map(); + using map_type = HeapMapType; + using iterator = map_type::const_iterator; + using const_iterator = map_type::const_iterator; + + memalloc_heap_map() = default; ~memalloc_heap_map(); // Delete copy constructor and assignment operator memalloc_heap_map(const memalloc_heap_map&) = delete; memalloc_heap_map& operator=(const memalloc_heap_map&) = delete; - size_t size() const; + /* Size of the map */ + size_t size() const { return map.size(); } + + /* Check if key exists in the map */ + bool contains(void* key) const { return map.find(key) != map.end(); } /* Insert a traceback for a sampled allocation with the given address. * If there is already an entry for the given key, the old value will be * replaced with the given value, and the old value will be returned */ traceback_t* insert(void* key, traceback_t* value); - bool contains(void* key) const; - - /* Retrieve the sampled allocation with the given address from m. + /* Retrieve the sampled allocation with the given address from the map. * Returns nullptr if the allocation wasn't found */ traceback_t* remove(void* key); + /* Export the map contents to a Python list */ PyObject* export_to_python() const; /* Copy the contents of src into this map, removing the items from src */ void destructive_copy_from(memalloc_heap_map& src); - class iterator - { - public: - // Iterator traits - using iterator_category = std::forward_iterator_tag; - using value_type = std::pair; - using difference_type = std::ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - - iterator(); - iterator(const memalloc_heap_map& map); - ~iterator() = default; - - // Iterator operations - iterator& operator++(); - iterator operator++(int); - value_type operator*() const; - bool operator==(const iterator& other) const; - bool operator!=(const iterator& other) const; - - private: - HeapSamples_CIter iter; - friend class memalloc_heap_map; - }; - - iterator begin() const; - iterator end() const; + /* Iterators for range-based for loops */ + const_iterator begin() const { return map.begin(); } + const_iterator end() const { return map.end(); } private: - HeapSamples map; + map_type map; }; diff --git a/ddtrace/profiling/collector/test/CMakeLists.txt b/ddtrace/profiling/collector/test/CMakeLists.txt index 35b8f67f869..2a363cad1c7 100644 --- a/ddtrace/profiling/collector/test/CMakeLists.txt +++ b/ddtrace/profiling/collector/test/CMakeLists.txt @@ -1,45 +1,54 @@ -cmake_minimum_required(VERSION 3.10) -project(test_memalloc_heap_map) +cmake_minimum_required(VERSION 3.19) +# Download and configure Google Test include(FetchContent) -FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.15.2) -set(gtest_force_shared_crt - ON - CACHE BOOL "" FORCE) + +# Prevent Google Test from installing (it tries to install to /usr/local which fails in CI) set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) +set(INSTALL_GMOCK + OFF + CACHE BOOL "" FORCE) + +FetchContent_Declare(googletest URL https://github.com/google/googletest/archive/refs/tags/v1.14.0.zip) + +# For Windows: Prevent overriding the parent project's compiler/linker settings +set(gtest_force_shared_crt + ON + CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) -include(GoogleTest) +# Find Python find_package( Python3 COMPONENTS Interpreter Development REQUIRED) -# Add the test executable -add_executable(test_memalloc_heap_map test_memalloc_heap_map.cpp) - -# Include directories -target_include_directories(test_memalloc_heap_map PRIVATE .. ${Python3_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR}/../vendor) +# Test executable for heap map +add_executable(test_memalloc_heap_map ../_memalloc_heap_map.cpp ../_memalloc_tb.cpp test_memalloc_heap_map.cpp) -# Link libraries -target_link_libraries(test_memalloc_heap_map PRIVATE gtest gtest_main ${Python3_LIBRARIES}) +target_include_directories(test_memalloc_heap_map PRIVATE ${Python3_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/..) -# Add source files that need to be compiled -target_sources(test_memalloc_heap_map PRIVATE ../_memalloc_heap_map.cpp ../_memalloc_tb.cpp ../_memalloc_reentrant.cpp) +target_link_libraries(test_memalloc_heap_map PRIVATE GTest::gtest_main Python3::Python) -# Python library linking -if(Python3_LIBRARY) - target_link_libraries(test_memalloc_heap_map PRIVATE ${Python3_LIBRARY}) +# Link Abseil if available +if(NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" + OR (DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}" + STREQUAL "false")))) + target_link_libraries(test_memalloc_heap_map PRIVATE absl::flat_hash_map) endif() -# Compiler flags -target_compile_definitions(test_memalloc_heap_map PRIVATE PY_SSIZE_T_CLEAN) +# Add NDEBUG flag for release builds +if(CMAKE_BUILD_TYPE STREQUAL "Release" + OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" + OR CMAKE_BUILD_TYPE STREQUAL "MinSizeRel") + target_compile_definitions(test_memalloc_heap_map PRIVATE NDEBUG) +else() + target_compile_definitions(test_memalloc_heap_map PRIVATE UNDEBUG) +endif() -# Discover and register tests +# Register the test with CTest +include(GoogleTest) gtest_discover_tests(test_memalloc_heap_map) diff --git a/ddtrace/profiling/collector/vendor/cwisstable.h b/ddtrace/profiling/collector/vendor/cwisstable.h deleted file mode 100644 index 99d62c55898..00000000000 --- a/ddtrace/profiling/collector/vendor/cwisstable.h +++ /dev/null @@ -1,3278 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// THIS IS A GENERATED FILE! DO NOT EDIT DIRECTLY! -// Generated using unify.py, by concatenating, in order: -// #include "cwisstable/internal/base.h" -// #include "cwisstable/internal/bits.h" -// #include "cwisstable/internal/control_byte.h" -// #include "cwisstable/internal/capacity.h" -// #include "cwisstable/internal/probe.h" -// #include "cwisstable/internal/absl_hash.h" -// #include "cwisstable/hash.h" -// #include "cwisstable/internal/extract.h" -// #include "cwisstable/policy.h" -// #include "cwisstable/internal/raw_table.h" -// #include "cwisstable/declare.h" - -#ifndef CWISSTABLE_H_ -#define CWISSTABLE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if _WIN_64 -#include // for __umul128 -#endif - -/// cwisstable/internal/base.h ///////////////////////////////////////////////// -/// Feature detection and basic helper macros. - -/// C++11 compatibility macros. -/// -/// Atomic support, due to incompatibilities between C++ and C11 atomic syntax. -/// - `CWISS_ATOMIC_T(Type)` names an atomic version of `Type`. We must use this -/// instead of `_Atomic(Type)` to name an atomic type. -/// - `CWISS_ATOMIC_INC(value)` will atomically increment `value` without -/// performing synchronization. This is used as a weak entropy source -/// elsewhere. -/// -/// `extern "C"` support via `CWISS_END_EXTERN` and `CWISS_END_EXTERN`, -/// which open and close an `extern "C"` block in C++ mode. -#ifdef __cplusplus - #include - #define CWISS_ATOMIC_T(Type_) std::atomic - #define CWISS_ATOMIC_INC(val_) (val_).fetch_add(1, std::memory_order_relaxed) - - #define CWISS_BEGIN_EXTERN extern "C" { - #define CWISS_END_EXTERN } -#else - #include - #define CWISS_ATOMIC_T(Type_) _Atomic(Type_) - #define CWISS_ATOMIC_INC(val_) \ - atomic_fetch_add_explicit(&(val_), 1, memory_order_relaxed) - - #define CWISS_BEGIN_EXTERN - #define CWISS_END_EXTERN -#endif - -/// Compiler detection macros. -/// -/// The following macros are defined: -/// - `CWISS_IS_CLANG` detects Clang. -/// - `CWISS_IS_GCC` detects GCC (and *not* Clang pretending to be GCC). -/// - `CWISS_IS_MSVC` detects MSCV (and *not* clang-cl). -/// - `CWISS_IS_GCCISH` detects GCC and GCC-mode Clang. -/// - `CWISS_IS_MSVCISH` detects MSVC and clang-cl. -#ifdef __clang__ - #define CWISS_IS_CLANG 1 -#else - #define CWISS_IS_CLANG 0 -#endif -#if defined(__GNUC__) - #define CWISS_IS_GCCISH 1 -#else - #define CWISS_IS_GCCISH 0 -#endif -#if defined(_MSC_VER) - #define CWISS_IS_MSVCISH 1 -#else - #define CWISS_IS_MSVCISH 0 -#endif -#define CWISS_IS_GCC (CWISS_IS_GCCISH && !CWISS_IS_CLANG) -#define CWISS_IS_MSVC (CWISS_IS_MSVCISH && !CWISS_IS_CLANG) - -#define CWISS_PRAGMA(pragma_) _Pragma(#pragma_) - -#if CWISS_IS_GCCISH - #define CWISS_GCC_PUSH CWISS_PRAGMA(GCC diagnostic push) - #define CWISS_GCC_ALLOW(w_) CWISS_PRAGMA(GCC diagnostic ignored w_) - #define CWISS_GCC_POP CWISS_PRAGMA(GCC diagnostic pop) -#else - #define CWISS_GCC_PUSH - #define CWISS_GCC_ALLOW(w_) - #define CWISS_GCC_POP -#endif - -/// Warning control around `CWISS` symbol definitions. These macros will -/// disable certain false-positive warnings that `CWISS` definitions tend to -/// emit. -#define CWISS_BEGIN \ - CWISS_GCC_PUSH \ - CWISS_GCC_ALLOW("-Wunused-function") \ - CWISS_GCC_ALLOW("-Wunused-parameter") \ - CWISS_GCC_ALLOW("-Wcast-qual") \ - CWISS_GCC_ALLOW("-Wmissing-field-initializers") -#define CWISS_END CWISS_GCC_POP - -/// `CWISS_HAVE_SSE2` is nonzero if we have SSE2 support. -/// -/// `-DCWISS_HAVE_SSE2` can be used to override it; it is otherwise detected -/// via the usual non-portable feature-detection macros. -#ifndef CWISS_HAVE_SSE2 - #if defined(__SSE2__) || \ - (CWISS_IS_MSVCISH && \ - (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2))) - #define CWISS_HAVE_SSE2 1 - #else - #define CWISS_HAVE_SSE2 0 - #endif -#endif - -/// `CWISS_HAVE_SSSE2` is nonzero if we have SSSE2 support. -/// -/// `-DCWISS_HAVE_SSSE2` can be used to override it; it is otherwise detected -/// via the usual non-portable feature-detection macros. -#ifndef CWISS_HAVE_SSSE3 - #ifdef __SSSE3__ - #define CWISS_HAVE_SSSE3 1 - #else - #define CWISS_HAVE_SSSE3 0 - #endif -#endif - -#if CWISS_HAVE_SSE2 - #include -#endif - -#if CWISS_HAVE_SSSE3 - #if !CWISS_HAVE_SSE2 - #error "Bad configuration: SSSE3 implies SSE2!" - #endif - #include -#endif - -/// `CWISS_HAVE_BUILTIN` will, in Clang, detect whether a Clang language -/// extension is enabled. -/// -/// See https://clang.llvm.org/docs/LanguageExtensions.html. -#ifdef __has_builtin - #define CWISS_HAVE_CLANG_BUILTIN(x_) __has_builtin(x_) -#else - #define CWISS_HAVE_CLANG_BUILTIN(x_) 0 -#endif - -/// `CWISS_HAVE_GCC_ATTRIBUTE` detects if a particular `__attribute__(())` is -/// present. -/// -/// GCC: https://gcc.gnu.org/gcc-5/changes.html -/// Clang: https://clang.llvm.org/docs/LanguageExtensions.html -#ifdef __has_attribute - #define CWISS_HAVE_GCC_ATTRIBUTE(x_) __has_attribute(x_) -#else - #define CWISS_HAVE_GCC_ATTRIBUTE(x_) 0 -#endif - -#ifdef __has_feature - #define CWISS_HAVE_FEATURE(x_) __has_feature(x_) -#else - #define CWISS_HAVE_FEATURE(x_) 0 -#endif - -/// `CWISS_THREAD_LOCAL` expands to the appropriate TLS annotation, if one is -/// available. -#if CWISS_IS_GCCISH - #define CWISS_THREAD_LOCAL __thread -#endif - -/// `CWISS_CHECK` will evaluate `cond_` and, if false, print an error and crash -/// the program. -/// -/// This is like `assert()` but unconditional. -#define CWISS_CHECK(cond_, ...) \ - do { \ - if (cond_) break; \ - fprintf(stderr, "CWISS_CHECK failed at %s:%d\n", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - fflush(stderr); \ - abort(); \ - } while (false) - -/// `CWISS_DCHECK` is like `CWISS_CHECK` but is disabled by `NDEBUG`. -#ifdef NDEBUG - #define CWISS_DCHECK(cond_, ...) ((void)0) -#else - #define CWISS_DCHECK CWISS_CHECK -#endif - -/// `CWISS_LIKELY` and `CWISS_UNLIKELY` provide a prediction hint to the -/// compiler to encourage branches to be scheduled in a particular way. -#if CWISS_HAVE_CLANG_BUILTIN(__builtin_expect) || CWISS_IS_GCC - #define CWISS_LIKELY(cond_) (__builtin_expect(false || (cond_), true)) - #define CWISS_UNLIKELY(cond_) (__builtin_expect(false || (cond_), false)) -#else - #define CWISS_LIKELY(cond_) (cond_) - #define CWISS_UNLIKELY(cond_) (cond_) -#endif - -/// `CWISS_INLINE_ALWAYS` informs the compiler that it should try really hard -/// to inline a function. -#if CWISS_HAVE_GCC_ATTRIBUTE(always_inline) - #define CWISS_INLINE_ALWAYS __attribute__((always_inline)) -#else - #define CWISS_INLINE_ALWAYS -#endif - -/// `CWISS_INLINE_NEVER` informs the compiler that it should avoid inlining a -/// function. -#if CWISS_HAVE_GCC_ATTRIBUTE(noinline) - #define CWISS_INLINE_NEVER __attribute__((noinline)) -#else - #define CWISS_INLINE_NEVER -#endif - -/// `CWISS_PREFETCH` informs the compiler that it should issue prefetch -/// instructions. -#if CWISS_HAVE_CLANG_BUILTIN(__builtin_prefetch) || CWISS_IS_GCC - #define CWISS_HAVE_PREFETCH 1 - #define CWISS_PREFETCH(addr_, locality_) \ - __builtin_prefetch(addr_, 0, locality_); -#else - #define CWISS_HAVE_PREFETCH 0 - #define CWISS_PREFETCH(addr_, locality_) ((void)0) -#endif - -/// `CWISS_HAVE_ASAN` and `CWISS_HAVE_MSAN` detect the presence of some of the -/// sanitizers. -#if defined(__SANITIZE_ADDRESS__) || CWISS_HAVE_FEATURE(address_sanitizer) - #define CWISS_HAVE_ASAN 1 -#else - #define CWISS_HAVE_ASAN 0 -#endif -#if defined(__SANITIZE_MEMORY__) || \ - (CWISS_HAVE_FEATURE(memory_sanitizer) && !defined(__native_client__)) - #define CWISS_HAVE_MSAN 1 -#else - #define CWISS_HAVE_MSAN 0 -#endif - -#if CWISS_HAVE_ASAN - #include -#endif - -#if CWISS_HAVE_MSAN - #include -#endif - -CWISS_BEGIN -CWISS_BEGIN_EXTERN -/// Informs a sanitizer runtime that this memory is invalid. -static inline void CWISS_PoisonMemory(const void* m, size_t s) { -#if CWISS_HAVE_ASAN - ASAN_POISON_MEMORY_REGION(m, s); -#endif -#if CWISS_HAVE_MSAN - __msan_poison(m, s); -#endif - (void)m; - (void)s; -} - -/// Informs a sanitizer runtime that this memory is no longer invalid. -static inline void CWISS_UnpoisonMemory(const void* m, size_t s) { -#if CWISS_HAVE_ASAN - ASAN_UNPOISON_MEMORY_REGION(m, s); -#endif -#if CWISS_HAVE_MSAN - __msan_unpoison(m, s); -#endif - (void)m; - (void)s; -} -CWISS_END_EXTERN -CWISS_END -/// cwisstable/internal/base.h ///////////////////////////////////////////////// - -/// cwisstable/internal/bits.h ///////////////////////////////////////////////// -/// Bit manipulation utilities. - -CWISS_BEGIN -CWISS_BEGIN_EXTERN - -/// Counts the number of trailing zeroes in the binary representation of `x`. -CWISS_INLINE_ALWAYS -static inline uint32_t CWISS_TrailingZeroes64(uint64_t x) { -#if CWISS_HAVE_CLANG_BUILTIN(__builtin_ctzll) || CWISS_IS_GCC - static_assert(sizeof(unsigned long long) == sizeof(x), - "__builtin_ctzll does not take 64-bit arg"); - return __builtin_ctzll(x); -#elif CWISS_IS_MSVC - unsigned long result = 0; - #if defined(_M_X64) || defined(_M_ARM64) - _BitScanForward64(&result, x); - #else - if (((uint32_t)x) == 0) { - _BitScanForward(&result, (unsigned long)(x >> 32)); - return result + 32; - } - _BitScanForward(&result, (unsigned long)(x)); - #endif - return result; -#else - uint32_t c = 63; - x &= ~x + 1; - if (x & 0x00000000FFFFFFFF) c -= 32; - if (x & 0x0000FFFF0000FFFF) c -= 16; - if (x & 0x00FF00FF00FF00FF) c -= 8; - if (x & 0x0F0F0F0F0F0F0F0F) c -= 4; - if (x & 0x3333333333333333) c -= 2; - if (x & 0x5555555555555555) c -= 1; - return c; -#endif -} - -/// Counts the number of leading zeroes in the binary representation of `x`. -CWISS_INLINE_ALWAYS -static inline uint32_t CWISS_LeadingZeroes64(uint64_t x) { -#if CWISS_HAVE_CLANG_BUILTIN(__builtin_clzll) || CWISS_IS_GCC - static_assert(sizeof(unsigned long long) == sizeof(x), - "__builtin_clzll does not take 64-bit arg"); - // Handle 0 as a special case because __builtin_clzll(0) is undefined. - return x == 0 ? 64 : __builtin_clzll(x); -#elif CWISS_IS_MSVC - unsigned long result = 0; - #if defined(_M_X64) || defined(_M_ARM64) - if (_BitScanReverse64(&result, x)) { - return 63 - result; - } - #else - // BEGIN MODIFICATION - // unsigned long result = 0; // multiple definition - // END MODIFICATION - if ((x >> 32) && _BitScanReverse(&result, (unsigned long)(x >> 32))) { - return 31 - result; - } - // BEGIN MODIFICATION - if (_BitScanReverse(&result, (unsigned long)(x))) { - return 63 - result; - } - // END MODIFICATION - #endif - return 64; -#else - uint32_t zeroes = 60; - if (x >> 32) { - zeroes -= 32; - x >>= 32; - } - if (x >> 16) { - zeroes -= 16; - x >>= 16; - } - if (x >> 8) { - zeroes -= 8; - x >>= 8; - } - if (x >> 4) { - zeroes -= 4; - x >>= 4; - } - return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[x] + zeroes; -#endif -} - -/// Counts the number of trailing zeroes in the binary representation of `x_` in -/// a type-generic fashion. -#define CWISS_TrailingZeros(x_) (CWISS_TrailingZeroes64(x_)) - -/// Counts the number of leading zeroes in the binary representation of `x_` in -/// a type-generic fashion. -#define CWISS_LeadingZeros(x_) \ - (CWISS_LeadingZeroes64(x_) - \ - (uint32_t)((sizeof(unsigned long long) - sizeof(x_)) * 8)) - -/// Computes the number of bits necessary to represent `x_`, i.e., the bit index -/// of the most significant one. -#define CWISS_BitWidth(x_) \ - (((uint32_t)(sizeof(x_) * 8)) - CWISS_LeadingZeros(x_)) - -#define CWISS_RotateLeft(x_, bits_) \ - (((x_) << bits_) | ((x_) >> (sizeof(x_) * 8 - bits_))) - -/// The return type of `CWISS_Mul128`. -typedef struct { - uint64_t lo, hi; -} CWISS_U128; - -/// Computes a double-width multiplication operation. -static inline CWISS_U128 CWISS_Mul128(uint64_t a, uint64_t b) { - // BEGIN MODIFICATION -#if defined(_WIN_64) - uint64_t hi = 0; - // __umul128 is defined in intrin.h - uint64_t lo = __umul128(a, b, &hi); - return (CWISS_U128){lo, hi}; -#elif defined(__x86_64__) || defined(__aarch_64__) - __uint128_t p = a; - p *= b; - return (CWISS_U128){(uint64_t)p, (uint64_t)(p >> 64)}; -#else - // If this is compiled for a 32-bit processor, - // we're going to use a custom policy, not this function. - return (CWISS_U128){0,0}; -#endif - // END MODIFICATION -} - -/// Loads an unaligned u32. -static inline uint32_t CWISS_Load32(const void* p) { - uint32_t v; - memcpy(&v, p, sizeof(v)); - return v; -} - -/// Loads an unaligned u64. -static inline uint64_t CWISS_Load64(const void* p) { - uint64_t v; - memcpy(&v, p, sizeof(v)); - return v; -} - -/// Reads 9 to 16 bytes from p. -static inline CWISS_U128 CWISS_Load9To16(const void* p, size_t len) { - const unsigned char* p8 = (const unsigned char*)p; - uint64_t lo = CWISS_Load64(p8); - uint64_t hi = CWISS_Load64(p8 + len - 8); - return (CWISS_U128){lo, hi >> (128 - len * 8)}; -} - -/// Reads 4 to 8 bytes from p. -static inline uint64_t CWISS_Load4To8(const void* p, size_t len) { - const unsigned char* p8 = (const unsigned char*)p; - uint64_t lo = CWISS_Load32(p8); - uint64_t hi = CWISS_Load32(p8 + len - 4); - return lo | (hi << (len - 4) * 8); -} - -/// Reads 1 to 3 bytes from p. -static inline uint32_t CWISS_Load1To3(const void* p, size_t len) { - const unsigned char* p8 = (const unsigned char*)p; - uint32_t mem0 = p8[0]; - uint32_t mem1 = p8[len / 2]; - uint32_t mem2 = p8[len - 1]; - return (mem0 | (mem1 << (len / 2 * 8)) | (mem2 << ((len - 1) * 8))); -} - -/// A abstract bitmask, such as that emitted by a SIMD instruction. -/// -/// Specifically, this type implements a simple bitset whose representation is -/// controlled by `width` and `shift`. `width` is the number of abstract bits in -/// the bitset, while `shift` is the log-base-two of the width of an abstract -/// bit in the representation. -/// -/// For example, when `width` is 16 and `shift` is zero, this is just an -/// ordinary 16-bit bitset occupying the low 16 bits of `mask`. When `width` is -/// 8 and `shift` is 3, abstract bits are represented as the bytes `0x00` and -/// `0x80`, and it occupies all 64 bits of the bitmask. -typedef struct { - /// The mask, in the representation specified by `width` and `shift`. - uint64_t mask; - /// The number of abstract bits in the mask. - uint32_t width; - /// The log-base-two width of an abstract bit. - uint32_t shift; -} CWISS_BitMask; - -/// Returns the index of the lowest abstract bit set in `self`. -static inline uint32_t CWISS_BitMask_LowestBitSet(const CWISS_BitMask* self) { - return CWISS_TrailingZeros(self->mask) >> self->shift; -} - -/// Returns the index of the highest abstract bit set in `self`. -static inline uint32_t CWISS_BitMask_HighestBitSet(const CWISS_BitMask* self) { - return (uint32_t)(CWISS_BitWidth(self->mask) - 1) >> self->shift; -} - -/// Return the number of trailing zero abstract bits. -static inline uint32_t CWISS_BitMask_TrailingZeros(const CWISS_BitMask* self) { - return CWISS_TrailingZeros(self->mask) >> self->shift; -} - -/// Return the number of leading zero abstract bits. -static inline uint32_t CWISS_BitMask_LeadingZeros(const CWISS_BitMask* self) { - uint32_t total_significant_bits = self->width << self->shift; - uint32_t extra_bits = sizeof(self->mask) * 8 - total_significant_bits; - return (uint32_t)(CWISS_LeadingZeros(self->mask << extra_bits)) >> - self->shift; -} - -/// Iterates over the one bits in the mask. -/// -/// If the mask is empty, returns `false`; otherwise, returns the index of the -/// lowest one bit in the mask, and removes it from the set. -static inline bool CWISS_BitMask_next(CWISS_BitMask* self, uint32_t* bit) { - if (self->mask == 0) { - return false; - } - - *bit = CWISS_BitMask_LowestBitSet(self); - self->mask &= (self->mask - 1); - return true; -} - -CWISS_END_EXTERN -CWISS_END -/// cwisstable/internal/bits.h ///////////////////////////////////////////////// - -/// cwisstable/internal/control_byte.h ///////////////////////////////////////// -CWISS_BEGIN -CWISS_BEGIN_EXTERN - -/// Control bytes and groups: the core of SwissTable optimization. -/// -/// Control bytes are bytes (collected into groups of a platform-specific size) -/// that define the state of the corresponding slot in the slot array. Group -/// manipulation is tightly optimized to be as efficient as possible. - -/// A `CWISS_ControlByte` is a single control byte, which can have one of four -/// states: empty, deleted, full (which has an associated seven-bit hash) and -/// the sentinel. They have the following bit patterns: -/// -/// ``` -/// empty: 1 0 0 0 0 0 0 0 -/// deleted: 1 1 1 1 1 1 1 0 -/// full: 0 h h h h h h h // h represents the hash bits. -/// sentinel: 1 1 1 1 1 1 1 1 -/// ``` -/// -/// These values are specifically tuned for SSE-flavored SIMD; future ports to -/// other SIMD platforms may require choosing new values. The static_asserts -/// below detail the source of these choices. -typedef int8_t CWISS_ControlByte; -#define CWISS_kEmpty (INT8_C(-128)) -#define CWISS_kDeleted (INT8_C(-2)) -#define CWISS_kSentinel (INT8_C(-1)) -// TODO: Wrap CWISS_ControlByte in a single-field struct to get strict-aliasing -// benefits. - -static_assert( - (CWISS_kEmpty & CWISS_kDeleted & CWISS_kSentinel & 0x80) != 0, - "Special markers need to have the MSB to make checking for them efficient"); -static_assert( - CWISS_kEmpty < CWISS_kSentinel && CWISS_kDeleted < CWISS_kSentinel, - "CWISS_kEmpty and CWISS_kDeleted must be smaller than " - "CWISS_kSentinel to make the SIMD test of IsEmptyOrDeleted() efficient"); -static_assert( - CWISS_kSentinel == -1, - "CWISS_kSentinel must be -1 to elide loading it from memory into SIMD " - "registers (pcmpeqd xmm, xmm)"); -static_assert(CWISS_kEmpty == -128, - "CWISS_kEmpty must be -128 to make the SIMD check for its " - "existence efficient (psignb xmm, xmm)"); -static_assert( - (~CWISS_kEmpty & ~CWISS_kDeleted & CWISS_kSentinel & 0x7F) != 0, - "CWISS_kEmpty and CWISS_kDeleted must share an unset bit that is not " - "shared by CWISS_kSentinel to make the scalar test for " - "MatchEmptyOrDeleted() efficient"); -static_assert(CWISS_kDeleted == -2, - "CWISS_kDeleted must be -2 to make the implementation of " - "ConvertSpecialToEmptyAndFullToDeleted efficient"); - -/// Returns a pointer to a control byte group that can be used by empty tables. -static inline CWISS_ControlByte* CWISS_EmptyGroup() { - // A single block of empty control bytes for tables without any slots - // allocated. This enables removing a branch in the hot path of find(). - alignas(16) static const CWISS_ControlByte kEmptyGroup[16] = { - CWISS_kSentinel, CWISS_kEmpty, CWISS_kEmpty, CWISS_kEmpty, - CWISS_kEmpty, CWISS_kEmpty, CWISS_kEmpty, CWISS_kEmpty, - CWISS_kEmpty, CWISS_kEmpty, CWISS_kEmpty, CWISS_kEmpty, - CWISS_kEmpty, CWISS_kEmpty, CWISS_kEmpty, CWISS_kEmpty, - }; - - // Const must be cast away here; no uses of this function will actually write - // to it, because it is only used for empty tables. - return (CWISS_ControlByte*)&kEmptyGroup; -} - -/// Returns a hash seed. -/// -/// The seed consists of the ctrl_ pointer, which adds enough entropy to ensure -/// non-determinism of iteration order in most cases. -static inline size_t CWISS_HashSeed(const CWISS_ControlByte* ctrl) { - // The low bits of the pointer have little or no entropy because of - // alignment. We shift the pointer to try to use higher entropy bits. A - // good number seems to be 12 bits, because that aligns with page size. - return ((uintptr_t)ctrl) >> 12; -} - -/// Extracts the H1 portion of a hash: the high 57 bits mixed with a per-table -/// salt. -static inline size_t CWISS_H1(size_t hash, const CWISS_ControlByte* ctrl) { - return (hash >> 7) ^ CWISS_HashSeed(ctrl); -} - -/// Extracts the H2 portion of a hash: the low 7 bits, which can be used as -/// control byte. -typedef uint8_t CWISS_h2_t; -static inline CWISS_h2_t CWISS_H2(size_t hash) { return hash & 0x7F; } - -/// Returns whether `c` is empty. -static inline bool CWISS_IsEmpty(CWISS_ControlByte c) { - return c == CWISS_kEmpty; -} - -/// Returns whether `c` is full. -static inline bool CWISS_IsFull(CWISS_ControlByte c) { return c >= 0; } - -/// Returns whether `c` is deleted. -static inline bool CWISS_IsDeleted(CWISS_ControlByte c) { - return c == CWISS_kDeleted; -} - -/// Returns whether `c` is empty or deleted. -static inline bool CWISS_IsEmptyOrDeleted(CWISS_ControlByte c) { - return c < CWISS_kSentinel; -} - -/// Asserts that `ctrl` points to a full control byte. -#define CWISS_AssertIsFull(ctrl) \ - CWISS_CHECK((ctrl) != NULL && CWISS_IsFull(*(ctrl)), \ - "Invalid operation on iterator (%p/%d). The element might have " \ - "been erased, or the table might have rehashed.", \ - (ctrl), (ctrl) ? *(ctrl) : -1) - -/// Asserts that `ctrl` is either null OR points to a full control byte. -#define CWISS_AssertIsValid(ctrl) \ - CWISS_CHECK((ctrl) == NULL || CWISS_IsFull(*(ctrl)), \ - "Invalid operation on iterator (%p/%d). The element might have " \ - "been erased, or the table might have rehashed.", \ - (ctrl), (ctrl) ? *(ctrl) : -1) - -/// Constructs a `BitMask` with the correct parameters for whichever -/// implementation of `CWISS_Group` is in use. -#define CWISS_Group_BitMask(x) \ - (CWISS_BitMask){(uint64_t)(x), CWISS_Group_kWidth, CWISS_Group_kShift}; - -// TODO(#4): Port this to NEON. -#if CWISS_HAVE_SSE2 -// Reference guide for intrinsics used below: -// -// * __m128i: An XMM (128-bit) word. -// -// * _mm_setzero_si128: Returns a zero vector. -// * _mm_set1_epi8: Returns a vector with the same i8 in each lane. -// -// * _mm_subs_epi8: Saturating-subtracts two i8 vectors. -// * _mm_and_si128: Ands two i128s together. -// * _mm_or_si128: Ors two i128s together. -// * _mm_andnot_si128: And-nots two i128s together. -// -// * _mm_cmpeq_epi8: Component-wise compares two i8 vectors for equality, -// filling each lane with 0x00 or 0xff. -// * _mm_cmpgt_epi8: Same as above, but using > rather than ==. -// -// * _mm_loadu_si128: Performs an unaligned load of an i128. -// * _mm_storeu_si128: Performs an unaligned store of a u128. -// -// * _mm_sign_epi8: Retains, negates, or zeroes each i8 lane of the first -// argument if the corresponding lane of the second -// argument is positive, negative, or zero, respectively. -// * _mm_movemask_epi8: Selects the sign bit out of each i8 lane and produces a -// bitmask consisting of those bits. -// * _mm_shuffle_epi8: Selects i8s from the first argument, using the low -// four bits of each i8 lane in the second argument as -// indices. -typedef __m128i CWISS_Group; - #define CWISS_Group_kWidth ((size_t)16) - #define CWISS_Group_kShift 0 - -// https://github.com/abseil/abseil-cpp/issues/209 -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 -// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char -// Work around this by using the portable implementation of Group -// when using -funsigned-char under GCC. -static inline CWISS_Group CWISS_mm_cmpgt_epi8_fixed(CWISS_Group a, - CWISS_Group b) { - if (CWISS_IS_GCC && CHAR_MIN == 0) { // std::is_unsigned_v - const CWISS_Group mask = _mm_set1_epi8(0x80); - const CWISS_Group diff = _mm_subs_epi8(b, a); - return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); - } - return _mm_cmpgt_epi8(a, b); -} - -static inline CWISS_Group CWISS_Group_new(const CWISS_ControlByte* pos) { - return _mm_loadu_si128((const CWISS_Group*)pos); -} - -// Returns a bitmask representing the positions of slots that match hash. -static inline CWISS_BitMask CWISS_Group_Match(const CWISS_Group* self, - CWISS_h2_t hash) { - return CWISS_Group_BitMask( - _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_set1_epi8(hash), *self))) -} - -// Returns a bitmask representing the positions of empty slots. -static inline CWISS_BitMask CWISS_Group_MatchEmpty(const CWISS_Group* self) { - #if CWISS_HAVE_SSSE3 - // This only works because ctrl_t::kEmpty is -128. - return CWISS_Group_BitMask(_mm_movemask_epi8(_mm_sign_epi8(*self, *self))); - #else - return CWISS_Group_Match(self, CWISS_kEmpty); - #endif -} - -// Returns a bitmask representing the positions of empty or deleted slots. -static inline CWISS_BitMask CWISS_Group_MatchEmptyOrDeleted( - const CWISS_Group* self) { - CWISS_Group special = _mm_set1_epi8((uint8_t)CWISS_kSentinel); - return CWISS_Group_BitMask( - _mm_movemask_epi8(CWISS_mm_cmpgt_epi8_fixed(special, *self))); -} - -// Returns the number of trailing empty or deleted elements in the group. -static inline uint32_t CWISS_Group_CountLeadingEmptyOrDeleted( - const CWISS_Group* self) { - CWISS_Group special = _mm_set1_epi8((uint8_t)CWISS_kSentinel); - return CWISS_TrailingZeros((uint32_t)( - _mm_movemask_epi8(CWISS_mm_cmpgt_epi8_fixed(special, *self)) + 1)); -} - -static inline void CWISS_Group_ConvertSpecialToEmptyAndFullToDeleted( - const CWISS_Group* self, CWISS_ControlByte* dst) { - CWISS_Group msbs = _mm_set1_epi8((char)-128); - CWISS_Group x126 = _mm_set1_epi8(126); - #if CWISS_HAVE_SSSE3 - CWISS_Group res = _mm_or_si128(_mm_shuffle_epi8(x126, *self), msbs); - #else - CWISS_Group zero = _mm_setzero_si128(); - CWISS_Group special_mask = CWISS_mm_cmpgt_epi8_fixed(zero, *self); - CWISS_Group res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); - #endif - _mm_storeu_si128((CWISS_Group*)dst, res); -} -#else // CWISS_HAVE_SSE2 -typedef uint64_t CWISS_Group; - #define CWISS_Group_kWidth ((size_t)8) - #define CWISS_Group_kShift 3 - -// NOTE: Endian-hostile. -static inline CWISS_Group CWISS_Group_new(const CWISS_ControlByte* pos) { - CWISS_Group val; - memcpy(&val, pos, sizeof(val)); - return val; -} - -static inline CWISS_BitMask CWISS_Group_Match(const CWISS_Group* self, - CWISS_h2_t hash) { - // For the technique, see: - // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord - // (Determine if a word has a byte equal to n). - // - // Caveat: there are false positives but: - // - they only occur if there is a real match - // - they never occur on ctrl_t::kEmpty, ctrl_t::kDeleted, ctrl_t::kSentinel - // - they will be handled gracefully by subsequent checks in code - // - // Example: - // v = 0x1716151413121110 - // hash = 0x12 - // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 - uint64_t msbs = 0x8080808080808080ULL; - uint64_t lsbs = 0x0101010101010101ULL; - uint64_t x = *self ^ (lsbs * hash); - return CWISS_Group_BitMask((x - lsbs) & ~x & msbs); -} - -static inline CWISS_BitMask CWISS_Group_MatchEmpty(const CWISS_Group* self) { - uint64_t msbs = 0x8080808080808080ULL; - return CWISS_Group_BitMask((*self & (~*self << 6)) & msbs); -} - -static inline CWISS_BitMask CWISS_Group_MatchEmptyOrDeleted( - const CWISS_Group* self) { - uint64_t msbs = 0x8080808080808080ULL; - return CWISS_Group_BitMask((*self & (~*self << 7)) & msbs); -} - -static inline uint32_t CWISS_Group_CountLeadingEmptyOrDeleted( - const CWISS_Group* self) { - uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; - return (CWISS_TrailingZeros(((~*self & (*self >> 7)) | gaps) + 1) + 7) >> 3; -} - -static inline void CWISS_Group_ConvertSpecialToEmptyAndFullToDeleted( - const CWISS_Group* self, CWISS_ControlByte* dst) { - uint64_t msbs = 0x8080808080808080ULL; - uint64_t lsbs = 0x0101010101010101ULL; - uint64_t x = *self & msbs; - uint64_t res = (~x + (x >> 7)) & ~lsbs; - memcpy(dst, &res, sizeof(res)); -} -#endif // CWISS_HAVE_SSE2 - -CWISS_END_EXTERN -CWISS_END -/// cwisstable/internal/control_byte.h ///////////////////////////////////////// - -/// cwisstable/internal/capacity.h ///////////////////////////////////////////// -/// Capacity, load factor, and allocation size computations for a SwissTable. -/// -/// A SwissTable's backing array consists of control bytes followed by slots -/// that may or may not contain objects. -/// -/// The layout of the backing array, for `capacity` slots, is thus, as a -/// pseudo-struct: -/// ``` -/// struct CWISS_BackingArray { -/// // Control bytes for the "real" slots. -/// CWISS_ControlByte ctrl[capacity]; -/// // Always `CWISS_kSentinel`. This is used by iterators to find when to -/// // stop and serves no other purpose. -/// CWISS_ControlByte sentinel; -/// // A copy of the first `kWidth - 1` elements of `ctrl`. This is used so -/// // that if a probe sequence picks a value near the end of `ctrl`, -/// // `CWISS_Group` will have valid control bytes to look at. -/// // -/// // As an interesting special-case, such probe windows will never choose -/// // the zeroth slot as a candidate, because they will see `kSentinel` -/// // instead of the correct H2 value. -/// CWISS_ControlByte clones[kWidth - 1]; -/// // Alignment padding equal to `alignof(slot_type)`. -/// char padding_; -/// // The actual slot data. -/// char slots[capacity * sizeof(slot_type)]; -/// }; -/// ``` -/// -/// The length of this array is computed by `CWISS_AllocSize()`. - -CWISS_BEGIN -CWISS_BEGIN_EXTERN - -/// Returns he number of "cloned control bytes". -/// -/// This is the number of control bytes that are present both at the beginning -/// of the control byte array and at the end, such that we can create a -/// `CWISS_Group_kWidth`-width probe window starting from any control byte. -static inline size_t CWISS_NumClonedBytes(void) { - return CWISS_Group_kWidth - 1; -} - -/// Returns whether `n` is a valid capacity (i.e., number of slots). -/// -/// A valid capacity is a non-zero integer `2^m - 1`. -static inline bool CWISS_IsValidCapacity(size_t n) { - return ((n + 1) & n) == 0 && n > 0; -} - -/// Returns some per-call entropy. -/// -/// Currently, the entropy is produced by XOR'ing the address of a (preferably -/// thread-local) value with a perpetually-incrementing value. -static inline size_t RandomSeed(void) { -#ifdef CWISS_THREAD_LOCAL - static CWISS_THREAD_LOCAL size_t counter; - size_t value = ++counter; -#else - static volatile CWISS_ATOMIC_T(size_t) counter; - size_t value = CWISS_ATOMIC_INC(counter); -#endif - return value ^ ((size_t)&counter); -} - -/// Mixes a randomly generated per-process seed with `hash` and `ctrl` to -/// randomize insertion order within groups. -CWISS_INLINE_NEVER static bool CWISS_ShouldInsertBackwards( - size_t hash, const CWISS_ControlByte* ctrl) { - // To avoid problems with weak hashes and single bit tests, we use % 13. - // TODO(kfm,sbenza): revisit after we do unconditional mixing - return (CWISS_H1(hash, ctrl) ^ RandomSeed()) % 13 > 6; -} - -/// Applies the following mapping to every byte in the control array: -/// * kDeleted -> kEmpty -/// * kEmpty -> kEmpty -/// * _ -> kDeleted -/// -/// Preconditions: `CWISS_IsValidCapacity(capacity)`, -/// `ctrl[capacity]` == `kSentinel`, `ctrl[i] != kSentinel for i < capacity`. -CWISS_INLINE_NEVER static void CWISS_ConvertDeletedToEmptyAndFullToDeleted( - CWISS_ControlByte* ctrl, size_t capacity) { - CWISS_DCHECK(ctrl[capacity] == CWISS_kSentinel, "bad ctrl value at %zu: %02x", - capacity, ctrl[capacity]); - CWISS_DCHECK(CWISS_IsValidCapacity(capacity), "invalid capacity: %zu", - capacity); - - for (CWISS_ControlByte* pos = ctrl; pos < ctrl + capacity; - pos += CWISS_Group_kWidth) { - CWISS_Group g = CWISS_Group_new(pos); - CWISS_Group_ConvertSpecialToEmptyAndFullToDeleted(&g, pos); - } - // Copy the cloned ctrl bytes. - memcpy(ctrl + capacity + 1, ctrl, CWISS_NumClonedBytes()); - ctrl[capacity] = CWISS_kSentinel; -} - -/// Sets `ctrl` to `{kEmpty, ..., kEmpty, kSentinel}`, marking the entire -/// array as deleted. -static inline void CWISS_ResetCtrl(size_t capacity, CWISS_ControlByte* ctrl, - const void* slots, size_t slot_size) { - memset(ctrl, CWISS_kEmpty, capacity + 1 + CWISS_NumClonedBytes()); - ctrl[capacity] = CWISS_kSentinel; - CWISS_PoisonMemory(slots, slot_size * capacity); -} - -/// Sets `ctrl[i]` to `h`. -/// -/// Unlike setting it directly, this function will perform bounds checks and -/// mirror the value to the cloned tail if necessary. -static inline void CWISS_SetCtrl(size_t i, CWISS_ControlByte h, size_t capacity, - CWISS_ControlByte* ctrl, const void* slots, - size_t slot_size) { - CWISS_DCHECK(i < capacity, "CWISS_SetCtrl out-of-bounds: %zu >= %zu", i, - capacity); - - const char* slot = ((const char*)slots) + i * slot_size; - if (CWISS_IsFull(h)) { - CWISS_UnpoisonMemory(slot, slot_size); - } else { - CWISS_PoisonMemory(slot, slot_size); - } - - // This is intentionally branchless. If `i < kWidth`, it will write to the - // cloned bytes as well as the "real" byte; otherwise, it will store `h` - // twice. - size_t mirrored_i = ((i - CWISS_NumClonedBytes()) & capacity) + - (CWISS_NumClonedBytes() & capacity); - ctrl[i] = h; - ctrl[mirrored_i] = h; -} - -/// Converts `n` into the next valid capacity, per `CWISS_IsValidCapacity`. -static inline size_t CWISS_NormalizeCapacity(size_t n) { - return n ? SIZE_MAX >> CWISS_LeadingZeros(n) : 1; -} - -// General notes on capacity/growth methods below: -// - We use 7/8th as maximum load factor. For 16-wide groups, that gives an -// average of two empty slots per group. -// - For (capacity+1) >= Group::kWidth, growth is 7/8*capacity. -// - For (capacity+1) < Group::kWidth, growth == capacity. In this case, we -// never need to probe (the whole table fits in one group) so we don't need a -// load factor less than 1. - -/// Given `capacity`, applies the load factor; i.e., it returns the maximum -/// number of values we should put into the table before a rehash. -static inline size_t CWISS_CapacityToGrowth(size_t capacity) { - CWISS_DCHECK(CWISS_IsValidCapacity(capacity), "invalid capacity: %zu", - capacity); - // `capacity*7/8` - if (CWISS_Group_kWidth == 8 && capacity == 7) { - // x-x/8 does not work when x==7. - return 6; - } - return capacity - capacity / 8; -} - -/// Given `growth`, "unapplies" the load factor to find how large the capacity -/// should be to stay within the load factor. -/// -/// This might not be a valid capacity and `CWISS_NormalizeCapacity()` may be -/// necessary. -static inline size_t CWISS_GrowthToLowerboundCapacity(size_t growth) { - // `growth*8/7` - if (CWISS_Group_kWidth == 8 && growth == 7) { - // x+(x-1)/7 does not work when x==7. - return 8; - } - return growth + (size_t)((((int64_t)growth) - 1) / 7); -} - -// The allocated block consists of `capacity + 1 + NumClonedBytes()` control -// bytes followed by `capacity` slots, which must be aligned to `slot_align`. -// SlotOffset returns the offset of the slots into the allocated block. - -/// Given the capacity of a table, computes the offset (from the start of the -/// backing allocation) at which the slots begin. -static inline size_t CWISS_SlotOffset(size_t capacity, size_t slot_align) { - CWISS_DCHECK(CWISS_IsValidCapacity(capacity), "invalid capacity: %zu", - capacity); - const size_t num_control_bytes = capacity + 1 + CWISS_NumClonedBytes(); - return (num_control_bytes + slot_align - 1) & (~slot_align + 1); -} - -/// Given the capacity of a table, computes the total size of the backing -/// array. -static inline size_t CWISS_AllocSize(size_t capacity, size_t slot_size, - size_t slot_align) { - return CWISS_SlotOffset(capacity, slot_align) + capacity * slot_size; -} - -/// Whether a table is "small". A small table fits entirely into a probing -/// group, i.e., has a capacity equal to the size of a `CWISS_Group`. -/// -/// In small mode we are able to use the whole capacity. The extra control -/// bytes give us at least one "empty" control byte to stop the iteration. -/// This is important to make 1 a valid capacity. -/// -/// In small mode only the first `capacity` control bytes after the sentinel -/// are valid. The rest contain dummy ctrl_t::kEmpty values that do not -/// represent a real slot. This is important to take into account on -/// `CWISS_FindFirstNonFull()`, where we never try -/// `CWISS_ShouldInsertBackwards()` for small tables. -static inline bool CWISS_IsSmall(size_t capacity) { - return capacity < CWISS_Group_kWidth - 1; -} - -CWISS_END_EXTERN -CWISS_END -/// cwisstable/internal/capacity.h ///////////////////////////////////////////// - -/// cwisstable/internal/probe.h //////////////////////////////////////////////// -/// Table probing functions. -/// -/// "Probing" refers to the process of trying to find the matching entry for a -/// given lookup by repeatedly searching for values throughout the table. - -CWISS_BEGIN -CWISS_BEGIN_EXTERN - -/// The state for a probe sequence. -/// -/// Currently, the sequence is a triangular progression of the form -/// ``` -/// p(i) := kWidth/2 * (i^2 - i) + hash (mod mask + 1) -/// ``` -/// -/// The use of `kWidth` ensures that each probe step does not overlap groups; -/// the sequence effectively outputs the addresses of *groups* (although not -/// necessarily aligned to any boundary). The `CWISS_Group` machinery allows us -/// to check an entire group with minimal branching. -/// -/// Wrapping around at `mask + 1` is important, but not for the obvious reason. -/// As described in capacity.h, the first few entries of the control byte array -/// is mirrored at the end of the array, which `CWISS_Group` will find and use -/// for selecting candidates. However, when those candidates' slots are -/// actually inspected, there are no corresponding slots for the cloned bytes, -/// so we need to make sure we've treated those offsets as "wrapping around". -typedef struct { - size_t mask_; - size_t offset_; - size_t index_; -} CWISS_ProbeSeq; - -/// Creates a new probe sequence using `hash` as the initial value of the -/// sequence and `mask` (usually the capacity of the table) as the mask to -/// apply to each value in the progression. -static inline CWISS_ProbeSeq CWISS_ProbeSeq_new(size_t hash, size_t mask) { - return (CWISS_ProbeSeq){ - .mask_ = mask, - .offset_ = hash & mask, - }; -} - -/// Returns the slot `i` indices ahead of `self` within the bounds expressed by -/// `mask`. -static inline size_t CWISS_ProbeSeq_offset(const CWISS_ProbeSeq* self, - size_t i) { - return (self->offset_ + i) & self->mask_; -} - -/// Advances the sequence; the value can be obtained by calling -/// `CWISS_ProbeSeq_offset()` or inspecting `offset_`. -static inline void CWISS_ProbeSeq_next(CWISS_ProbeSeq* self) { - self->index_ += CWISS_Group_kWidth; - self->offset_ += self->index_; - self->offset_ &= self->mask_; -} - -/// Begins a probing operation on `ctrl`, using `hash`. -static inline CWISS_ProbeSeq CWISS_ProbeSeq_Start(const CWISS_ControlByte* ctrl, - size_t hash, - size_t capacity) { - return CWISS_ProbeSeq_new(CWISS_H1(hash, ctrl), capacity); -} - -// The return value of `CWISS_FindFirstNonFull()`. -typedef struct { - size_t offset; - size_t probe_length; -} CWISS_FindInfo; - -/// Probes an array of control bits using a probe sequence derived from `hash`, -/// and returns the offset corresponding to the first deleted or empty slot. -/// -/// Behavior when the entire table is full is undefined. -/// -/// NOTE: this function must work with tables having both empty and deleted -/// slots in the same group. Such tables appear during -/// `CWISS_RawTable_DropDeletesWithoutResize()`. -static inline CWISS_FindInfo CWISS_FindFirstNonFull( - const CWISS_ControlByte* ctrl, size_t hash, size_t capacity) { - CWISS_ProbeSeq seq = CWISS_ProbeSeq_Start(ctrl, hash, capacity); - while (true) { - CWISS_Group g = CWISS_Group_new(ctrl + seq.offset_); - CWISS_BitMask mask = CWISS_Group_MatchEmptyOrDeleted(&g); - if (mask.mask) { -#ifndef NDEBUG - // We want to add entropy even when ASLR is not enabled. - // In debug build we will randomly insert in either the front or back of - // the group. - // TODO(kfm,sbenza): revisit after we do unconditional mixing - if (!CWISS_IsSmall(capacity) && CWISS_ShouldInsertBackwards(hash, ctrl)) { - return (CWISS_FindInfo){ - CWISS_ProbeSeq_offset(&seq, CWISS_BitMask_HighestBitSet(&mask)), - seq.index_}; - } -#endif - return (CWISS_FindInfo){ - CWISS_ProbeSeq_offset(&seq, CWISS_BitMask_TrailingZeros(&mask)), - seq.index_}; - } - CWISS_ProbeSeq_next(&seq); - CWISS_DCHECK(seq.index_ <= capacity, "full table!"); - } -} - -CWISS_END_EXTERN -CWISS_END -/// cwisstable/internal/probe.h //////////////////////////////////////////////// - -/// cwisstable/internal/absl_hash.h //////////////////////////////////////////// -/// Implementation details of AbslHash. - -CWISS_BEGIN -CWISS_BEGIN_EXTERN - -static inline uint64_t CWISS_AbslHash_LowLevelMix(uint64_t v0, uint64_t v1) { -#ifndef __aarch64__ - // The default bit-mixer uses 64x64->128-bit multiplication. - CWISS_U128 p = CWISS_Mul128(v0, v1); - return p.hi ^ p.lo; -#else - // The default bit-mixer above would perform poorly on some ARM microarchs, - // where calculating a 128-bit product requires a sequence of two - // instructions with a high combined latency and poor throughput. - // Instead, we mix bits using only 64-bit arithmetic, which is faster. - uint64_t p = v0 ^ CWISS_RotateLeft(v1, 40); - p *= v1 ^ CWISS_RotateLeft(v0, 39); - return p ^ (p >> 11); -#endif -} - -CWISS_INLINE_NEVER -static uint64_t CWISS_AbslHash_LowLevelHash(const void* data, size_t len, - uint64_t seed, - const uint64_t salt[5]) { - const char* ptr = (const char*)data; - uint64_t starting_length = (uint64_t)len; - uint64_t current_state = seed ^ salt[0]; - - if (len > 64) { - // If we have more than 64 bytes, we're going to handle chunks of 64 - // bytes at a time. We're going to build up two separate hash states - // which we will then hash together. - uint64_t duplicated_state = current_state; - - do { - uint64_t chunk[8]; - memcpy(chunk, ptr, sizeof(chunk)); - - uint64_t cs0 = CWISS_AbslHash_LowLevelMix(chunk[0] ^ salt[1], - chunk[1] ^ current_state); - uint64_t cs1 = CWISS_AbslHash_LowLevelMix(chunk[2] ^ salt[2], - chunk[3] ^ current_state); - current_state = (cs0 ^ cs1); - - uint64_t ds0 = CWISS_AbslHash_LowLevelMix(chunk[4] ^ salt[3], - chunk[5] ^ duplicated_state); - uint64_t ds1 = CWISS_AbslHash_LowLevelMix(chunk[6] ^ salt[4], - chunk[7] ^ duplicated_state); - duplicated_state = (ds0 ^ ds1); - - ptr += 64; - len -= 64; - } while (len > 64); - - current_state = current_state ^ duplicated_state; - } - - // We now have a data `ptr` with at most 64 bytes and the current state - // of the hashing state machine stored in current_state. - while (len > 16) { - uint64_t a = CWISS_Load64(ptr); - uint64_t b = CWISS_Load64(ptr + 8); - - current_state = CWISS_AbslHash_LowLevelMix(a ^ salt[1], b ^ current_state); - - ptr += 16; - len -= 16; - } - - // We now have a data `ptr` with at most 16 bytes. - uint64_t a = 0; - uint64_t b = 0; - if (len > 8) { - // When we have at least 9 and at most 16 bytes, set A to the first 64 - // bits of the input and B to the last 64 bits of the input. Yes, they will - // overlap in the middle if we are working with less than the full 16 - // bytes. - a = CWISS_Load64(ptr); - b = CWISS_Load64(ptr + len - 8); - } else if (len > 3) { - // If we have at least 4 and at most 8 bytes, set A to the first 32 - // bits and B to the last 32 bits. - a = CWISS_Load32(ptr); - b = CWISS_Load32(ptr + len - 4); - } else if (len > 0) { - // If we have at least 1 and at most 3 bytes, read all of the provided - // bits into A, with some adjustments. - a = CWISS_Load1To3(ptr, len); - } - - uint64_t w = CWISS_AbslHash_LowLevelMix(a ^ salt[1], b ^ current_state); - uint64_t z = salt[1] ^ starting_length; - return CWISS_AbslHash_LowLevelMix(w, z); -} - -// A non-deterministic seed. -// -// The current purpose of this seed is to generate non-deterministic results -// and prevent having users depend on the particular hash values. -// It is not meant as a security feature right now, but it leaves the door -// open to upgrade it to a true per-process random seed. A true random seed -// costs more and we don't need to pay for that right now. -// -// On platforms with ASLR, we take advantage of it to make a per-process -// random value. -// See https://en.wikipedia.org/wiki/Address_space_layout_randomization -// -// On other platforms this is still going to be non-deterministic but most -// probably per-build and not per-process. -static const void* const CWISS_AbslHash_kSeed = &CWISS_AbslHash_kSeed; - -// The salt array used by LowLevelHash. This array is NOT the mechanism used to -// make absl::Hash non-deterministic between program invocations. See `Seed()` -// for that mechanism. -// -// Any random values are fine. These values are just digits from the decimal -// part of pi. -// https://en.wikipedia.org/wiki/Nothing-up-my-sleeve_number -static const uint64_t CWISS_AbslHash_kHashSalt[5] = { - 0x243F6A8885A308D3, 0x13198A2E03707344, 0xA4093822299F31D0, - 0x082EFA98EC4E6C89, 0x452821E638D01377, -}; - -#define CWISS_AbslHash_kPiecewiseChunkSize ((size_t)1024) - -typedef uint64_t CWISS_AbslHash_State_; -#define CWISS_AbslHash_kInit_ ((CWISS_AbslHash_State_)CWISS_AbslHash_kSeed) - -static inline void CWISS_AbslHash_Mix(CWISS_AbslHash_State_* state, - uint64_t v) { - const uint64_t kMul = sizeof(size_t) == 4 ? 0xcc9e2d51 : 0x9ddfea08eb382d69; - *state = CWISS_AbslHash_LowLevelMix(*state + v, kMul); -} - -CWISS_INLINE_NEVER -static uint64_t CWISS_AbslHash_Hash64(const void* val, size_t len) { - return CWISS_AbslHash_LowLevelHash(val, len, CWISS_AbslHash_kInit_, - CWISS_AbslHash_kHashSalt); -} - -CWISS_END_EXTERN -CWISS_END -/// cwisstable/internal/absl_hash.h //////////////////////////////////////////// - -/// cwisstable/hash.h ////////////////////////////////////////////////////////// -/// Hash functions. -/// -/// This file provides some hash functions to use with cwisstable types. -/// -/// Every hash function defines four symbols: -/// - `CWISS__State`, the state of the hash function. -/// - `CWISS__kInit`, the initial value of the hash state. -/// - `void CWISS__Write(State*, const void*, size_t)`, write some more -/// data into the hash state. -/// - `size_t CWISS__Finish(State)`, digest the state into a final hash -/// value. -/// -/// Currently available are two hashes: `FxHash`, which is small and fast, and -/// `AbslHash`, the hash function used by Abseil. -/// -/// `AbslHash` is the default hash function. - -CWISS_BEGIN -CWISS_BEGIN_EXTERN - -typedef size_t CWISS_FxHash_State; -#define CWISS_FxHash_kInit ((CWISS_FxHash_State)0) -static inline void CWISS_FxHash_Write(CWISS_FxHash_State* state, - const void* val, size_t len) { - const size_t kSeed = (size_t)(UINT64_C(0x517cc1b727220a95)); - const uint32_t kRotate = 5; - - const char* p = (const char*)val; - CWISS_FxHash_State state_ = *state; - while (len > 0) { - size_t word = 0; - size_t to_read = len >= sizeof(state_) ? sizeof(state_) : len; - memcpy(&word, p, to_read); - - state_ = CWISS_RotateLeft(state_, kRotate); - state_ ^= word; - state_ *= kSeed; - - len -= to_read; - p += to_read; - } - *state = state_; -} -static inline size_t CWISS_FxHash_Finish(CWISS_FxHash_State state) { - return state; -} - -typedef CWISS_AbslHash_State_ CWISS_AbslHash_State; -#define CWISS_AbslHash_kInit CWISS_AbslHash_kInit_ -static inline void CWISS_AbslHash_Write(CWISS_AbslHash_State* state, - const void* val, size_t len) { - const char* val8 = (const char*)val; - if (CWISS_LIKELY(len < CWISS_AbslHash_kPiecewiseChunkSize)) { - goto CWISS_AbslHash_Write_small; - } - - while (len >= CWISS_AbslHash_kPiecewiseChunkSize) { - CWISS_AbslHash_Mix( - state, CWISS_AbslHash_Hash64(val8, CWISS_AbslHash_kPiecewiseChunkSize)); - len -= CWISS_AbslHash_kPiecewiseChunkSize; - val8 += CWISS_AbslHash_kPiecewiseChunkSize; - } - -CWISS_AbslHash_Write_small:; - uint64_t v; - if (len > 16) { - v = CWISS_AbslHash_Hash64(val8, len); - } else if (len > 8) { - CWISS_U128 p = CWISS_Load9To16(val8, len); - CWISS_AbslHash_Mix(state, p.lo); - v = p.hi; - } else if (len >= 4) { - v = CWISS_Load4To8(val8, len); - } else if (len > 0) { - v = CWISS_Load1To3(val8, len); - } else { - // Empty ranges have no effect. - return; - } - - CWISS_AbslHash_Mix(state, v); -} -static inline size_t CWISS_AbslHash_Finish(CWISS_AbslHash_State state) { - return state; -} - -CWISS_END_EXTERN -CWISS_END -/// cwisstable/hash.h ////////////////////////////////////////////////////////// - -/// cwisstable/internal/extract.h ////////////////////////////////////////////// -/// Macro keyword-arguments machinery. -/// -/// This file defines a number of macros used by policy.h to implement its -/// policy construction macros. -/// -/// The way they work is more-or-less like this: -/// -/// `CWISS_EXTRACT(foo, ...)` will find the first parenthesized pair that -/// matches exactly `(foo, whatever)`, where `foo` is part of a small set of -/// tokens defined in this file. To do so, this first expands into -/// -/// ``` -/// CWISS_EXTRACT1(CWISS_EXTRACT_foo, (k, v), ...) -/// ``` -/// -/// where `(k, v)` is the first pair in the macro arguments. This in turn -/// expands into -/// -/// ``` -/// CWISS_SELECT01(CWISS_EXTRACT_foo (k, v), CWISS_EXTRACT_VALUE, (k, v), -/// CWISS_EXTRACT2, (needle, __VA_ARGS__), CWISS_NOTHING) -/// ``` -/// -/// At this point, the preprocessor will expand `CWISS_EXTRACT_foo (k, v)` into -/// `CWISS_EXTRACT_foo_k`, which will be further expanded into `tok,tok,tok` if -/// `k` is the token `foo`, because we've defined `CWISS_EXTRACT_foo_foo` as a -/// macro. -/// -/// `CWISS_SELECT01` will then delete the first three arguments, and the fourth -/// and fifth arguments will be juxtaposed. -/// -/// In the case that `k` does not match, `CWISS_EXTRACT_foo (k, v), IDENT, (k, -/// v),` is deleted from the call, and the rest of the macro expands into -/// `CWISS_EXTRACT2(needle, __VA_ARGS__, _)` repeating the cycle but with a -/// different name. -/// -/// In the case that `k` matches, the `tok,tok,tok` is deleted, and we get -/// `CWISS_EXTRACT_VALUE(k, v)`, which expands to `v`. - -#define CWISS_EXTRACT(needle_, default_, ...) \ - (CWISS_EXTRACT_RAW(needle_, default_, __VA_ARGS__)) - -#define CWISS_EXTRACT_RAW(needle_, default_, ...) \ - CWISS_EXTRACT00(CWISS_EXTRACT_##needle_, __VA_ARGS__, (needle_, default_)) - -#define CWISS_EXTRACT_VALUE(key, val) val - -// NOTE: Everything below this line is generated by cwisstable/extract.py! -// !!! - -#define CWISS_EXTRACT_obj_copy(key_, val_) CWISS_EXTRACT_obj_copyZ##key_ -#define CWISS_EXTRACT_obj_copyZobj_copy \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_obj_dtor(key_, val_) CWISS_EXTRACT_obj_dtorZ##key_ -#define CWISS_EXTRACT_obj_dtorZobj_dtor \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_key_hash(key_, val_) CWISS_EXTRACT_key_hashZ##key_ -#define CWISS_EXTRACT_key_hashZkey_hash \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_key_eq(key_, val_) CWISS_EXTRACT_key_eqZ##key_ -#define CWISS_EXTRACT_key_eqZkey_eq CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_alloc_alloc(key_, val_) CWISS_EXTRACT_alloc_allocZ##key_ -#define CWISS_EXTRACT_alloc_allocZalloc_alloc \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_alloc_free(key_, val_) CWISS_EXTRACT_alloc_freeZ##key_ -#define CWISS_EXTRACT_alloc_freeZalloc_free \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_slot_size(key_, val_) CWISS_EXTRACT_slot_sizeZ##key_ -#define CWISS_EXTRACT_slot_sizeZslot_size \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_slot_align(key_, val_) CWISS_EXTRACT_slot_alignZ##key_ -#define CWISS_EXTRACT_slot_alignZslot_align \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_slot_init(key_, val_) CWISS_EXTRACT_slot_initZ##key_ -#define CWISS_EXTRACT_slot_initZslot_init \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_slot_transfer(key_, val_) \ - CWISS_EXTRACT_slot_transferZ##key_ -#define CWISS_EXTRACT_slot_transferZslot_transfer \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_slot_get(key_, val_) CWISS_EXTRACT_slot_getZ##key_ -#define CWISS_EXTRACT_slot_getZslot_get \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_slot_dtor(key_, val_) CWISS_EXTRACT_slot_dtorZ##key_ -#define CWISS_EXTRACT_slot_dtorZslot_dtor \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING -#define CWISS_EXTRACT_modifiers(key_, val_) CWISS_EXTRACT_modifiersZ##key_ -#define CWISS_EXTRACT_modifiersZmodifiers \ - CWISS_NOTHING, CWISS_NOTHING, CWISS_NOTHING - -#define CWISS_EXTRACT00(needle_, kv_, ...) \ - CWISS_SELECT00(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT01, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT01(needle_, kv_, ...) \ - CWISS_SELECT01(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT02, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT02(needle_, kv_, ...) \ - CWISS_SELECT02(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT03, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT03(needle_, kv_, ...) \ - CWISS_SELECT03(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT04, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT04(needle_, kv_, ...) \ - CWISS_SELECT04(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT05, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT05(needle_, kv_, ...) \ - CWISS_SELECT05(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT06, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT06(needle_, kv_, ...) \ - CWISS_SELECT06(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT07, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT07(needle_, kv_, ...) \ - CWISS_SELECT07(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT08, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT08(needle_, kv_, ...) \ - CWISS_SELECT08(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT09, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT09(needle_, kv_, ...) \ - CWISS_SELECT09(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT0A, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT0A(needle_, kv_, ...) \ - CWISS_SELECT0A(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT0B, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT0B(needle_, kv_, ...) \ - CWISS_SELECT0B(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT0C, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT0C(needle_, kv_, ...) \ - CWISS_SELECT0C(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT0D, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT0D(needle_, kv_, ...) \ - CWISS_SELECT0D(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT0E, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT0E(needle_, kv_, ...) \ - CWISS_SELECT0E(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT0F, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT0F(needle_, kv_, ...) \ - CWISS_SELECT0F(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT10, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT10(needle_, kv_, ...) \ - CWISS_SELECT10(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT11, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT11(needle_, kv_, ...) \ - CWISS_SELECT11(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT12, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT12(needle_, kv_, ...) \ - CWISS_SELECT12(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT13, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT13(needle_, kv_, ...) \ - CWISS_SELECT13(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT14, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT14(needle_, kv_, ...) \ - CWISS_SELECT14(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT15, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT15(needle_, kv_, ...) \ - CWISS_SELECT15(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT16, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT16(needle_, kv_, ...) \ - CWISS_SELECT16(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT17, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT17(needle_, kv_, ...) \ - CWISS_SELECT17(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT18, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT18(needle_, kv_, ...) \ - CWISS_SELECT18(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT19, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT19(needle_, kv_, ...) \ - CWISS_SELECT19(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT1A, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT1A(needle_, kv_, ...) \ - CWISS_SELECT1A(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT1B, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT1B(needle_, kv_, ...) \ - CWISS_SELECT1B(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT1C, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT1C(needle_, kv_, ...) \ - CWISS_SELECT1C(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT1D, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT1D(needle_, kv_, ...) \ - CWISS_SELECT1D(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT1E, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT1E(needle_, kv_, ...) \ - CWISS_SELECT1E(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT1F, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT1F(needle_, kv_, ...) \ - CWISS_SELECT1F(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT20, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT20(needle_, kv_, ...) \ - CWISS_SELECT20(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT21, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT21(needle_, kv_, ...) \ - CWISS_SELECT21(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT22, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT22(needle_, kv_, ...) \ - CWISS_SELECT22(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT23, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT23(needle_, kv_, ...) \ - CWISS_SELECT23(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT24, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT24(needle_, kv_, ...) \ - CWISS_SELECT24(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT25, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT25(needle_, kv_, ...) \ - CWISS_SELECT25(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT26, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT26(needle_, kv_, ...) \ - CWISS_SELECT26(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT27, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT27(needle_, kv_, ...) \ - CWISS_SELECT27(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT28, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT28(needle_, kv_, ...) \ - CWISS_SELECT28(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT29, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT29(needle_, kv_, ...) \ - CWISS_SELECT29(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT2A, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT2A(needle_, kv_, ...) \ - CWISS_SELECT2A(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT2B, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT2B(needle_, kv_, ...) \ - CWISS_SELECT2B(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT2C, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT2C(needle_, kv_, ...) \ - CWISS_SELECT2C(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT2D, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT2D(needle_, kv_, ...) \ - CWISS_SELECT2D(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT2E, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT2E(needle_, kv_, ...) \ - CWISS_SELECT2E(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT2F, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT2F(needle_, kv_, ...) \ - CWISS_SELECT2F(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT30, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT30(needle_, kv_, ...) \ - CWISS_SELECT30(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT31, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT31(needle_, kv_, ...) \ - CWISS_SELECT31(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT32, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT32(needle_, kv_, ...) \ - CWISS_SELECT32(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT33, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT33(needle_, kv_, ...) \ - CWISS_SELECT33(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT34, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT34(needle_, kv_, ...) \ - CWISS_SELECT34(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT35, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT35(needle_, kv_, ...) \ - CWISS_SELECT35(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT36, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT36(needle_, kv_, ...) \ - CWISS_SELECT36(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT37, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT37(needle_, kv_, ...) \ - CWISS_SELECT37(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT38, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT38(needle_, kv_, ...) \ - CWISS_SELECT38(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT39, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT39(needle_, kv_, ...) \ - CWISS_SELECT39(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT3A, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT3A(needle_, kv_, ...) \ - CWISS_SELECT3A(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT3B, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT3B(needle_, kv_, ...) \ - CWISS_SELECT3B(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT3C, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT3C(needle_, kv_, ...) \ - CWISS_SELECT3C(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT3D, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT3D(needle_, kv_, ...) \ - CWISS_SELECT3D(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT3E, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT3E(needle_, kv_, ...) \ - CWISS_SELECT3E(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT3F, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) -#define CWISS_EXTRACT3F(needle_, kv_, ...) \ - CWISS_SELECT3F(needle_ kv_, CWISS_EXTRACT_VALUE, kv_, CWISS_EXTRACT40, \ - (needle_, __VA_ARGS__), CWISS_NOTHING) - -#define CWISS_SELECT00(x_, ...) CWISS_SELECT00_(x_, __VA_ARGS__) -#define CWISS_SELECT01(x_, ...) CWISS_SELECT01_(x_, __VA_ARGS__) -#define CWISS_SELECT02(x_, ...) CWISS_SELECT02_(x_, __VA_ARGS__) -#define CWISS_SELECT03(x_, ...) CWISS_SELECT03_(x_, __VA_ARGS__) -#define CWISS_SELECT04(x_, ...) CWISS_SELECT04_(x_, __VA_ARGS__) -#define CWISS_SELECT05(x_, ...) CWISS_SELECT05_(x_, __VA_ARGS__) -#define CWISS_SELECT06(x_, ...) CWISS_SELECT06_(x_, __VA_ARGS__) -#define CWISS_SELECT07(x_, ...) CWISS_SELECT07_(x_, __VA_ARGS__) -#define CWISS_SELECT08(x_, ...) CWISS_SELECT08_(x_, __VA_ARGS__) -#define CWISS_SELECT09(x_, ...) CWISS_SELECT09_(x_, __VA_ARGS__) -#define CWISS_SELECT0A(x_, ...) CWISS_SELECT0A_(x_, __VA_ARGS__) -#define CWISS_SELECT0B(x_, ...) CWISS_SELECT0B_(x_, __VA_ARGS__) -#define CWISS_SELECT0C(x_, ...) CWISS_SELECT0C_(x_, __VA_ARGS__) -#define CWISS_SELECT0D(x_, ...) CWISS_SELECT0D_(x_, __VA_ARGS__) -#define CWISS_SELECT0E(x_, ...) CWISS_SELECT0E_(x_, __VA_ARGS__) -#define CWISS_SELECT0F(x_, ...) CWISS_SELECT0F_(x_, __VA_ARGS__) -#define CWISS_SELECT10(x_, ...) CWISS_SELECT10_(x_, __VA_ARGS__) -#define CWISS_SELECT11(x_, ...) CWISS_SELECT11_(x_, __VA_ARGS__) -#define CWISS_SELECT12(x_, ...) CWISS_SELECT12_(x_, __VA_ARGS__) -#define CWISS_SELECT13(x_, ...) CWISS_SELECT13_(x_, __VA_ARGS__) -#define CWISS_SELECT14(x_, ...) CWISS_SELECT14_(x_, __VA_ARGS__) -#define CWISS_SELECT15(x_, ...) CWISS_SELECT15_(x_, __VA_ARGS__) -#define CWISS_SELECT16(x_, ...) CWISS_SELECT16_(x_, __VA_ARGS__) -#define CWISS_SELECT17(x_, ...) CWISS_SELECT17_(x_, __VA_ARGS__) -#define CWISS_SELECT18(x_, ...) CWISS_SELECT18_(x_, __VA_ARGS__) -#define CWISS_SELECT19(x_, ...) CWISS_SELECT19_(x_, __VA_ARGS__) -#define CWISS_SELECT1A(x_, ...) CWISS_SELECT1A_(x_, __VA_ARGS__) -#define CWISS_SELECT1B(x_, ...) CWISS_SELECT1B_(x_, __VA_ARGS__) -#define CWISS_SELECT1C(x_, ...) CWISS_SELECT1C_(x_, __VA_ARGS__) -#define CWISS_SELECT1D(x_, ...) CWISS_SELECT1D_(x_, __VA_ARGS__) -#define CWISS_SELECT1E(x_, ...) CWISS_SELECT1E_(x_, __VA_ARGS__) -#define CWISS_SELECT1F(x_, ...) CWISS_SELECT1F_(x_, __VA_ARGS__) -#define CWISS_SELECT20(x_, ...) CWISS_SELECT20_(x_, __VA_ARGS__) -#define CWISS_SELECT21(x_, ...) CWISS_SELECT21_(x_, __VA_ARGS__) -#define CWISS_SELECT22(x_, ...) CWISS_SELECT22_(x_, __VA_ARGS__) -#define CWISS_SELECT23(x_, ...) CWISS_SELECT23_(x_, __VA_ARGS__) -#define CWISS_SELECT24(x_, ...) CWISS_SELECT24_(x_, __VA_ARGS__) -#define CWISS_SELECT25(x_, ...) CWISS_SELECT25_(x_, __VA_ARGS__) -#define CWISS_SELECT26(x_, ...) CWISS_SELECT26_(x_, __VA_ARGS__) -#define CWISS_SELECT27(x_, ...) CWISS_SELECT27_(x_, __VA_ARGS__) -#define CWISS_SELECT28(x_, ...) CWISS_SELECT28_(x_, __VA_ARGS__) -#define CWISS_SELECT29(x_, ...) CWISS_SELECT29_(x_, __VA_ARGS__) -#define CWISS_SELECT2A(x_, ...) CWISS_SELECT2A_(x_, __VA_ARGS__) -#define CWISS_SELECT2B(x_, ...) CWISS_SELECT2B_(x_, __VA_ARGS__) -#define CWISS_SELECT2C(x_, ...) CWISS_SELECT2C_(x_, __VA_ARGS__) -#define CWISS_SELECT2D(x_, ...) CWISS_SELECT2D_(x_, __VA_ARGS__) -#define CWISS_SELECT2E(x_, ...) CWISS_SELECT2E_(x_, __VA_ARGS__) -#define CWISS_SELECT2F(x_, ...) CWISS_SELECT2F_(x_, __VA_ARGS__) -#define CWISS_SELECT30(x_, ...) CWISS_SELECT30_(x_, __VA_ARGS__) -#define CWISS_SELECT31(x_, ...) CWISS_SELECT31_(x_, __VA_ARGS__) -#define CWISS_SELECT32(x_, ...) CWISS_SELECT32_(x_, __VA_ARGS__) -#define CWISS_SELECT33(x_, ...) CWISS_SELECT33_(x_, __VA_ARGS__) -#define CWISS_SELECT34(x_, ...) CWISS_SELECT34_(x_, __VA_ARGS__) -#define CWISS_SELECT35(x_, ...) CWISS_SELECT35_(x_, __VA_ARGS__) -#define CWISS_SELECT36(x_, ...) CWISS_SELECT36_(x_, __VA_ARGS__) -#define CWISS_SELECT37(x_, ...) CWISS_SELECT37_(x_, __VA_ARGS__) -#define CWISS_SELECT38(x_, ...) CWISS_SELECT38_(x_, __VA_ARGS__) -#define CWISS_SELECT39(x_, ...) CWISS_SELECT39_(x_, __VA_ARGS__) -#define CWISS_SELECT3A(x_, ...) CWISS_SELECT3A_(x_, __VA_ARGS__) -#define CWISS_SELECT3B(x_, ...) CWISS_SELECT3B_(x_, __VA_ARGS__) -#define CWISS_SELECT3C(x_, ...) CWISS_SELECT3C_(x_, __VA_ARGS__) -#define CWISS_SELECT3D(x_, ...) CWISS_SELECT3D_(x_, __VA_ARGS__) -#define CWISS_SELECT3E(x_, ...) CWISS_SELECT3E_(x_, __VA_ARGS__) -#define CWISS_SELECT3F(x_, ...) CWISS_SELECT3F_(x_, __VA_ARGS__) - -#define CWISS_SELECT00_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT01_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT02_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT03_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT04_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT05_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT06_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT07_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT08_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT09_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT0A_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT0B_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT0C_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT0D_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT0E_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT0F_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT10_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT11_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT12_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT13_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT14_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT15_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT16_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT17_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT18_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT19_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT1A_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT1B_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT1C_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT1D_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT1E_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT1F_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT20_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT21_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT22_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT23_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT24_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT25_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT26_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT27_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT28_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT29_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT2A_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT2B_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT2C_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT2D_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT2E_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT2F_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT30_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT31_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT32_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT33_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT34_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT35_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT36_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT37_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT38_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT39_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT3A_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT3B_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT3C_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT3D_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT3E_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -#define CWISS_SELECT3F_(ignored_, _call_, _args_, call_, args_, ...) call_ args_ -/// cwisstable/internal/extract.h ////////////////////////////////////////////// - -/// cwisstable/policy.h //////////////////////////////////////////////////////// -/// Hash table policies. -/// -/// Table policies are `cwisstable`'s generic code mechanism. All code in -/// `cwisstable`'s internals is completely agnostic to: -/// - The layout of the elements. -/// - The storage strategy for the elements (inline, indirect in the heap). -/// - Hashing, comparison, and allocation. -/// -/// This information is provided to `cwisstable`'s internals by way of a -/// *policy*: a vtable describing how to move elements around, hash them, -/// compare them, allocate storage for them, and so on and on. This design is -/// inspired by Abseil's equivalent, which is a template parameter used for -/// sharing code between all the SwissTable-backed containers. -/// -/// Unlike Abseil, policies are part of `cwisstable`'s public interface. Due to -/// C's lack of any mechanism for detecting the gross properties of types, -/// types with unwritten invariants, such as C strings (NUL-terminated byte -/// arrays), users must be able to carefully describe to `cwisstable` how to -/// correctly do things to their type. DESIGN.md goes into detailed rationale -/// for this polymorphism strategy. -/// -/// # Defining a Policy -/// -/// Policies are defined as read-only globals and passed around by pointer to -/// different `cwisstable` functions; macros are provided for doing this, since -/// most of these functions will not vary significantly from one type to -/// another. There are four of them: -/// -/// - `CWISS_DECLARE_FLAT_SET_POLICY(kPolicy, Type, ...)` -/// - `CWISS_DECLARE_FLAT_MAP_POLICY(kPolicy, Key, Value, ...)` -/// - `CWISS_DECLARE_NODE_SET_POLICY(kPolicy, Type, ...)` -/// - `CWISS_DECLARE_NODE_MAP_POLICY(kPolicy, Key, Value, ...)` -/// -/// These correspond to the four SwissTable types in Abseil: two map types and -/// two set types; "flat" means that elements are stored inline in the backing -/// array, whereas "node" means that the element is stored in its own heap -/// allocation, making it stable across rehashings (which SwissTable does more -/// or less whenever it feels like it). -/// -/// Each macro expands to a read-only global variable definition (with the name -/// `kPolicy`, i.e, the first variable) dedicated for the specified type(s). -/// The arguments that follow are overrides for the default values of each field -/// in the policy; all but the size and alignment fields of `CWISS_ObjectPolicy` -/// may be overridden. To override the field `kPolicy.foo.bar`, pass -/// `(foo_bar, value)` to the macro. If multiple such pairs are passed in, the -/// first one found wins. `examples/stringmap.c` provides an example of how to -/// use this functionality. -/// -/// For "common" uses, where the key and value are plain-old-data, `declare.h` -/// has dedicated macros, and fussing with policies directly is unnecessary. - -CWISS_BEGIN -CWISS_BEGIN_EXTERN - -/// A policy describing the basic laying properties of a type. -/// -/// This type describes how to move values of a particular type around. -typedef struct { - /// The layout of the stored object. - size_t size, align; - - /// Performs a deep copy of `src` onto a fresh location `dst`. - void (*copy)(void* dst, const void* src); - - /// Destroys an object. - /// - /// This member may, as an optimization, be null. This will cause it to - /// behave as a no-op, and may be more efficient than making this an empty - /// function. - void (*dtor)(void* val); -} CWISS_ObjectPolicy; - -/// A policy describing the hashing properties of a type. -/// -/// This type describes the necessary information for putting a value into a -/// hash table. -/// -/// A *heterogenous* key policy is one whose equality function expects different -/// argument types, which can be used for so-called heterogenous lookup: finding -/// an element of a table by comparing it to a somewhat different type. If the -/// table element is, for example, a `std::string`[1]-like type, it could still -/// be found via a non-owning version like a `std::string_view`[2]. This is -/// important for making efficient use of a SwissTable. -/// -/// [1]: For non C++ programmers: a growable string type implemented as a -/// `struct { char* ptr; size_t size, capacity; }`. -/// [2]: Similarly, a `std::string_view` is a pointer-length pair to a string -/// *somewhere*; unlike a C-style string, it might be a substring of a -/// larger allocation elsewhere. -typedef struct { - /// Computes the hash of a value. - /// - /// This function must be such that if two elements compare equal, they must - /// have the same hash (but not vice-versa). - /// - /// If this policy is heterogenous, this function must be defined so that - /// given the original key policy of the table's element type, if - /// `hetero->eq(a, b)` holds, then `hetero->hash(a) == original->hash(b)`. - /// In other words, the obvious condition for a hash table to work correctly - /// with this policy. - size_t (*hash)(const void* val); - - /// Compares two values for equality. - /// - /// This function is actually not symmetric: the first argument will always be - /// the value being searched for, and the second will be a pointer to the - /// candidate entry. In particular, this means they can be different types: - /// in C++ parlance, `needle` could be a `std::string_view`, while `candidate` - /// could be a `std::string`. - bool (*eq)(const void* needle, const void* candidate); -} CWISS_KeyPolicy; - -/// A policy for allocation. -/// -/// This type provides access to a custom allocator. -typedef struct { - /// Allocates memory. - /// - /// This function must never fail and never return null, unlike `malloc`. This - /// function does not need to tolerate zero sized allocations. - void* (*alloc)(size_t size, size_t align); - - /// Deallocates memory allocated by `alloc`. - /// - /// This function is passed the same size/alignment as was passed to `alloc`, - /// allowing for sized-delete optimizations. - void (*free)(void* array, size_t size, size_t align); -} CWISS_AllocPolicy; - -/// A policy for allocating space for slots. -/// -/// This allows us to distinguish between inline storage (more cache-friendly) -/// and outline (pointer-stable). -typedef struct { - /// The layout of a slot value. - /// - /// Usually, this will be the same as for the object type, *or* the layout - /// of a pointer (for outline storage). - size_t size, align; - - /// Initializes a new slot at the given location. - /// - /// This function does not initialize the value *in* the slot; it simply sets - /// up the slot so that a value can be `memcpy`'d or otherwise emplaced into - /// the slot. - void (*init)(void* slot); - - /// Destroys a slot, including the destruction of the value it contains. - /// - /// This function may, as an optimization, be null. This will cause it to - /// behave as a no-op. - void (*del)(void* slot); - - /// Transfers a slot. - /// - /// `dst` must be uninitialized; `src` must be initialized. After this - /// function, their roles will be switched: `dst` will be initialized and - /// contain the value from `src`; `src` will be initialized. - /// - /// This function need not actually copy the underlying value. - void (*transfer)(void* dst, void* src); - - /// Extracts a pointer to the value inside the a slot. - /// - /// This function does not need to tolerate nulls. - void* (*get)(void* slot); -} CWISS_SlotPolicy; - -/// A hash table policy. -/// -/// See the header documentation for more information. -typedef struct { - const CWISS_ObjectPolicy* obj; - const CWISS_KeyPolicy* key; - const CWISS_AllocPolicy* alloc; - const CWISS_SlotPolicy* slot; -} CWISS_Policy; - -/// Declares a hash set policy with inline storage for the given type. -/// -/// See the header documentation for more information. -#define CWISS_DECLARE_FLAT_SET_POLICY(kPolicy_, Type_, ...) \ - CWISS_DECLARE_POLICY_(kPolicy_, Type_, Type_, __VA_ARGS__) - -/// Declares a hash map policy with inline storage for the given key and value -/// types. -/// -/// See the header documentation for more information. -#define CWISS_DECLARE_FLAT_MAP_POLICY(kPolicy_, K_, V_, ...) \ - typedef struct { \ - K_ k; \ - V_ v; \ - } kPolicy_##_Entry; \ - CWISS_DECLARE_POLICY_(kPolicy_, kPolicy_##_Entry, K_, __VA_ARGS__) - -/// Declares a hash set policy with pointer-stable storage for the given type. -/// -/// See the header documentation for more information. -#define CWISS_DECLARE_NODE_SET_POLICY(kPolicy_, Type_, ...) \ - CWISS_DECLARE_NODE_FUNCTIONS_(kPolicy_, Type_, Type_, __VA_ARGS__) \ - CWISS_DECLARE_POLICY_(kPolicy_, Type_, Type_, __VA_ARGS__, \ - CWISS_NODE_OVERRIDES_(kPolicy_)) - -/// Declares a hash map policy with pointer-stable storage for the given key and -/// value types. -/// -/// See the header documentation for more information. -#define CWISS_DECLARE_NODE_MAP_POLICY(kPolicy_, K_, V_, ...) \ - typedef struct { \ - K_ k; \ - V_ v; \ - } kPolicy_##_Entry; \ - CWISS_DECLARE_NODE_FUNCTIONS_(kPolicy_, kPolicy_##_Entry, K_, __VA_ARGS__) \ - CWISS_DECLARE_POLICY_(kPolicy_, kPolicy_##_Entry, K_, __VA_ARGS__, \ - CWISS_NODE_OVERRIDES_(kPolicy_)) - -// ---- PUBLIC API ENDS HERE! ---- - -#define CWISS_DECLARE_POLICY_(kPolicy_, Type_, Key_, ...) \ - CWISS_BEGIN \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - inline void kPolicy_##_DefaultCopy(void* dst, const void* src) { \ - memcpy(dst, src, sizeof(Type_)); \ - } \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - inline size_t kPolicy_##_DefaultHash(const void* val) { \ - CWISS_AbslHash_State state = CWISS_AbslHash_kInit; \ - CWISS_AbslHash_Write(&state, val, sizeof(Key_)); \ - return CWISS_AbslHash_Finish(state); \ - } \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - inline bool kPolicy_##_DefaultEq(const void* a, const void* b) { \ - return memcmp(a, b, sizeof(Key_)) == 0; \ - } \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - inline void kPolicy_##_DefaultSlotInit(void* slot) {} \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - inline void kPolicy_##_DefaultSlotTransfer(void* dst, void* src) { \ - memcpy(dst, src, sizeof(Type_)); \ - } \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - inline void* kPolicy_##_DefaultSlotGet(void* slot) { return slot; } \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - inline void kPolicy_##_DefaultSlotDtor(void* slot) { \ - if (CWISS_EXTRACT(obj_dtor, NULL, __VA_ARGS__) != NULL) { \ - CWISS_EXTRACT(obj_dtor, (void (*)(void*))NULL, __VA_ARGS__)(slot); \ - } \ - } \ - \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - const CWISS_ObjectPolicy kPolicy_##_ObjectPolicy = { \ - sizeof(Type_), \ - alignof(Type_), \ - CWISS_EXTRACT(obj_copy, kPolicy_##_DefaultCopy, __VA_ARGS__), \ - CWISS_EXTRACT(obj_dtor, NULL, __VA_ARGS__), \ - }; \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - const CWISS_KeyPolicy kPolicy_##_KeyPolicy = { \ - CWISS_EXTRACT(key_hash, kPolicy_##_DefaultHash, __VA_ARGS__), \ - CWISS_EXTRACT(key_eq, kPolicy_##_DefaultEq, __VA_ARGS__), \ - }; \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - const CWISS_AllocPolicy kPolicy_##_AllocPolicy = { \ - CWISS_EXTRACT(alloc_alloc, CWISS_DefaultMalloc, __VA_ARGS__), \ - CWISS_EXTRACT(alloc_free, CWISS_DefaultFree, __VA_ARGS__), \ - }; \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - const CWISS_SlotPolicy kPolicy_##_SlotPolicy = { \ - CWISS_EXTRACT(slot_size, sizeof(Type_), __VA_ARGS__), \ - CWISS_EXTRACT(slot_align, sizeof(Type_), __VA_ARGS__), \ - CWISS_EXTRACT(slot_init, kPolicy_##_DefaultSlotInit, __VA_ARGS__), \ - CWISS_EXTRACT(slot_dtor, kPolicy_##_DefaultSlotDtor, __VA_ARGS__), \ - CWISS_EXTRACT(slot_transfer, kPolicy_##_DefaultSlotTransfer, \ - __VA_ARGS__), \ - CWISS_EXTRACT(slot_get, kPolicy_##_DefaultSlotGet, __VA_ARGS__), \ - }; \ - CWISS_END \ - CWISS_EXTRACT_RAW(modifiers, static, __VA_ARGS__) \ - const CWISS_Policy kPolicy_ = { \ - &kPolicy_##_ObjectPolicy, \ - &kPolicy_##_KeyPolicy, \ - &kPolicy_##_AllocPolicy, \ - &kPolicy_##_SlotPolicy, \ - } - -#define CWISS_DECLARE_NODE_FUNCTIONS_(kPolicy_, Type_, ...) \ - CWISS_BEGIN \ - static inline void kPolicy_##_NodeSlotInit(void* slot) { \ - void* node = CWISS_EXTRACT(alloc_alloc, CWISS_DefaultMalloc, __VA_ARGS__)( \ - sizeof(Type_), alignof(Type_)); \ - memcpy(slot, &node, sizeof(node)); \ - } \ - static inline void kPolicy_##_NodeSlotDtor(void* slot) { \ - if (CWISS_EXTRACT(obj_dtor, NULL, __VA_ARGS__) != NULL) { \ - CWISS_EXTRACT(obj_dtor, (void (*)(void*))NULL, __VA_ARGS__) \ - (*(void**)slot); \ - } \ - CWISS_EXTRACT(alloc_free, CWISS_DefaultFree, __VA_ARGS__) \ - (*(void**)slot, sizeof(Type_), alignof(Type_)); \ - } \ - static inline void kPolicy_##_NodeSlotTransfer(void* dst, void* src) { \ - memcpy(dst, src, sizeof(void*)); \ - } \ - static inline void* kPolicy_##_NodeSlotGet(void* slot) { \ - return *((void**)slot); \ - } \ - CWISS_END - -#define CWISS_NODE_OVERRIDES_(kPolicy_) \ - (slot_size, sizeof(void*)), (slot_align, alignof(void*)), \ - (slot_init, kPolicy_##_NodeSlotInit), \ - (slot_dtor, kPolicy_##_NodeSlotDtor), \ - (slot_transfer, kPolicy_##_NodeSlotTransfer), \ - (slot_get, kPolicy_##_NodeSlotGet) - -static inline void* CWISS_DefaultMalloc(size_t size, size_t align) { - void* p = malloc(size); // TODO: Check alignment. - CWISS_CHECK(p != NULL, "malloc() returned null"); - return p; -} -static inline void CWISS_DefaultFree(void* array, size_t size, size_t align) { - free(array); -} - -CWISS_END_EXTERN -CWISS_END -/// cwisstable/policy.h //////////////////////////////////////////////////////// - -/// cwisstable/internal/raw_table.h //////////////////////////////////////////// -/// The SwissTable implementation. -/// -/// `CWISS_RawTable` is the core data structure that all SwissTables wrap. -/// -/// All functions in this header take a `const CWISS_Policy*`, which describes -/// how to manipulate the elements in a table. The same pointer (i.e., same -/// address and provenance) passed to the function that created the -/// `CWISS_RawTable` MUST be passed to all subsequent function calls, and it -/// must not be mutated at any point between those calls. Failure to adhere to -/// these requirements is UB. -/// -/// It is STRONGLY recommended that this pointer point to a const global. - -CWISS_BEGIN -CWISS_BEGIN_EXTERN - -/// A SwissTable. -/// -/// This is absl::container_internal::raw_hash_set in Abseil. -typedef struct { - /// The control bytes (and, also, a pointer to the base of the backing array). - /// - /// This contains `capacity_ + 1 + CWISS_NumClonedBytes()` entries. - CWISS_ControlByte* ctrl_; - /// The beginning of the slots, located at `CWISS_SlotOffset()` bytes after - /// `ctrl_`. May be null for empty tables. - char* slots_; - /// The number of filled slots. - size_t size_; - /// The total number of available slots. - size_t capacity_; - /// The number of slots we can still fill before a rehash. See - /// `CWISS_CapacityToGrowth()`. - size_t growth_left_; -} CWISS_RawTable; - -/// Prints full details about the internal state of `self` to `stderr`. -static inline void CWISS_RawTable_dump(const CWISS_Policy* policy, - const CWISS_RawTable* self) { - fprintf(stderr, "ptr: %p, len: %zu, cap: %zu, growth: %zu\n", self->ctrl_, - self->size_, self->capacity_, self->growth_left_); - if (self->capacity_ == 0) { - return; - } - - size_t ctrl_bytes = self->capacity_ + CWISS_NumClonedBytes(); - for (size_t i = 0; i <= ctrl_bytes; ++i) { - fprintf(stderr, "[%4zu] %p / ", i, &self->ctrl_[i]); - switch (self->ctrl_[i]) { - case CWISS_kSentinel: - fprintf(stderr, "kSentinel: //\n"); - continue; - case CWISS_kEmpty: - fprintf(stderr, " kEmpty"); - break; - case CWISS_kDeleted: - fprintf(stderr, " kDeleted"); - break; - default: - fprintf(stderr, " H2(0x%02x)", self->ctrl_[i]); - break; - } - - if (i >= self->capacity_) { - fprintf(stderr, ": \n"); - continue; - } - - char* slot = self->slots_ + i * policy->slot->size; - fprintf(stderr, ": %p /", slot); - for (size_t j = 0; j < policy->slot->size; ++j) { - fprintf(stderr, " %02x", (unsigned char)slot[j]); - } - char* elem = (char*)policy->slot->get(slot); - if (elem != slot && CWISS_IsFull(self->ctrl_[i])) { - fprintf(stderr, " ->"); - for (size_t j = 0; j < policy->obj->size; ++j) { - fprintf(stderr, " %02x", (unsigned char)elem[j]); - } - } - fprintf(stderr, "\n"); - } -} - -/// An iterator into a SwissTable. -/// -/// Unlike a C++ iterator, there is no "end" to compare to. Instead, -/// `CWISS_RawIter_get()` will yield a null pointer once the iterator is -/// exhausted. -/// -/// Invariants: -/// - `ctrl_` and `slot_` are always in sync (i.e., the pointed to control byte -/// corresponds to the pointed to slot), or both are null. `set_` may be null -/// in the latter case. -/// - `ctrl_` always points to a full slot. -typedef struct { - CWISS_RawTable* set_; - CWISS_ControlByte* ctrl_; - char* slot_; -} CWISS_RawIter; - -/// Fixes up `ctrl_` to point to a full by advancing it and `slot_` until they -/// reach one. -/// -/// If a sentinel is reached, we null both of them out instead. -static inline void CWISS_RawIter_SkipEmptyOrDeleted(const CWISS_Policy* policy, - CWISS_RawIter* self) { - while (CWISS_IsEmptyOrDeleted(*self->ctrl_)) { - CWISS_Group g = CWISS_Group_new(self->ctrl_); - uint32_t shift = CWISS_Group_CountLeadingEmptyOrDeleted(&g); - self->ctrl_ += shift; - self->slot_ += shift * policy->slot->size; - } - - // Not sure why this is a branch rather than a cmov; Abseil uses a branch. - if (CWISS_UNLIKELY(*self->ctrl_ == CWISS_kSentinel)) { - self->ctrl_ = NULL; - self->slot_ = NULL; - } -} - -/// Creates a valid iterator starting at the `index`th slot. -static inline CWISS_RawIter CWISS_RawTable_iter_at(const CWISS_Policy* policy, - CWISS_RawTable* self, - size_t index) { - CWISS_RawIter iter = { - self, - self->ctrl_ + index, - self->slots_ + index * policy->slot->size, - }; - CWISS_RawIter_SkipEmptyOrDeleted(policy, &iter); - CWISS_AssertIsValid(iter.ctrl_); - return iter; -} - -/// Creates an iterator for `self`. -static inline CWISS_RawIter CWISS_RawTable_iter(const CWISS_Policy* policy, - CWISS_RawTable* self) { - return CWISS_RawTable_iter_at(policy, self, 0); -} - -/// Creates a valid iterator starting at the `index`th slot, accepting a `const` -/// pointer instead. -static inline CWISS_RawIter CWISS_RawTable_citer_at(const CWISS_Policy* policy, - const CWISS_RawTable* self, - size_t index) { - return CWISS_RawTable_iter_at(policy, (CWISS_RawTable*)self, index); -} - -/// Creates an iterator for `self`, accepting a `const` pointer instead. -static inline CWISS_RawIter CWISS_RawTable_citer(const CWISS_Policy* policy, - const CWISS_RawTable* self) { - return CWISS_RawTable_iter(policy, (CWISS_RawTable*)self); -} - -/// Returns a pointer into the currently pointed-to slot (*not* to the slot -/// itself, but rather its contents). -/// -/// Returns null if the iterator has been exhausted. -static inline void* CWISS_RawIter_get(const CWISS_Policy* policy, - const CWISS_RawIter* self) { - CWISS_AssertIsValid(self->ctrl_); - if (self->slot_ == NULL) { - return NULL; - } - - return policy->slot->get(self->slot_); -} - -/// Advances the iterator and returns the result of `CWISS_RawIter_get()`. -/// -/// Calling on an empty iterator is UB. -static inline void* CWISS_RawIter_next(const CWISS_Policy* policy, - CWISS_RawIter* self) { - CWISS_AssertIsFull(self->ctrl_); - ++self->ctrl_; - self->slot_ += policy->slot->size; - - CWISS_RawIter_SkipEmptyOrDeleted(policy, self); - return CWISS_RawIter_get(policy, self); -} - -/// Erases, but does not destroy, the value pointed to by `it`. -static inline void CWISS_RawTable_EraseMetaOnly(const CWISS_Policy* policy, - CWISS_RawIter it) { - CWISS_DCHECK(CWISS_IsFull(*it.ctrl_), "erasing a dangling iterator"); - --it.set_->size_; - const size_t index = (size_t)(it.ctrl_ - it.set_->ctrl_); - const size_t index_before = (index - CWISS_Group_kWidth) & it.set_->capacity_; - CWISS_Group g_after = CWISS_Group_new(it.ctrl_); - CWISS_BitMask empty_after = CWISS_Group_MatchEmpty(&g_after); - CWISS_Group g_before = CWISS_Group_new(it.set_->ctrl_ + index_before); - CWISS_BitMask empty_before = CWISS_Group_MatchEmpty(&g_before); - - // We count how many consecutive non empties we have to the right and to the - // left of `it`. If the sum is >= kWidth then there is at least one probe - // window that might have seen a full group. - bool was_never_full = - empty_before.mask && empty_after.mask && - (size_t)(CWISS_BitMask_TrailingZeros(&empty_after) + - CWISS_BitMask_LeadingZeros(&empty_before)) < CWISS_Group_kWidth; - - CWISS_SetCtrl(index, was_never_full ? CWISS_kEmpty : CWISS_kDeleted, - it.set_->capacity_, it.set_->ctrl_, it.set_->slots_, - policy->slot->size); - it.set_->growth_left_ += was_never_full; - // infoz().RecordErase(); -} - -/// Computes a lower bound for the expected available growth and applies it to -/// `self_`. -static inline void CWISS_RawTable_ResetGrowthLeft(const CWISS_Policy* policy, - CWISS_RawTable* self) { - self->growth_left_ = CWISS_CapacityToGrowth(self->capacity_) - self->size_; -} - -/// Allocates a backing array for `self` and initializes its control bits. This -/// reads `capacity_` and updates all other fields based on the result of the -/// allocation. -/// -/// This does not free the currently held array; `capacity_` must be nonzero. -static inline void CWISS_RawTable_InitializeSlots(const CWISS_Policy* policy, - CWISS_RawTable* self) { - CWISS_DCHECK(self->capacity_, "capacity should be nonzero"); - // Folks with custom allocators often make unwarranted assumptions about the - // behavior of their classes vis-a-vis trivial destructability and what - // calls they will or wont make. Avoid sampling for people with custom - // allocators to get us out of this mess. This is not a hard guarantee but - // a workaround while we plan the exact guarantee we want to provide. - // - // People are often sloppy with the exact type of their allocator (sometimes - // it has an extra const or is missing the pair, but rebinds made it work - // anyway). To avoid the ambiguity, we work off SlotAlloc which we have - // bound more carefully. - // - // NOTE(mcyoung): Not relevant in C but kept in case we decide to do custom - // alloc. - /*if (std::is_same>::value && - slots_ == nullptr) { - infoz() = Sample(sizeof(slot_type)); - }*/ - - char* mem = - (char*) // Cast for C++. - policy->alloc->alloc(CWISS_AllocSize(self->capacity_, policy->slot->size, - policy->slot->align), - policy->slot->align); - - self->ctrl_ = (CWISS_ControlByte*)mem; - self->slots_ = mem + CWISS_SlotOffset(self->capacity_, policy->slot->align); - CWISS_ResetCtrl(self->capacity_, self->ctrl_, self->slots_, - policy->slot->size); - CWISS_RawTable_ResetGrowthLeft(policy, self); - - // infoz().RecordStorageChanged(size_, capacity_); -} - -/// Destroys all slots in the backing array, frees the backing array, and clears -/// all top-level book-keeping data. -static inline void CWISS_RawTable_DestroySlots(const CWISS_Policy* policy, - CWISS_RawTable* self) { - if (!self->capacity_) return; - - if (policy->slot->del != NULL) { - for (size_t i = 0; i != self->capacity_; ++i) { - if (CWISS_IsFull(self->ctrl_[i])) { - policy->slot->del(self->slots_ + i * policy->slot->size); - } - } - } - - policy->alloc->free( - self->ctrl_, - CWISS_AllocSize(self->capacity_, policy->slot->size, policy->slot->align), - policy->slot->align); - self->ctrl_ = CWISS_EmptyGroup(); - self->slots_ = NULL; - self->size_ = 0; - self->capacity_ = 0; - self->growth_left_ = 0; -} - -/// Grows the table to the given capacity, triggering a rehash. -static inline void CWISS_RawTable_Resize(const CWISS_Policy* policy, - CWISS_RawTable* self, - size_t new_capacity) { - CWISS_DCHECK(CWISS_IsValidCapacity(new_capacity), "invalid capacity: %zu", - new_capacity); - - CWISS_ControlByte* old_ctrl = self->ctrl_; - char* old_slots = self->slots_; - const size_t old_capacity = self->capacity_; - self->capacity_ = new_capacity; - CWISS_RawTable_InitializeSlots(policy, self); - - size_t total_probe_length = 0; - for (size_t i = 0; i != old_capacity; ++i) { - if (CWISS_IsFull(old_ctrl[i])) { - size_t hash = policy->key->hash( - policy->slot->get(old_slots + i * policy->slot->size)); - CWISS_FindInfo target = - CWISS_FindFirstNonFull(self->ctrl_, hash, self->capacity_); - size_t new_i = target.offset; - total_probe_length += target.probe_length; - CWISS_SetCtrl(new_i, CWISS_H2(hash), self->capacity_, self->ctrl_, - self->slots_, policy->slot->size); - policy->slot->transfer(self->slots_ + new_i * policy->slot->size, - old_slots + i * policy->slot->size); - } - } - if (old_capacity) { - CWISS_UnpoisonMemory(old_slots, policy->slot->size * old_capacity); - policy->alloc->free( - old_ctrl, - CWISS_AllocSize(old_capacity, policy->slot->size, policy->slot->align), - policy->slot->align); - } - // infoz().RecordRehash(total_probe_length); -} - -/// Prunes control bits to remove as many tombstones as possible. -/// -/// See the comment on `CWISS_RawTable_rehash_and_grow_if_necessary()`. -CWISS_INLINE_NEVER -static void CWISS_RawTable_DropDeletesWithoutResize(const CWISS_Policy* policy, - CWISS_RawTable* self) { - CWISS_DCHECK(CWISS_IsValidCapacity(self->capacity_), "invalid capacity: %zu", - self->capacity_); - CWISS_DCHECK(!CWISS_IsSmall(self->capacity_), - "unexpected small capacity: %zu", self->capacity_); - // Algorithm: - // - mark all DELETED slots as EMPTY - // - mark all FULL slots as DELETED - // - for each slot marked as DELETED - // hash = Hash(element) - // target = find_first_non_full(hash) - // if target is in the same group - // mark slot as FULL - // else if target is EMPTY - // transfer element to target - // mark slot as EMPTY - // mark target as FULL - // else if target is DELETED - // swap current element with target element - // mark target as FULL - // repeat procedure for current slot with moved from element (target) - CWISS_ConvertDeletedToEmptyAndFullToDeleted(self->ctrl_, self->capacity_); - size_t total_probe_length = 0; - // Unfortunately because we do not know this size statically, we need to take - // a trip to the allocator. Alternatively we could use a variable length - // alloca... - void* slot = policy->alloc->alloc(policy->slot->size, policy->slot->align); - - for (size_t i = 0; i != self->capacity_; ++i) { - if (!CWISS_IsDeleted(self->ctrl_[i])) continue; - - char* old_slot = self->slots_ + i * policy->slot->size; - size_t hash = policy->key->hash(policy->slot->get(old_slot)); - - const CWISS_FindInfo target = - CWISS_FindFirstNonFull(self->ctrl_, hash, self->capacity_); - const size_t new_i = target.offset; - total_probe_length += target.probe_length; - - char* new_slot = self->slots_ + new_i * policy->slot->size; - - // Verify if the old and new i fall within the same group wrt the hash. - // If they do, we don't need to move the object as it falls already in the - // best probe we can. - const size_t probe_offset = - CWISS_ProbeSeq_Start(self->ctrl_, hash, self->capacity_).offset_; -#define CWISS_ProbeIndex(pos_) \ - (((pos_ - probe_offset) & self->capacity_) / CWISS_Group_kWidth) - - // Element doesn't move. - if (CWISS_LIKELY(CWISS_ProbeIndex(new_i) == CWISS_ProbeIndex(i))) { - CWISS_SetCtrl(i, CWISS_H2(hash), self->capacity_, self->ctrl_, - self->slots_, policy->slot->size); - continue; - } - if (CWISS_IsEmpty(self->ctrl_[new_i])) { - // Transfer element to the empty spot. - // SetCtrl poisons/unpoisons the slots so we have to call it at the - // right time. - CWISS_SetCtrl(new_i, CWISS_H2(hash), self->capacity_, self->ctrl_, - self->slots_, policy->slot->size); - policy->slot->transfer(new_slot, old_slot); - CWISS_SetCtrl(i, CWISS_kEmpty, self->capacity_, self->ctrl_, self->slots_, - policy->slot->size); - } else { - CWISS_DCHECK(CWISS_IsDeleted(self->ctrl_[new_i]), - "bad ctrl value at %zu: %02x", new_i, self->ctrl_[new_i]); - CWISS_SetCtrl(new_i, CWISS_H2(hash), self->capacity_, self->ctrl_, - self->slots_, policy->slot->size); - // Until we are done rehashing, DELETED marks previously FULL slots. - // Swap i and new_i elements. - - policy->slot->transfer(slot, old_slot); - policy->slot->transfer(old_slot, new_slot); - policy->slot->transfer(new_slot, slot); - --i; // repeat - } -#undef CWISS_ProbeSeq_Start_index - } - CWISS_RawTable_ResetGrowthLeft(policy, self); - policy->alloc->free(slot, policy->slot->size, policy->slot->align); - // infoz().RecordRehash(total_probe_length); -} - -/// Called whenever the table *might* need to conditionally grow. -/// -/// This function is an optimization opportunity to perform a rehash even when -/// growth is unnecessary, because vacating tombstones is beneficial for -/// performance in the long-run. -static inline void CWISS_RawTable_rehash_and_grow_if_necessary( - const CWISS_Policy* policy, CWISS_RawTable* self) { - if (self->capacity_ == 0) { - CWISS_RawTable_Resize(policy, self, 1); - } else if (self->capacity_ > CWISS_Group_kWidth && - // Do these calculations in 64-bit to avoid overflow. - self->size_ * UINT64_C(32) <= self->capacity_ * UINT64_C(25)) { - // Squash DELETED without growing if there is enough capacity. - // - // Rehash in place if the current size is <= 25/32 of capacity_. - // Rationale for such a high factor: 1) drop_deletes_without_resize() is - // faster than resize, and 2) it takes quite a bit of work to add - // tombstones. In the worst case, seems to take approximately 4 - // insert/erase pairs to create a single tombstone and so if we are - // rehashing because of tombstones, we can afford to rehash-in-place as - // long as we are reclaiming at least 1/8 the capacity without doing more - // than 2X the work. (Where "work" is defined to be size() for rehashing - // or rehashing in place, and 1 for an insert or erase.) But rehashing in - // place is faster per operation than inserting or even doubling the size - // of the table, so we actually afford to reclaim even less space from a - // resize-in-place. The decision is to rehash in place if we can reclaim - // at about 1/8th of the usable capacity (specifically 3/28 of the - // capacity) which means that the total cost of rehashing will be a small - // fraction of the total work. - // - // Here is output of an experiment using the BM_CacheInSteadyState - // benchmark running the old case (where we rehash-in-place only if we can - // reclaim at least 7/16*capacity_) vs. this code (which rehashes in place - // if we can recover 3/32*capacity_). - // - // Note that although in the worst-case number of rehashes jumped up from - // 15 to 190, but the number of operations per second is almost the same. - // - // Abridged output of running BM_CacheInSteadyState benchmark from - // raw_hash_set_benchmark. N is the number of insert/erase operations. - // - // | OLD (recover >= 7/16 | NEW (recover >= 3/32) - // size | N/s LoadFactor NRehashes | N/s LoadFactor NRehashes - // 448 | 145284 0.44 18 | 140118 0.44 19 - // 493 | 152546 0.24 11 | 151417 0.48 28 - // 538 | 151439 0.26 11 | 151152 0.53 38 - // 583 | 151765 0.28 11 | 150572 0.57 50 - // 628 | 150241 0.31 11 | 150853 0.61 66 - // 672 | 149602 0.33 12 | 150110 0.66 90 - // 717 | 149998 0.35 12 | 149531 0.70 129 - // 762 | 149836 0.37 13 | 148559 0.74 190 - // 807 | 149736 0.39 14 | 151107 0.39 14 - // 852 | 150204 0.42 15 | 151019 0.42 15 - CWISS_RawTable_DropDeletesWithoutResize(policy, self); - } else { - // Otherwise grow the container. - CWISS_RawTable_Resize(policy, self, self->capacity_ * 2 + 1); - } -} - -/// Prefetches the backing array to dodge potential TLB misses. -/// This is intended to overlap with execution of calculating the hash for a -/// key. -static inline void CWISS_RawTable_PrefetchHeapBlock( - const CWISS_Policy* policy, const CWISS_RawTable* self) { - CWISS_PREFETCH(self->ctrl_, 1); -} - -/// Issues CPU prefetch instructions for the memory needed to find or insert -/// a key. -/// -/// NOTE: This is a very low level operation and should not be used without -/// specific benchmarks indicating its importance. -static inline void CWISS_RawTable_Prefetch(const CWISS_Policy* policy, - const CWISS_RawTable* self, - const void* key) { - (void)key; -#if CWISS_HAVE_PREFETCH - CWISS_RawTable_PrefetchHeapBlock(policy, self); - CWISS_ProbeSeq seq = CWISS_ProbeSeq_Start(self->ctrl_, policy->key->hash(key), - self->capacity_); - CWISS_PREFETCH(self->ctrl_ + seq.offset_, 3); - CWISS_PREFETCH(self->ctrl_ + seq.offset_ * policy->slot->size, 3); -#endif -} - -/// The return type of `CWISS_RawTable_PrepareInsert()`. -typedef struct { - size_t index; - bool inserted; -} CWISS_PrepareInsert; - -/// Given the hash of a value not currently in the table, finds the next viable -/// slot index to insert it at. -/// -/// If the table does not actually have space, UB. -CWISS_INLINE_NEVER -static size_t CWISS_RawTable_PrepareInsert(const CWISS_Policy* policy, - CWISS_RawTable* self, size_t hash) { - CWISS_FindInfo target = - CWISS_FindFirstNonFull(self->ctrl_, hash, self->capacity_); - if (CWISS_UNLIKELY(self->growth_left_ == 0 && - !CWISS_IsDeleted(self->ctrl_[target.offset]))) { - CWISS_RawTable_rehash_and_grow_if_necessary(policy, self); - target = CWISS_FindFirstNonFull(self->ctrl_, hash, self->capacity_); - } - ++self->size_; - self->growth_left_ -= CWISS_IsEmpty(self->ctrl_[target.offset]); - CWISS_SetCtrl(target.offset, CWISS_H2(hash), self->capacity_, self->ctrl_, - self->slots_, policy->slot->size); - // infoz().RecordInsert(hash, target.probe_length); - return target.offset; -} - -/// Attempts to find `key` in the table; if it isn't found, returns where to -/// insert it, instead. -static inline CWISS_PrepareInsert CWISS_RawTable_FindOrPrepareInsert( - const CWISS_Policy* policy, const CWISS_KeyPolicy* key_policy, - CWISS_RawTable* self, const void* key) { - CWISS_RawTable_PrefetchHeapBlock(policy, self); - size_t hash = key_policy->hash(key); - CWISS_ProbeSeq seq = CWISS_ProbeSeq_Start(self->ctrl_, hash, self->capacity_); - while (true) { - CWISS_Group g = CWISS_Group_new(self->ctrl_ + seq.offset_); - CWISS_BitMask match = CWISS_Group_Match(&g, CWISS_H2(hash)); - uint32_t i; - while (CWISS_BitMask_next(&match, &i)) { - size_t idx = CWISS_ProbeSeq_offset(&seq, i); - char* slot = self->slots_ + idx * policy->slot->size; - if (CWISS_LIKELY(key_policy->eq(key, policy->slot->get(slot)))) - return (CWISS_PrepareInsert){idx, false}; - } - if (CWISS_LIKELY(CWISS_Group_MatchEmpty(&g).mask)) break; - CWISS_ProbeSeq_next(&seq); - CWISS_DCHECK(seq.index_ <= self->capacity_, "full table!"); - } - return (CWISS_PrepareInsert){CWISS_RawTable_PrepareInsert(policy, self, hash), - true}; -} - -/// Prepares a slot to insert an element into. -/// -/// This function does all the work of calling the appropriate policy functions -/// to initialize the slot. -static inline void* CWISS_RawTable_PreInsert(const CWISS_Policy* policy, - CWISS_RawTable* self, size_t i) { - void* dst = self->slots_ + i * policy->slot->size; - policy->slot->init(dst); - return policy->slot->get(dst); -} - -/// Creates a new empty table with the given capacity. -static inline CWISS_RawTable CWISS_RawTable_new(const CWISS_Policy* policy, - size_t capacity) { - CWISS_RawTable self = { - .ctrl_ = CWISS_EmptyGroup(), - }; - - if (capacity != 0) { - self.capacity_ = CWISS_NormalizeCapacity(capacity); - CWISS_RawTable_InitializeSlots(policy, &self); - } - - return self; -} - -/// Ensures that at least `n` more elements can be inserted without a resize -/// (although this function my itself resize and rehash the table). -static inline void CWISS_RawTable_reserve(const CWISS_Policy* policy, - CWISS_RawTable* self, size_t n) { - if (n <= self->size_ + self->growth_left_) { - return; - } - - n = CWISS_NormalizeCapacity(CWISS_GrowthToLowerboundCapacity(n)); - CWISS_RawTable_Resize(policy, self, n); - - // This is after resize, to ensure that we have completed the allocation - // and have potentially sampled the hashtable. - // infoz().RecordReservation(n); -} - -/// Creates a duplicate of this table. -static inline CWISS_RawTable CWISS_RawTable_dup(const CWISS_Policy* policy, - const CWISS_RawTable* self) { - CWISS_RawTable copy = CWISS_RawTable_new(policy, 0); - - CWISS_RawTable_reserve(policy, ©, self->size_); - // Because the table is guaranteed to be empty, we can do something faster - // than a full `insert`. In particular we do not need to take a trip to - // `CWISS_RawTable_rehash_and_grow_if_necessary()` because we are already - // big enough (since `self` is a priori) and tombstones cannot be created - // during this process. - for (CWISS_RawIter iter = CWISS_RawTable_citer(policy, self); - CWISS_RawIter_get(policy, &iter); CWISS_RawIter_next(policy, &iter)) { - void* v = CWISS_RawIter_get(policy, &iter); - size_t hash = policy->key->hash(v); - - CWISS_FindInfo target = - CWISS_FindFirstNonFull(copy.ctrl_, hash, copy.capacity_); - CWISS_SetCtrl(target.offset, CWISS_H2(hash), copy.capacity_, copy.ctrl_, - copy.slots_, policy->slot->size); - void* slot = CWISS_RawTable_PreInsert(policy, ©, target.offset); - policy->obj->copy(slot, v); - // infoz().RecordInsert(hash, target.probe_length); - } - copy.size_ = self->size_; - copy.growth_left_ -= self->size_; - return copy; -} - -/// Destroys this table, destroying its elements and freeing the backing array. -static inline void CWISS_RawTable_destroy(const CWISS_Policy* policy, - CWISS_RawTable* self) { - CWISS_RawTable_DestroySlots(policy, self); -} - -/// Returns whether the table is empty. -static inline bool CWISS_RawTable_empty(const CWISS_Policy* policy, - const CWISS_RawTable* self) { - return !self->size_; -} - -/// Returns the number of elements in the table. -static inline size_t CWISS_RawTable_size(const CWISS_Policy* policy, - const CWISS_RawTable* self) { - return self->size_; -} - -/// Returns the total capacity of the table, which is different from the number -/// of elements that would cause it to get resized. -static inline size_t CWISS_RawTable_capacity(const CWISS_Policy* policy, - const CWISS_RawTable* self) { - return self->capacity_; -} - -/// Clears the table, erasing every element contained therein. -static inline void CWISS_RawTable_clear(const CWISS_Policy* policy, - CWISS_RawTable* self) { - // Iterating over this container is O(bucket_count()). When bucket_count() - // is much greater than size(), iteration becomes prohibitively expensive. - // For clear() it is more important to reuse the allocated array when the - // container is small because allocation takes comparatively long time - // compared to destruction of the elements of the container. So we pick the - // largest bucket_count() threshold for which iteration is still fast and - // past that we simply deallocate the array. - if (self->capacity_ > 127) { - CWISS_RawTable_DestroySlots(policy, self); - - // infoz().RecordClearedReservation(); - } else if (self->capacity_) { - if (policy->slot->del != NULL) { - for (size_t i = 0; i != self->capacity_; ++i) { - if (CWISS_IsFull(self->ctrl_[i])) { - policy->slot->del(self->slots_ + i * policy->slot->size); - } - } - } - - self->size_ = 0; - CWISS_ResetCtrl(self->capacity_, self->ctrl_, self->slots_, - policy->slot->size); - CWISS_RawTable_ResetGrowthLeft(policy, self); - } - CWISS_DCHECK(!self->size_, "size was still nonzero"); - // infoz().RecordStorageChanged(0, capacity_); -} - -/// The return type of `CWISS_RawTable_insert()`. -typedef struct { - /// An iterator referring to the relevant element. - CWISS_RawIter iter; - /// True if insertion actually occurred; false if the element was already - /// present. - bool inserted; -} CWISS_Insert; - -/// "Inserts" `val` into the table if it isn't already present. -/// -/// This function does not perform insertion; it behaves exactly like -/// `CWISS_RawTable_insert()` up until it would copy-initialize the new -/// element, instead returning a valid iterator pointing to uninitialized data. -/// -/// This allows, for example, lazily constructing the parts of the element that -/// do not figure into the hash or equality. -/// -/// If this function returns `true` in `inserted`, the caller has *no choice* -/// but to insert, i.e., they may not change their minds at that point. -/// -/// `key_policy` is a possibly heterogenous key policy for comparing `key`'s -/// type to types in the map. `key_policy` may be `&policy->key`. -static inline CWISS_Insert CWISS_RawTable_deferred_insert( - const CWISS_Policy* policy, const CWISS_KeyPolicy* key_policy, - CWISS_RawTable* self, const void* key) { - CWISS_PrepareInsert res = - CWISS_RawTable_FindOrPrepareInsert(policy, key_policy, self, key); - - if (res.inserted) { - CWISS_RawTable_PreInsert(policy, self, res.index); - } - return (CWISS_Insert){CWISS_RawTable_citer_at(policy, self, res.index), - res.inserted}; -} - -/// Inserts `val` (by copy) into the table if it isn't already present. -/// -/// Returns an iterator pointing to the element in the map and whether it was -/// just inserted or was already present. -static inline CWISS_Insert CWISS_RawTable_insert(const CWISS_Policy* policy, - CWISS_RawTable* self, - const void* val) { - CWISS_PrepareInsert res = - CWISS_RawTable_FindOrPrepareInsert(policy, policy->key, self, val); - - if (res.inserted) { - void* slot = CWISS_RawTable_PreInsert(policy, self, res.index); - policy->obj->copy(slot, val); - } - return (CWISS_Insert){CWISS_RawTable_citer_at(policy, self, res.index), - res.inserted}; -} - -/// Tries to find the corresponding entry for `key` using `hash` as a hint. -/// If not found, returns a null iterator. -/// -/// `key_policy` is a possibly heterogenous key policy for comparing `key`'s -/// type to types in the map. `key_policy` may be `&policy->key`. -/// -/// If `hash` is not actually the hash of `key`, UB. -static inline CWISS_RawIter CWISS_RawTable_find_hinted( - const CWISS_Policy* policy, const CWISS_KeyPolicy* key_policy, - const CWISS_RawTable* self, const void* key, size_t hash) { - CWISS_ProbeSeq seq = CWISS_ProbeSeq_Start(self->ctrl_, hash, self->capacity_); - while (true) { - CWISS_Group g = CWISS_Group_new(self->ctrl_ + seq.offset_); - CWISS_BitMask match = CWISS_Group_Match(&g, CWISS_H2(hash)); - uint32_t i; - while (CWISS_BitMask_next(&match, &i)) { - char* slot = - self->slots_ + CWISS_ProbeSeq_offset(&seq, i) * policy->slot->size; - if (CWISS_LIKELY(key_policy->eq(key, policy->slot->get(slot)))) - return CWISS_RawTable_citer_at(policy, self, - CWISS_ProbeSeq_offset(&seq, i)); - } - if (CWISS_LIKELY(CWISS_Group_MatchEmpty(&g).mask)) - return (CWISS_RawIter){0}; - CWISS_ProbeSeq_next(&seq); - CWISS_DCHECK(seq.index_ <= self->capacity_, "full table!"); - } -} - -/// Tries to find the corresponding entry for `key`. -/// If not found, returns a null iterator. -/// -/// `key_policy` is a possibly heterogenous key policy for comparing `key`'s -/// type to types in the map. `key_policy` may be `&policy->key`. -static inline CWISS_RawIter CWISS_RawTable_find( - const CWISS_Policy* policy, const CWISS_KeyPolicy* key_policy, - const CWISS_RawTable* self, const void* key) { - return CWISS_RawTable_find_hinted(policy, key_policy, self, key, - key_policy->hash(key)); -} - -/// Erases the element pointed to by the given valid iterator. -/// This function will invalidate the iterator. -static inline void CWISS_RawTable_erase_at(const CWISS_Policy* policy, - CWISS_RawIter it) { - CWISS_AssertIsFull(it.ctrl_); - if (policy->slot->del != NULL) { - policy->slot->del(it.slot_); - } - CWISS_RawTable_EraseMetaOnly(policy, it); -} - -/// Erases the entry corresponding to `key`, if present. Returns true if -/// deletion occured. -/// -/// `key_policy` is a possibly heterogenous key policy for comparing `key`'s -/// type to types in the map. `key_policy` may be `&policy->key`. -static inline bool CWISS_RawTable_erase(const CWISS_Policy* policy, - const CWISS_KeyPolicy* key_policy, - CWISS_RawTable* self, const void* key) { - CWISS_RawIter it = CWISS_RawTable_find(policy, key_policy, self, key); - if (it.slot_ == NULL) return false; - CWISS_RawTable_erase_at(policy, it); - return true; -} - -/// Triggers a rehash, growing to at least a capacity of `n`. -static inline void CWISS_RawTable_rehash(const CWISS_Policy* policy, - CWISS_RawTable* self, size_t n) { - if (n == 0 && self->capacity_ == 0) return; - if (n == 0 && self->size_ == 0) { - CWISS_RawTable_DestroySlots(policy, self); - // infoz().RecordStorageChanged(0, 0); - // infoz().RecordClearedReservation(); - return; - } - - // bitor is a faster way of doing `max` here. We will round up to the next - // power-of-2-minus-1, so bitor is good enough. - size_t m = CWISS_NormalizeCapacity( - n | CWISS_GrowthToLowerboundCapacity(self->size_)); - // n == 0 unconditionally rehashes as per the standard. - if (n == 0 || m > self->capacity_) { - CWISS_RawTable_Resize(policy, self, m); - - // This is after resize, to ensure that we have completed the allocation - // and have potentially sampled the hashtable. - // infoz().RecordReservation(n); - } -} - -/// Returns whether `key` is contained in this table. -/// -/// `key_policy` is a possibly heterogenous key policy for comparing `key`'s -/// type to types in the map. `key_policy` may be `&policy->key`. -static inline bool CWISS_RawTable_contains(const CWISS_Policy* policy, - const CWISS_KeyPolicy* key_policy, - const CWISS_RawTable* self, - const void* key) { - return CWISS_RawTable_find(policy, key_policy, self, key).slot_ != NULL; -} - -CWISS_END_EXTERN -CWISS_END -/// cwisstable/internal/raw_table.h //////////////////////////////////////////// - -/// cwisstable/declare.h /////////////////////////////////////////////////////// -/// SwissTable code generation macros. -/// -/// This file is the entry-point for users of `cwisstable`. It exports six -/// macros for generating different kinds of tables. Four correspond to Abseil's -/// four SwissTable containers: -/// -/// - `CWISS_DECLARE_FLAT_HASHSET(Set, Type)` -/// - `CWISS_DECLARE_FLAT_HASHMAP(Map, Key, Value)` -/// - `CWISS_DECLARE_NODE_HASHSET(Set, Type)` -/// - `CWISS_DECLARE_NODE_HASHMAP(Map, Key, Value)` -/// -/// These expand to a type (with the same name as the first argument) and and -/// a collection of strongly-typed functions associated to it (the generated -/// API is described below). These macros use the default policy (see policy.h) -/// for each of the four containers; custom policies may be used instead via -/// the following macros: -/// -/// - `CWISS_DECLARE_HASHSET_WITH(Set, Type, kPolicy)` -/// - `CWISS_DECLARE_HASHMAP_WITH(Map, Key, Value, kPolicy)` -/// -/// `kPolicy` must be a constant global variable referring to an appropriate -/// property for the element types of the container. -/// -/// The generated API is safe: the functions are well-typed and automatically -/// pass the correct policy pointer. Because the pointer is a constant -/// expression, it promotes devirtualization when inlining. -/// -/// # Generated API -/// -/// See `set_api.h` and `map_api.h` for detailed listings of what the generated -/// APIs look like. - -CWISS_BEGIN -CWISS_BEGIN_EXTERN - -/// Generates a new hash set type with inline storage and the default -/// plain-old-data policies. -/// -/// See header documentation for examples of generated API. -#define CWISS_DECLARE_FLAT_HASHSET(HashSet_, Type_) \ - CWISS_DECLARE_FLAT_SET_POLICY(HashSet_##_kPolicy, Type_, (_, _)); \ - CWISS_DECLARE_HASHSET_WITH(HashSet_, Type_, HashSet_##_kPolicy) - -/// Generates a new hash set type with outline storage and the default -/// plain-old-data policies. -/// -/// See header documentation for examples of generated API. -#define CWISS_DECLARE_NODE_HASHSET(HashSet_, Type_) \ - CWISS_DECLARE_NODE_SET_POLICY(HashSet_##_kPolicy, Type_, (_, _)); \ - CWISS_DECLARE_HASHSET_WITH(HashSet_, Type_, HashSet_##_kPolicy) - -/// Generates a new hash map type with inline storage and the default -/// plain-old-data policies. -/// -/// See header documentation for examples of generated API. -#define CWISS_DECLARE_FLAT_HASHMAP(HashMap_, K_, V_) \ - CWISS_DECLARE_FLAT_MAP_POLICY(HashMap_##_kPolicy, K_, V_, (_, _)); \ - CWISS_DECLARE_HASHMAP_WITH(HashMap_, K_, V_, HashMap_##_kPolicy) - -/// Generates a new hash map type with outline storage and the default -/// plain-old-data policies. -/// -/// See header documentation for examples of generated API. -#define CWISS_DECLARE_NODE_HASHMAP(HashMap_, K_, V_) \ - CWISS_DECLARE_NODE_MAP_POLICY(HashMap_##_kPolicy, K_, V_, (_, _)); \ - CWISS_DECLARE_HASHMAP_WITH(HashMap_, K_, V_, HashMap_##_kPolicy) - -/// Generates a new hash set type using the given policy. -/// -/// See header documentation for examples of generated API. -#define CWISS_DECLARE_HASHSET_WITH(HashSet_, Type_, kPolicy_) \ - typedef Type_ HashSet_##_Entry; \ - typedef Type_ HashSet_##_Key; \ - CWISS_DECLARE_COMMON_(HashSet_, HashSet_##_Entry, HashSet_##_Key, kPolicy_) - -/// Generates a new hash map type using the given policy. -/// -/// See header documentation for examples of generated API. -#define CWISS_DECLARE_HASHMAP_WITH(HashMap_, K_, V_, kPolicy_) \ - typedef struct { \ - K_ key; \ - V_ val; \ - } HashMap_##_Entry; \ - typedef K_ HashMap_##_Key; \ - CWISS_DECLARE_COMMON_(HashMap_, HashMap_##_Entry, HashMap_##_Key, kPolicy_) - -/// Declares a heterogenous lookup for an existing SwissTable type. -/// -/// This macro will expect to find the following functions: -/// - size_t __hash(const Key*); -/// - bool
__eq(const Key*, const
_Key*); -/// -/// These functions will be used to build the heterogenous key policy. -#define CWISS_DECLARE_LOOKUP(HashSet_, Key_) \ - CWISS_DECLARE_LOOKUP_NAMED(HashSet_, Key_, Key_) - -/// Declares a heterogenous lookup for an existing SwissTable type. -/// -/// This is like `CWISS_DECLARE_LOOKUP`, but allows customizing the name used -/// in the `_by_` prefix on the names, as well as the names of the extension -/// point functions. -#define CWISS_DECLARE_LOOKUP_NAMED(HashSet_, LookupName_, Key_) \ - CWISS_BEGIN \ - static inline size_t HashSet_##_##LookupName_##_SyntheticHash( \ - const void* val) { \ - return HashSet_##_##LookupName_##_hash((const Key_*)val); \ - } \ - static inline bool HashSet_##_##LookupName_##_SyntheticEq(const void* a, \ - const void* b) { \ - return HashSet_##_##LookupName_##_eq((const Key_*)a, \ - (const HashSet_##_Entry*)b); \ - } \ - static const CWISS_KeyPolicy HashSet_##_##LookupName_##_kPolicy = { \ - HashSet_##_##LookupName_##_SyntheticHash, \ - HashSet_##_##LookupName_##_SyntheticEq, \ - }; \ - \ - static inline const CWISS_KeyPolicy* HashSet_##_##LookupName_##_policy( \ - void) { \ - return &HashSet_##_##LookupName_##_kPolicy; \ - } \ - \ - static inline HashSet_##_Insert HashSet_##_deferred_insert_by_##LookupName_( \ - HashSet_* self, const Key_* key) { \ - CWISS_Insert ret = CWISS_RawTable_deferred_insert( \ - HashSet_##_policy(), &HashSet_##_##LookupName_##_kPolicy, &self->set_, \ - key); \ - return (HashSet_##_Insert){{ret.iter}, ret.inserted}; \ - } \ - static inline HashSet_##_CIter HashSet_##_cfind_hinted_by_##LookupName_( \ - const HashSet_* self, const Key_* key, size_t hash) { \ - return (HashSet_##_CIter){CWISS_RawTable_find_hinted( \ - HashSet_##_policy(), &HashSet_##_##LookupName_##_kPolicy, &self->set_, \ - key, hash)}; \ - } \ - static inline HashSet_##_Iter HashSet_##_find_hinted_by_##LookupName_( \ - HashSet_* self, const Key_* key, size_t hash) { \ - return (HashSet_##_Iter){CWISS_RawTable_find_hinted( \ - HashSet_##_policy(), &HashSet_##_##LookupName_##_kPolicy, &self->set_, \ - key, hash)}; \ - } \ - \ - static inline HashSet_##_CIter HashSet_##_cfind_by_##LookupName_( \ - const HashSet_* self, const Key_* key) { \ - return (HashSet_##_CIter){CWISS_RawTable_find( \ - HashSet_##_policy(), &HashSet_##_##LookupName_##_kPolicy, &self->set_, \ - key)}; \ - } \ - static inline HashSet_##_Iter HashSet_##_find_by_##LookupName_( \ - HashSet_* self, const Key_* key) { \ - return (HashSet_##_Iter){CWISS_RawTable_find( \ - HashSet_##_policy(), &HashSet_##_##LookupName_##_kPolicy, &self->set_, \ - key)}; \ - } \ - \ - static inline bool HashSet_##_contains_by_##LookupName_( \ - const HashSet_* self, const Key_* key) { \ - return CWISS_RawTable_contains(HashSet_##_policy(), \ - &HashSet_##_##LookupName_##_kPolicy, \ - &self->set_, key); \ - } \ - \ - static inline bool HashSet_##_erase_by_##LookupName_(HashSet_* self, \ - const Key_* key) { \ - return CWISS_RawTable_erase(HashSet_##_policy(), \ - &HashSet_##_##LookupName_##_kPolicy, \ - &self->set_, key); \ - } \ - \ - CWISS_END \ - /* Force a semicolon. */ \ - struct HashSet_##_##LookupName_##_NeedsTrailingSemicolon_ { \ - int x; \ - } - -// ---- PUBLIC API ENDS HERE! ---- - -#define CWISS_DECLARE_COMMON_(HashSet_, Type_, Key_, kPolicy_) \ - CWISS_BEGIN \ - static inline const CWISS_Policy* HashSet_##_policy(void) { \ - return &kPolicy_; \ - } \ - \ - typedef struct { \ - CWISS_RawTable set_; \ - } HashSet_; \ - static inline void HashSet_##_dump(const HashSet_* self) { \ - CWISS_RawTable_dump(&kPolicy_, &self->set_); \ - } \ - \ - static inline HashSet_ HashSet_##_new(size_t bucket_count) { \ - return (HashSet_){CWISS_RawTable_new(&kPolicy_, bucket_count)}; \ - } \ - static inline HashSet_ HashSet_##_dup(const HashSet_* that) { \ - return (HashSet_){CWISS_RawTable_dup(&kPolicy_, &that->set_)}; \ - } \ - static inline void HashSet_##_destroy(HashSet_* self) { \ - CWISS_RawTable_destroy(&kPolicy_, &self->set_); \ - } \ - \ - typedef struct { \ - CWISS_RawIter it_; \ - } HashSet_##_Iter; \ - static inline HashSet_##_Iter HashSet_##_iter(HashSet_* self) { \ - return (HashSet_##_Iter){CWISS_RawTable_iter(&kPolicy_, &self->set_)}; \ - } \ - static inline Type_* HashSet_##_Iter_get(const HashSet_##_Iter* it) { \ - return (Type_*)CWISS_RawIter_get(&kPolicy_, &it->it_); \ - } \ - static inline Type_* HashSet_##_Iter_next(HashSet_##_Iter* it) { \ - return (Type_*)CWISS_RawIter_next(&kPolicy_, &it->it_); \ - } \ - \ - typedef struct { \ - CWISS_RawIter it_; \ - } HashSet_##_CIter; \ - static inline HashSet_##_CIter HashSet_##_citer(const HashSet_* self) { \ - return (HashSet_##_CIter){CWISS_RawTable_citer(&kPolicy_, &self->set_)}; \ - } \ - static inline const Type_* HashSet_##_CIter_get( \ - const HashSet_##_CIter* it) { \ - return (const Type_*)CWISS_RawIter_get(&kPolicy_, &it->it_); \ - } \ - static inline const Type_* HashSet_##_CIter_next(HashSet_##_CIter* it) { \ - return (const Type_*)CWISS_RawIter_next(&kPolicy_, &it->it_); \ - } \ - static inline HashSet_##_CIter HashSet_##_Iter_const(HashSet_##_Iter it) { \ - return (HashSet_##_CIter){it.it_}; \ - } \ - \ - static inline void HashSet_##_reserve(HashSet_* self, size_t n) { \ - CWISS_RawTable_reserve(&kPolicy_, &self->set_, n); \ - } \ - static inline void HashSet_##_rehash(HashSet_* self, size_t n) { \ - CWISS_RawTable_rehash(&kPolicy_, &self->set_, n); \ - } \ - \ - static inline bool HashSet_##_empty(const HashSet_* self) { \ - return CWISS_RawTable_empty(&kPolicy_, &self->set_); \ - } \ - static inline size_t HashSet_##_size(const HashSet_* self) { \ - return CWISS_RawTable_size(&kPolicy_, &self->set_); \ - } \ - static inline size_t HashSet_##_capacity(const HashSet_* self) { \ - return CWISS_RawTable_capacity(&kPolicy_, &self->set_); \ - } \ - \ - static inline void HashSet_##_clear(HashSet_* self) { \ - return CWISS_RawTable_clear(&kPolicy_, &self->set_); \ - } \ - \ - typedef struct { \ - HashSet_##_Iter iter; \ - bool inserted; \ - } HashSet_##_Insert; \ - static inline HashSet_##_Insert HashSet_##_deferred_insert( \ - HashSet_* self, const Key_* key) { \ - CWISS_Insert ret = CWISS_RawTable_deferred_insert(&kPolicy_, kPolicy_.key, \ - &self->set_, key); \ - return (HashSet_##_Insert){{ret.iter}, ret.inserted}; \ - } \ - static inline HashSet_##_Insert HashSet_##_insert(HashSet_* self, \ - const Type_* val) { \ - CWISS_Insert ret = CWISS_RawTable_insert(&kPolicy_, &self->set_, val); \ - return (HashSet_##_Insert){{ret.iter}, ret.inserted}; \ - } \ - \ - static inline HashSet_##_CIter HashSet_##_cfind_hinted( \ - const HashSet_* self, const Key_* key, size_t hash) { \ - return (HashSet_##_CIter){CWISS_RawTable_find_hinted( \ - &kPolicy_, kPolicy_.key, &self->set_, key, hash)}; \ - } \ - static inline HashSet_##_Iter HashSet_##_find_hinted( \ - HashSet_* self, const Key_* key, size_t hash) { \ - return (HashSet_##_Iter){CWISS_RawTable_find_hinted( \ - &kPolicy_, kPolicy_.key, &self->set_, key, hash)}; \ - } \ - static inline HashSet_##_CIter HashSet_##_cfind(const HashSet_* self, \ - const Key_* key) { \ - return (HashSet_##_CIter){ \ - CWISS_RawTable_find(&kPolicy_, kPolicy_.key, &self->set_, key)}; \ - } \ - static inline HashSet_##_Iter HashSet_##_find(HashSet_* self, \ - const Key_* key) { \ - return (HashSet_##_Iter){ \ - CWISS_RawTable_find(&kPolicy_, kPolicy_.key, &self->set_, key)}; \ - } \ - \ - static inline bool HashSet_##_contains(const HashSet_* self, \ - const Key_* key) { \ - return CWISS_RawTable_contains(&kPolicy_, kPolicy_.key, &self->set_, key); \ - } \ - \ - static inline void HashSet_##_erase_at(HashSet_##_Iter it) { \ - CWISS_RawTable_erase_at(&kPolicy_, it.it_); \ - } \ - static inline bool HashSet_##_erase(HashSet_* self, const Key_* key) { \ - return CWISS_RawTable_erase(&kPolicy_, kPolicy_.key, &self->set_, key); \ - } \ - \ - CWISS_END \ - /* Force a semicolon. */ struct HashSet_##_NeedsTrailingSemicolon_ { int x; } - -CWISS_END_EXTERN -CWISS_END -/// cwisstable/declare.h /////////////////////////////////////////////////////// - -#endif // CWISSTABLE_H_ diff --git a/setup.py b/setup.py index 975f2706548..d0ba3aba96e 100644 --- a/setup.py +++ b/setup.py @@ -700,26 +700,6 @@ def build_extension(self, ext): return raise else: - # For the memalloc extension, dynamically add libdd_wrapper to extra_objects - if ext.name == "ddtrace.profiling.collector._memalloc" and CURRENT_OS in ("Linux", "Darwin"): - dd_wrapper_suffix: t.Optional[str] = sysconfig.get_config_var("EXT_SUFFIX") - - wrapper_dir: Path - if IS_EDITABLE or getattr(self, "inplace", False): - # Editable build: use source directory - wrapper_dir = Path(__file__).parent / "ddtrace" / "internal" / "datadog" / "profiling" - else: - # Non-editable build: use build directory - wrapper_dir = ( - Path(__file__).parent / Path(self.build_lib) / "ddtrace" / "internal" / "datadog" / "profiling" - ) - - wrapper_path: Path = wrapper_dir / f"libdd_wrapper{dd_wrapper_suffix}" - if wrapper_path.exists(): - wrapper_path_str: str = str(wrapper_path) - if wrapper_path_str not in ext.extra_objects: - ext.extra_objects.append(wrapper_path_str) - super().build_extension(ext) if COMPILE_MODE.lower() in ("release", "minsizerel"): @@ -1097,34 +1077,14 @@ def get_exts_for(name): if CURRENT_OS in ("Linux", "Darwin") and is_64_bit_python(): if sys.version_info < (3, 14): + # Memory profiler now uses CMake to support Abseil dependency + MEMALLOC_DIR = HERE / "ddtrace" / "profiling" / "collector" ext_modules.append( - Extension( + CMakeExtension( "ddtrace.profiling.collector._memalloc", - sources=[ - "ddtrace/profiling/collector/_memalloc.cpp", - "ddtrace/profiling/collector/_memalloc_tb.cpp", - "ddtrace/profiling/collector/_memalloc_heap.cpp", - "ddtrace/profiling/collector/_memalloc_reentrant.cpp", - "ddtrace/profiling/collector/_memalloc_heap_map.cpp", - ], - include_dirs=[ - "ddtrace/internal/datadog/profiling/dd_wrapper/include", - ], - extra_link_args=( - ["-Wl,-rpath,$ORIGIN/../../internal/datadog/profiling", "-latomic"] - if CURRENT_OS == "Linux" - else ["-Wl,-rpath,@loader_path/../../internal/datadog/profiling"] - if CURRENT_OS == "Darwin" - else [] - ), - language="c++", - extra_compile_args=( - debug_compile_args - + (["-DNDEBUG"] if not debug_compile_args else ["-UNDEBUG"]) - + ["-D_POSIX_C_SOURCE=200809L", "-std=c++20"] - + fast_build_args - ), - ), + source_dir=MEMALLOC_DIR, + optional=False, + ) ) ext_modules.append(