diff --git a/.cmake-format b/.cmake-format new file mode 100644 index 0000000..9db8847 --- /dev/null +++ b/.cmake-format @@ -0,0 +1,67 @@ +{ + "parse": { + "additional_commands": { + "add_toolchain": { + "pargs": { + "nargs": 3 + } + }, + "add_vsyncer_check": { + "kwargs": { + "CFLAGS": "+", + "DEPENDENCIES": "+", + "MEMORY_MODELS": "+", + "SOURCE": 1, + "TARGET": 1, + "TIMEOUT": 1 + }, + "pargs": { + "flags": [], + "nargs": "*" + } + }, + "get_filename_wext": { + "pargs": { + "nargs": 2 + } + }, + "get_git_tag": { + "pargs": { + "nargs": 1 + } + }, + "get_includes": { + "pargs": { + "nargs": 1 + } + }, + "get_toolkit_qemu": { + "pargs": { + "nargs": 3 + } + }, + "project_tag": { + "pargs": { + "nargs": 0 + } + }, + "CPMAddPackage": {}, + "_CPMAddPackage": {}, + "vsync_install": { + "kwargs": { + "DIRECTORY": "+", + "DESTINATION": "+", + "COMPONENTS": "+", + "EXTRA_ARGS": "+" + }, + "pargs": { + "flags": [], + "nargs": "*" + } + } + } + }, + "format": { + "tab_size": 4 + } +} diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index c3c2ef4..f414308 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -23,14 +23,17 @@ jobs: container: ghcr.io/open-s4c/vsyncer-ci:sha-5f9eab426f2608a9fac2eabf15c7614c73a81736 strategy: matrix: - test-dir: [ {p: "test", c: "spinlock"}, {p: "test", c: "quack"}, {p: "test", c: "queue"}, {p: "verify", c: "unbounded_queue"}, {p: "verify", c: "listset"} ] + test-dir: [ {p: "test", c: "spinlock"}, {p: "test", c: "quack"}, {p: "test", c: "queue"}, + {p: "verify", c: "unbounded_queue"}, {p: "verify", c: "listset"} , + {p: "verify", c: "stack"}, {p: "verify", c: "thread"}, {p: "verify", c: "simpleht"}, + {p: "verify", c: "bitmap"}, {p: "verify", c: "treeset"}] steps: - name: Print vsyncer version run: vsyncer version - name: Check out repository code uses: actions/checkout@v4 - name: Configure Verification - run: cmake -S. -Bbuild -DVSYNCER_CHECK=ON + run: cmake -S. -Bbuild -DVSYNCER_CHECK=ON -DVSYNCER_CHECK_FULL=OFF - name: Build Verification Clients for ${{ matrix.test-dir }} run: cmake --build build/${{ matrix.test-dir.p }}/${{ matrix.test-dir.c }} - name: Run Verification diff --git a/CHANGELOG.md b/CHANGELOG.md index a3c8399..8261e8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,25 @@ is not guaranteed to result in increment of major version. Please note that the version correlates to the internal libvsync, which is a superset of what exists in open-s4c libvsync. + +### [3.6.0] + +### Added + +- cnalock, clhlock, arraylock, twalock, hmcslock, hclhlock, rec_seqlock and hemlock +- bitmap +- simpleht hashtable +- elimination and xbo backoff stacks +- mutex, cond, and once +- treeset + +### Fixed + +- implementation of `rwlock_acquired_by_readers`, which used to return +true also in the case of neither a writer nor a reader acquired the lock. +Now it returns true if and only if the lock is acquired by readers, as the name suggests. + + ### [3.5.0] ### Added diff --git a/CMakeLists.txt b/CMakeLists.txt index 97f7e31..6da313d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,11 +3,14 @@ cmake_minimum_required(VERSION 3.16) project( libvsync LANGUAGES C - VERSION 3.5.0 - DESCRIPTION "Verified library of atomics, synchronization primitives and concurrent data structures") + VERSION 3.6.0 + DESCRIPTION + "Verified library of atomics, synchronization primitives and concurrent data structures" +) option(LIBVSYNC_ADDRESS_SANITIZER "Compile with -fsanitize=address" "off") -option(LIBVSYNC_CODECHECK_BINSCOPE "Compile with necessary flags for binscope" "off") +option(LIBVSYNC_CODECHECK_BINSCOPE "Compile with necessary flags for binscope" + "off") include(GNUInstallDirs) include(cmake/export.cmake) @@ -23,9 +26,9 @@ install(DIRECTORY include/vsync DESTINATION include) install(FILES vmm/vmm.cat DESTINATION share/vsync) install(TARGETS vsync EXPORT ${PROJECT_TARGETS}) -############################################################## -# Check for memset_s memcpy_s existence -############################################################## +# ############################################################################## +# Check for memset_s memcpy_s existence +# ############################################################################## check_symbol_exists(memset_s "string.h" VSYNC_MEMSET_S_EXISTS) if(VSYNC_MEMSET_S_EXISTS) target_compile_definitions(vsync INTERFACE "VSYNC_MEMSET_S_EXISTS") @@ -36,20 +39,29 @@ if(VSYNC_MEMCPY_S_EXISTS) endif() if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) - ############################################################## - # Important Config must be set - ############################################################## + # ########################################################################## + # Important Config must be set + # ########################################################################## set(ATOMICS_DIR ${PROJECT_SOURCE_DIR}) - ############################################################## - # Commands - ############################################################## - set(FMT_CMD ${PROJECT_SOURCE_DIR}/scripts/clang-format.sh - ${CMAKE_SOURCE_DIR}) - set(STYLE_FILE ${PROJECT_SOURCE_DIR}/.clang-format) + # ########################################################################## + # Commands + # ########################################################################## + set(CMAKE_FMT_CMD ${PROJECT_SOURCE_DIR}/scripts/cmake-format.sh + ${CMAKE_SOURCE_DIR}) - add_custom_target(clang-format-apply - COMMAND env STYLE=${STYLE_FILE} SILENT=true ${FMT_CMD}) + set(CMAKE_STYLE_FILE ${PROJECT_SOURCE_DIR}/.cmake-format) + + add_custom_target(cmake-format-apply COMMAND env STYLE=${CMAKE_STYLE_FILE} + SILENT=true ${CMAKE_FMT_CMD}) + + set(CLANG_FMT_CMD ${PROJECT_SOURCE_DIR}/scripts/clang-format.sh + ${CMAKE_SOURCE_DIR}) + + set(CLANG_STYLE_FILE ${PROJECT_SOURCE_DIR}/.clang-format) + + add_custom_target(clang-format-apply COMMAND env STYLE=${CLANG_STYLE_FILE} + SILENT=true ${CLANG_FMT_CMD}) add_custom_target( sanitize-vsync @@ -77,8 +89,9 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) endif() if(LIBVSYNC_CODECHECK_BINSCOPE) target_compile_definitions(vsync INTERFACE _FORTIFY_SOURCE=2) - target_compile_options(vsync INTERFACE -fstack-protector-strong -fstack-protector-all -fPIE -fPIC -O2) + target_compile_options( + vsync INTERFACE -fstack-protector-strong -fstack-protector-all + -fPIE -fPIC -O2) target_link_options(vsync INTERFACE -s -pie -Wl,-z,relro,-z,now) endif() endif() - diff --git a/cmake/check.cmake b/cmake/check.cmake index ee75f8a..085d7db 100644 --- a/cmake/check.cmake +++ b/cmake/check.cmake @@ -1,3 +1,5 @@ +option(VSYNCER_CHECK "enable vsyncer checks" off) +option(VSYNCER_CHECK_FULL "disable quick vsyncer checks" off) function(add_vsyncer_check) # skip if VSYNCER_CHECK is not defined if(NOT VSYNCER_CHECK) @@ -47,8 +49,12 @@ function(add_vsyncer_check) CFLAGS # -I${PROJECT_SOURCE_DIR}/include # -DVSYNC_VERIFICATION # - -DVSYNC_VERIFICATION_QUICK -DVSYNC_SMR_NOT_AVAILABLE) + if(VSYNCER_CHECK_FULL) + set(TIMEOUT 3600) + else() + list(APPEND CFLAGS -DVSYNC_VERIFICATION_QUICK) + endif() # ########################################################################## # Define mode checker env vars # ########################################################################## @@ -88,12 +94,44 @@ function(add_vsyncer_check) foreach(WMM IN ITEMS ${WMMS}) set(TEST_NAME ${TARGET}_${WMM}) string(TOUPPER ${WMM} WMM_UP) - list(APPEND CFLAGS -DVSYNC_VERIFICATION_${WMM_UP}) - string(REPLACE ";" " " CFLAGS "${CFLAGS}") - string(REPLACE "\"'\"" "\"" CFLAGS "${CFLAGS}") + set(COMPILE_FLAGS ${CFLAGS} -DVSYNC_VERIFICATION_${WMM_UP}) + + # ###################################################################### + # Compile ll file with vsyncer + # ###################################################################### + + set(VSYNCER_CHECK_LL ${CMAKE_CURRENT_SOURCE_DIR}/${TEST_NAME}.ll) + + set(VSYNCER_COMPILE_CMD + env + CFLAGS="${COMPILE_FLAGS}" + vsyncer + compile + -d + --checker + ${CHECKER} + -o=${VSYNCER_CHECK_LL} + ${CLIENT} + | + tee + ${VSYNCER_CHECK_LL}.log + | + grep + -vE + "^# clang") + + add_custom_command( + OUTPUT ${VSYNCER_CHECK_LL} + COMMAND ${VSYNCER_COMPILE_CMD} + DEPENDS ${CLIENT}) + + add_custom_target(${TEST_NAME} ALL DEPENDS ${VSYNCER_CHECK_LL}) + + # ###################################################################### + # Run vsyncer check with ctest on the generated ll file + # ###################################################################### set(VSYNCER_CMD # env - CFLAGS=${CFLAGS} # ${CHECKER_ENV} # vsyncer check @@ -106,7 +144,9 @@ function(add_vsyncer_check) ${WMM} # --timeout ${TIMEOUT}s) - add_test(NAME ${TEST_NAME} COMMAND ${VSYNCER_CMD} ${CLIENT}) + add_test(NAME ${TEST_NAME} COMMAND ${VSYNCER_CMD} ${VSYNCER_CHECK_LL}) set_property(TEST ${TEST_NAME} PROPERTY SKIP_RETURN_CODE 1) + math(EXPR CTEST_TIMEOUT "${TIMEOUT} + 5") + set_tests_properties(${TEST_NAME} PROPERTIES TIMEOUT ${CTEST_TIMEOUT}) endforeach() endfunction() diff --git a/doc/api/vsync/GROUP_linearizable.md b/doc/api/vsync/GROUP_linearizable.md index 0d3ce0e..4e5aafe 100644 --- a/doc/api/vsync/GROUP_linearizable.md +++ b/doc/api/vsync/GROUP_linearizable.md @@ -13,13 +13,20 @@ _Group of algorithms linearizable algorithms._ | [vsync/map/listset_lazy.h](map/listset_lazy.h.md)|This is a partially parallel implementation with lock-free get. | ✔ | ❌ | ✔ | ❌ | | [vsync/map/listset_lf.h](map/listset_lf.h.md)|Lock-free implementation of listset. | ✔ | ✔ | ✔ | ❌ | | [vsync/map/listset_opt.h](map/listset_opt.h.md)|This implementation is an optimized verison of listset_fine. | ✔ | ❌ | ✔ | ❌ | +| [vsync/map/simpleht.h](map/simpleht.h.md)|Simple lock-free hashtable. | ✔ | ✔ | ❌ | ❌ | +| [vsync/map/treeset_bst_coarse.h](map/treeset_bst_coarse.h.md)|This implementation of treeset uses unbalanced binary search tree (BST) and coarse-grained locking. | ✔ | ❌ | ❌ | ❌ | +| [vsync/map/treeset_bst_fine.h](map/treeset_bst_fine.h.md)|This implementation of treeset uses unbalanced binary search tree (BST) and fine-grained locking. | ✔ | ❌ | ❌ | ❌ | +| [vsync/map/treeset_rb_coarse.h](map/treeset_rb_coarse.h.md)|This implementation of treeset uses balanced red-black tree (RB) and coarse-grained locking. | ✔ | ❌ | ❌ | ❌ | +| [vsync/map/treeset_rb_fine.h](map/treeset_rb_fine.h.md)|This implementation of treeset uses balanced red-black tree (RB) and fine-grained locking. | ✔ | ❌ | ❌ | ❌ | | [vsync/queue/bounded_locked.h](queue/bounded_locked.h.md)|Multi-producer, multi-consumer bounded queue protected by a spinlock. | ✔ | ❌ | ❌ | ❌ | | [vsync/queue/bounded_mpmc.h](queue/bounded_mpmc.h.md)|Lockless, multi-producer, multi-consumer bounded queue. | ✔ | ❌ | ❌ | ❌ | | [vsync/queue/bounded_spsc.h](queue/bounded_spsc.h.md)|Single-producer, single-consumer, wait-free bounded queue. | ✔ | ✔ | ❌ | ❌ | | [vsync/queue/unbounded_queue_lf.h](queue/unbounded_queue_lf.h.md)|Lock-free unbounded queue. | ✔ | ✔ | ✔ | ✔ | | [vsync/queue/unbounded_queue_lf_recycle.h](queue/unbounded_queue_lf_recycle.h.md)|Lock-free recycle unbounded queue. | ✔ | ✔ | ❌ | ✔ | | [vsync/queue/unbounded_queue_total.h](queue/unbounded_queue_total.h.md)|Unbounded blocking total queue. | ✔ | ❌ | ❌ | ✔ | +| [vsync/stack/elimination_stack.h](stack/elimination_stack.h.md)|Unbounded lock-free stack with elimination backoff. | ✔ | ✔ | ✔ | ❌ | | [vsync/stack/quack.h](stack/quack.h.md)|Lockfree concurrent stack/queue (Treiber stack) | ✔ | ✔ | ❌ | ❌ | +| [vsync/stack/xbo_stack.h](stack/xbo_stack.h.md)|Unbounded lock-free stack with exponential backoff. | ✔ | ✔ | ✔ | ❌ | --- diff --git a/doc/api/vsync/GROUP_lock_free.md b/doc/api/vsync/GROUP_lock_free.md index 7b205fe..7d7ea62 100644 --- a/doc/api/vsync/GROUP_lock_free.md +++ b/doc/api/vsync/GROUP_lock_free.md @@ -9,10 +9,13 @@ _Group of algorithms with lock-free progress condition._ | --- | --- | --- | --- | --- | --- | | [vsync/map/hashtable_standard.h](map/hashtable_standard.h.md)|This is a lock-free listset based hashtable. | ✔ | ✔ | ✔ | ❌ | | [vsync/map/listset_lf.h](map/listset_lf.h.md)|Lock-free implementation of listset. | ✔ | ✔ | ✔ | ❌ | +| [vsync/map/simpleht.h](map/simpleht.h.md)|Simple lock-free hashtable. | ✔ | ✔ | ❌ | ❌ | | [vsync/queue/bounded_spsc.h](queue/bounded_spsc.h.md)|Single-producer, single-consumer, wait-free bounded queue. | ✔ | ✔ | ❌ | ❌ | | [vsync/queue/unbounded_queue_lf.h](queue/unbounded_queue_lf.h.md)|Lock-free unbounded queue. | ✔ | ✔ | ✔ | ✔ | | [vsync/queue/unbounded_queue_lf_recycle.h](queue/unbounded_queue_lf_recycle.h.md)|Lock-free recycle unbounded queue. | ✔ | ✔ | ❌ | ✔ | +| [vsync/stack/elimination_stack.h](stack/elimination_stack.h.md)|Unbounded lock-free stack with elimination backoff. | ✔ | ✔ | ✔ | ❌ | | [vsync/stack/quack.h](stack/quack.h.md)|Lockfree concurrent stack/queue (Treiber stack) | ✔ | ✔ | ❌ | ❌ | +| [vsync/stack/xbo_stack.h](stack/xbo_stack.h.md)|Unbounded lock-free stack with exponential backoff. | ✔ | ✔ | ✔ | ❌ | --- diff --git a/doc/api/vsync/GROUP_requires_smr.md b/doc/api/vsync/GROUP_requires_smr.md index 1008940..c7879d0 100644 --- a/doc/api/vsync/GROUP_requires_smr.md +++ b/doc/api/vsync/GROUP_requires_smr.md @@ -16,6 +16,8 @@ Users are expected to couple the usage of these algorithms with an SMR scheme fr | [vsync/map/listset_lf.h](map/listset_lf.h.md)|Lock-free implementation of listset. | ✔ | ✔ | ✔ | ❌ | | [vsync/map/listset_opt.h](map/listset_opt.h.md)|This implementation is an optimized verison of listset_fine. | ✔ | ❌ | ✔ | ❌ | | [vsync/queue/unbounded_queue_lf.h](queue/unbounded_queue_lf.h.md)|Lock-free unbounded queue. | ✔ | ✔ | ✔ | ✔ | +| [vsync/stack/elimination_stack.h](stack/elimination_stack.h.md)|Unbounded lock-free stack with elimination backoff. | ✔ | ✔ | ✔ | ❌ | +| [vsync/stack/xbo_stack.h](stack/xbo_stack.h.md)|Unbounded lock-free stack with exponential backoff. | ✔ | ✔ | ✔ | ❌ | --- diff --git a/doc/api/vsync/README.md b/doc/api/vsync/README.md index fc04e6a..cf0a07d 100644 --- a/doc/api/vsync/README.md +++ b/doc/api/vsync/README.md @@ -16,11 +16,13 @@ _libvsync is library of robust atomics, synchronization primitives, concurrent d | Directory|Description| | --- | --- | | [vsync/atomic](atomic/README.md)|Rich interface of atomic operations and fences. | +| [vsync/bitmap](bitmap/README.md)|Bitmap implementations. | | [vsync/map](map/README.md)|This is a collection of algos that implement map interface. | | [vsync/queue](queue/README.md)|Queues, priority queues and ringbuffers. | | [vsync/smr](smr/README.md)|Safe Memory Reclamation Schemes. | | [vsync/spinlock](spinlock/README.md)|Spinlocks for kernel and userspace. | | [vsync/stack](stack/README.md)|Concurrent stacks. | +| [vsync/thread](thread/README.md)|Userspace synchronization primitives. | --- diff --git a/doc/api/vsync/atomic/await.h.md b/doc/api/vsync/atomic/await.h.md index df31c1e..c6882d3 100644 --- a/doc/api/vsync/atomic/await.h.md +++ b/doc/api/vsync/atomic/await.h.md @@ -40,12 +40,12 @@ node_t *next = vatomicptr_await_neq_acq(me->next, NULL); -The following example waits for the pointer me->next to be equal to pred. Once the condition is met, write NULL in me->next. The variable next contains the value that satisfied the condition. The operation has an release barrier. +The following example waits for the pointer me->next to be equal to pred. Once the condition is met, write NULL in me->next. The variable next contains the value that satisfied the condition. The operation has a release barrier. -``` -node_t *next = vatomicptr_await_eq_set_acq(me->next, pred, NULL); +```c +node_t *next = vatomicptr_await_eq_set_rel(me->next, pred, NULL); ``` diff --git a/doc/api/vsync/bitmap/README.md b/doc/api/vsync/bitmap/README.md new file mode 100644 index 0000000..a9ee9d3 --- /dev/null +++ b/doc/api/vsync/bitmap/README.md @@ -0,0 +1,13 @@ +# [vsync](../README.md) / bitmap +_Bitmap implementations._ + +--- +## File Index + + +| File|Description| +| --- | --- | +| [vsync/bitmap/bitmap.h](bitmap.h.md)|A bitmap implementation. | + + +--- diff --git a/doc/api/vsync/bitmap/bitmap.h.md b/doc/api/vsync/bitmap/bitmap.h.md new file mode 100644 index 0000000..8a0218c --- /dev/null +++ b/doc/api/vsync/bitmap/bitmap.h.md @@ -0,0 +1,327 @@ +# [vsync](../README.md) / [bitmap](README.md) / bitmap.h +_A bitmap implementation._ + +Bitmap with basic functionality. + + +### Example: + + + +```c + +#include +#include +#include +#include +#include + +#define WRITER_COUNT 11U +#define N (WRITER_COUNT + 1) +#define BIT_COUNT (VUINT64_WIDTH * WRITER_COUNT) + +vbitmap_t *g_bitmap; + +void +reader(void) +{ + vsize_t idx = 0; + vbitmap_iter_t iter = {0}; + vsize_t signal_count = 0; + vsize_t expected = BIT_COUNT - WRITER_COUNT; + + while (signal_count < expected) { + signal_count = 0; + vbitmap_iter_init(g_bitmap, &iter); + + while (vbitmap_iter_next(&iter, &idx)) { + signal_count++; + } + printf("%zu bits were raised, expected %zu\n", signal_count, expected); + } +} + +void +writer(vsize_t tid) +{ + vbool_t set = false; + + // each writer sets 64 bits + vsize_t from = tid * VUINT64_WIDTH; + vsize_t to = from + VUINT64_WIDTH - 1; + + vbitmap_set_range(g_bitmap, from, to); + + set = vbitmap_get(g_bitmap, from); + ASSERT(set == true); + + vbitmap_clr_bit(g_bitmap, from); + set = vbitmap_get(g_bitmap, from); + ASSERT(set == false); + + vbitmap_set_bit(g_bitmap, from); + set = vbitmap_get(g_bitmap, from); + ASSERT(set == true); +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + if (tid == WRITER_COUNT) { + reader(); + } else { + writer(tid); + } + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + vsize_t size = vbitmap_size(BIT_COUNT); + g_bitmap = malloc(size); + vbitmap_init(g_bitmap, BIT_COUNT, false); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + free(g_bitmap); + + return 0; +} +``` + + + + +### References: + +The interface is inspired by and similar to [ck_bitmap](https://github.com/concurrencykit/ck/blob/master/include/ck_bitmap.h) + +--- +# Functions + +| Function | Description | +|---|---| +| [vbitmap_size](bitmap.h.md#function-vbitmap_size) | Calculates the size of vbitmap_t object based on the given number of bits. | +| [vbitmap_init](bitmap.h.md#function-vbitmap_init) | Initializes the given `bitmap` object. | +| [vbitmap_clear](bitmap.h.md#function-vbitmap_clear) | Clears all bits in the given bitmap. | +| [vbitmap_get](bitmap.h.md#function-vbitmap_get) | Returns the value of the bit at index `bit_idx`. | +| [vbitmap_set_bit](bitmap.h.md#function-vbitmap_set_bit) | Sets the bit at index `bit_idx`. | +| [vbitmap_set_range](bitmap.h.md#function-vbitmap_set_range) | Sets all bits with indexes that are in range `[from:to]`. | +| [vbitmap_clr_range](bitmap.h.md#function-vbitmap_clr_range) | Clears all bits with indexes that are in range `[from:to]`. | +| [vbitmap_clr_bit](bitmap.h.md#function-vbitmap_clr_bit) | Clears the bit at index `bit_idx`. | +| [vbitmap_iter_init](bitmap.h.md#function-vbitmap_iter_init) | Initializes the given vbitmap_iter_t object. | +| [vbitmap_iter_next](bitmap.h.md#function-vbitmap_iter_next) | Iterates to the next set bit. | + +## Function `vbitmap_size` + +```c +static vsize_t vbitmap_size(vsize_t bit_count) +``` +_Calculates the size of vbitmap_t object based on the given number of bits._ + + + + +**Parameters:** + +- `bit_count`: number of bits. + + +**Returns:** vsize_t size of vbitmap_t object needed to accommodate `bit_count` of bits. + + + +## Function `vbitmap_init` + +```c +static void vbitmap_init(vbitmap_t *bitmap, vsize_t bit_count, vbool_t set) +``` +_Initializes the given_ `bitmap` _object._ + + +All bits are unset initially. + + + +**Parameters:** + +- `bitmap`: address of vbitmap_t object. +- `bit_count`: number of bits. +- `set`: true if all bits should be set, false otherwise. + + + + +## Function `vbitmap_clear` + +```c +static void vbitmap_clear(vbitmap_t *bitmap) +``` +_Clears all bits in the given bitmap._ + + + + +**Parameters:** + +- `bitmap`: address of vbitmap_t object. + + + + +## Function `vbitmap_get` + +```c +static vbool_t vbitmap_get(vbitmap_t *bitmap, vsize_t bit_idx) +``` +_Returns the value of the bit at index_ `bit_idx`_._ + + + + +**Parameters:** + +- `bitmap`: address of vbitmap_t object. +- `bit_idx`: index of the bit to set. + + +**Returns:** true the bit is set. + +**Returns:** false the bit is unset. + + + +## Function `vbitmap_set_bit` + +```c +static void vbitmap_set_bit(vbitmap_t *bitmap, vsize_t bit_idx) +``` +_Sets the bit at index_ `bit_idx`_._ + + + + +**Parameters:** + +- `bitmap`: address of vbitmap_t object. +- `bit_idx`: index of the bit to set. + + + + +## Function `vbitmap_set_range` + +```c +static void vbitmap_set_range(vbitmap_t *bitmap, vsize_t from, vsize_t to) +``` +_Sets all bits with indexes that are in range_ `[from:to]`_._ + + +> **Note:** the range is inclusive both bits at index `from` and index `to` will be set. + + + +**Parameters:** + +- `bitmap`: address of vbitmap_t object. +- `from`: index of the LSB in the range. +- `to`: index of the MSB in the range. + + +> **Note:** This function is not atomic. Bits inside the range might not be raised at once. Raising bits occur in multiple steps. + + +## Function `vbitmap_clr_range` + +```c +static void vbitmap_clr_range(vbitmap_t *bitmap, vsize_t from, vsize_t to) +``` +_Clears all bits with indexes that are in range_ `[from:to]`_._ + + +> **Note:** the range is inclusive both bits at index `from` and index `to` will be set. + + + +**Parameters:** + +- `bitmap`: address of vbitmap_t object. +- `from`: index of the LSB in the range. +- `to`: index of the MSB in the range. + + +> **Note:** This function is not atomic. Bits inside the range might not be cleared at once. Clearing bits occur in multiple steps. + + +## Function `vbitmap_clr_bit` + +```c +static void vbitmap_clr_bit(vbitmap_t *bitmap, vsize_t bit_idx) +``` +_Clears the bit at index_ `bit_idx`_._ + + + + +**Parameters:** + +- `bitmap`: address of vbitmap_t object. +- `bit_idx`: index of the bit to unset. + + + + +## Function `vbitmap_iter_init` + +```c +static void vbitmap_iter_init(vbitmap_t *bitmap, vbitmap_iter_t *iter) +``` +_Initializes the given vbitmap_iter_t object._ + + + + +**Parameters:** + +- `bitmap`: address of vbitmap_t to iterate over. +- `iter`: address of vbitmap_iter_t object. + + + + +## Function `vbitmap_iter_next` + +```c +static vbool_t vbitmap_iter_next(vbitmap_iter_t *iter, vsize_t *out_bit_idx) +``` +_Iterates to the next set bit._ + + + + +**Parameters:** + +- `iter`: address of vbitmap_iter_t object. +- `out_bit_idx`: index of the next set bit. + + +**Returns:** true next set bit is found. + +**Returns:** false no more set bits are found. + +**Precondition:** call `vbitmap_iter_init` to initialize `iter` before first use. + + + + +--- diff --git a/doc/api/vsync/map/README.md b/doc/api/vsync/map/README.md index da6217e..4b5f46a 100644 --- a/doc/api/vsync/map/README.md +++ b/doc/api/vsync/map/README.md @@ -13,6 +13,11 @@ _This is a collection of algos that implement map interface._ | [vsync/map/listset_lazy.h](listset_lazy.h.md)|This is a partially parallel implementation with lock-free get. | ✔ | ❌ | ✔ | | [vsync/map/listset_lf.h](listset_lf.h.md)|Lock-free implementation of listset. | ✔ | ✔ | ✔ | | [vsync/map/listset_opt.h](listset_opt.h.md)|This implementation is an optimized verison of listset_fine. | ✔ | ❌ | ✔ | +| [vsync/map/simpleht.h](simpleht.h.md)|Simple lock-free hashtable. | ✔ | ✔ | ❌ | +| [vsync/map/treeset_bst_coarse.h](treeset_bst_coarse.h.md)|This implementation of treeset uses unbalanced binary search tree (BST) and coarse-grained locking. | ✔ | ❌ | ❌ | +| [vsync/map/treeset_bst_fine.h](treeset_bst_fine.h.md)|This implementation of treeset uses unbalanced binary search tree (BST) and fine-grained locking. | ✔ | ❌ | ❌ | +| [vsync/map/treeset_rb_coarse.h](treeset_rb_coarse.h.md)|This implementation of treeset uses balanced red-black tree (RB) and coarse-grained locking. | ✔ | ❌ | ❌ | +| [vsync/map/treeset_rb_fine.h](treeset_rb_fine.h.md)|This implementation of treeset uses balanced red-black tree (RB) and fine-grained locking. | ✔ | ❌ | ❌ | --- diff --git a/doc/api/vsync/map/simpleht.h.md b/doc/api/vsync/map/simpleht.h.md new file mode 100644 index 0000000..582c37b --- /dev/null +++ b/doc/api/vsync/map/simpleht.h.md @@ -0,0 +1,410 @@ +# [vsync](../README.md) / [map](README.md) / simpleht.h +_Simple lock-free hashtable._ + +**Groups:** [Linearizable](../GROUP_linearizable.md), [Lock-free](../GROUP_lock_free.md) + +This is an adaptation of the simple lock-free hashtable mentioned in the reference. The adaptation allow for remove operation if users need it. Users should be aware that enabling the remove incurs overhead on add/get operations. The remove operation can be quite slow as it rebalances the hashtable and when that happens lock-freedom is compromised. + +This hashtable is ideal for when no remove operations are needed, in that case users should disable the support of remove by compiling with `-DVSIMPLEHT_DISABLE_REMOVE`. Or keep it, and make sure remove is rarely called. + +# Operating Conditions + +The hashtable is supposed to keep gaps for optimal performance. Users should create the table with at least a capacity of 2*N, where N is the maximum number of entries to be inserted. + +As long as the hashtable is not full, every search is guaranteed to finish either by locating the desired key, or by locating an entry whose key is zero, which means that the desired key does not exist in the hashtable. + +There is a tradeoff between performance and the number of empty slots in the hashtable. The performance degrades as the hashtable fills. lock-freedom of add is compromised if the hashtable becomes full. + + +### Example: + + + +```c + +#include +#include +#include +#include +#include + +#define N 3U +#define HASHTABLE_CAPACITY 32U +#define MIN_KEY 1U +#define MAX_KEY (HASHTABLE_CAPACITY / 2) + +typedef struct data_s { + vuintptr_t key; + // data +} data_t; + +vsimpleht_t g_hashtable; + +vuint64_t +hash_cb(vuintptr_t key) +{ + vuint64_t h = key; + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; +} + +vint8_t +cmp_key_cb(vuintptr_t a, vuintptr_t b) +{ + if (a < b) { + return -1; + } else if (a > b) { + return 1; + } else { + return 0; + } +} + +void +destroy_cb(void *data) +{ + free(data); +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + data_t *data = NULL; + vsimpleht_ret_t ret = VSIMPLEHT_RET_OK; + + vsimpleht_thread_register(&g_hashtable); + + for (vuintptr_t key = MIN_KEY; key < MAX_KEY; key++) { + data = vsimpleht_get(&g_hashtable, key); + if (data) { + printf("T%zu: found key %lu\n", tid, key); + ASSERT(data->key == key); + ret = vsimpleht_remove(&g_hashtable, key); + if (ret == VSIMPLEHT_RET_OK) { + printf("T%zu: removed key %lu\n", tid, key); + } else { + printf("T%zu: key %lu does not exist\n", tid, key); + } + } else { + data = malloc(sizeof(data_t)); + data->key = key; + ret = vsimpleht_add(&g_hashtable, key, data); + if (ret == VSIMPLEHT_RET_OK) { + printf("T%zu: added key %lu\n", tid, key); + } else if (ret == VSIMPLEHT_RET_TBL_FULL) { + printf("T%zu: table is full! cannot add\n", tid); + free(data); + } else if (ret == VSIMPLEHT_RET_KEY_EXISTS) { + printf("T%zu: key %lu already exists\n", tid, key); + free(data); + } + } + } + vsimpleht_thread_deregister(&g_hashtable); + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + vsize_t size = vsimpleht_buff_size(HASHTABLE_CAPACITY); + void *buffer = malloc(size); + + vsimpleht_init(&g_hashtable, buffer, HASHTABLE_CAPACITY, cmp_key_cb, + hash_cb, destroy_cb); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + vsimpleht_destroy(&g_hashtable); + free(buffer); + return 0; +} +``` + + + + +### References: + +[The World's Simplest Lock-Free Hash Table](https://preshing.com/20130605/the-worlds-simplest-lock-free-hash-table/) + +--- +# Macros + +| Macro | Description | +|---|---| +| [VSIMPLEHT_RELATIVE_THRESHOLD](simpleht.h.md#macro-vsimpleht_relative_threshold) | Controls when the rebalancing of the hashtable is triggered. | + +## Macro `VSIMPLEHT_RELATIVE_THRESHOLD` + + +_Controls when the rebalancing of the hashtable is triggered._ + + +Everytime the number of removed nodes is equal to capacity/VSIMPLEHT_RELATIVE_THRESHOLD. + + +--- +# Functions + +| Function | Description | +|---|---| +| [vsimpleht_buff_size](simpleht.h.md#function-vsimpleht_buff_size) | Calculates the size of the buffer needed by the hashtable. | +| [vsimpleht_init](simpleht.h.md#function-vsimpleht_init) | Initializes the hashtable. | +| [vsimpleht_destroy](simpleht.h.md#function-vsimpleht_destroy) | Destroys the stored hashtable values. | +| [vsimpleht_thread_register](simpleht.h.md#function-vsimpleht_thread_register) | Registers the caller thread. | +| [vsimpleht_thread_deregister](simpleht.h.md#function-vsimpleht_thread_deregister) | Deregisters the caller thread. | +| [vsimpleht_add](simpleht.h.md#function-vsimpleht_add) | Inserts the given value into the hashtable. | +| [vsimpleht_get](simpleht.h.md#function-vsimpleht_get) | Searches the hashtable for a value associated with the given key. | +| [vsimpleht_iter_init](simpleht.h.md#function-vsimpleht_iter_init) | Initializes the given iterator object `iter` for operating `tbl` object. | +| [vsimpleht_iter_next](simpleht.h.md#function-vsimpleht_iter_next) | Reads the current entry (key/value) the iterator has reached. | +| [vsimpleht_remove](simpleht.h.md#function-vsimpleht_remove) | Removes the node associated with the given key from the hashtable. | + +## Function `vsimpleht_buff_size` + +```c +static vsize_t vsimpleht_buff_size(vsize_t capacity) +``` +_Calculates the size of the buffer needed by the hashtable._ + + + + +**Parameters:** + +- `capacity`: capacity of the hashtable. Maximum number of items to be inserted in the hashtable. + + +**Returns:** vsize_t required buffer size to fit `len` items. + +> **Note:** capacity must be power of two. + + +## Function `vsimpleht_init` + +```c +static void vsimpleht_init(vsimpleht_t *tbl, void *buff, vsize_t capacity, vsimpleht_cmp_key_t cmp_fun, vsimpleht_hash_key_t hash_fun, vsimpleht_destroy_entry_t destroy_cb) +``` +_Initializes the hashtable._ + + + + +**Parameters:** + +- `tbl`: address of vsimpleht_t object. +- `buff`: address of the hashtable buffer. Allocated by the user. +- `capacity`: capacity of the hashtable. Same value passed to vsimpleht_buff_size. +- `cmp_key`: compare key callback function address. +- `hash_key`: address of callback function. Used to hash the key. +- `destroy_cb`: address of destroy callback function. Used to destroy the hashtable stored objects. Called via `vsimpleht_remove` and `vsimpleht_destroy`. + + + + +## Function `vsimpleht_destroy` + +```c +static void vsimpleht_destroy(vsimpleht_t *tbl) +``` +_Destroys the stored hashtable values._ + + + + +**Parameters:** + +- `tbl`: address of vsimpleht_t object. + + + + +## Function `vsimpleht_thread_register` + +```c +static void vsimpleht_thread_register(vsimpleht_t *tbl) +``` +_Registers the caller thread._ + + + + +**Parameters:** + +- `tbl`: address of vsimpleht_t object. + + +> **Note:** should be called once per thread, before the calling thread accesses the hashtable for the first time. + + +## Function `vsimpleht_thread_deregister` + +```c +static void vsimpleht_thread_deregister(vsimpleht_t *tbl) +``` +_Deregisters the caller thread._ + + + + +**Parameters:** + +- `tbl`: address of vsimpleht_t object. + + +> **Note:** should be called once per thread, after the calling thread accesses the hashtable for the last time. + + +## Function `vsimpleht_add` + +```c +static vsimpleht_ret_t vsimpleht_add(vsimpleht_t *tbl, vuintptr_t key, void *value) +``` +_Inserts the given value into the hashtable._ + + + + +**Parameters:** + +- `tbl`: address of vsimpleht_t object. +- `key`: key of the value to add. +- `value`: address of the object to insert. + + +**Returns:** VSIMPLEHT_RET_OK key does not exist, value was added. + +**Returns:** VSIMPLEHT_RET_KEY_EXISTS key exists, value was not added. + +**Returns:** VSIMPLEHT_RET_TBL_FULL table is full. Consider +- increasing the capacity +- decreasing VSIMPLEHT_RELATIVE_THRESHOLD +- removing items from the table> **Note:** neither key can be 0 nor value can be NULL. + + + + +## Function `vsimpleht_get` + +```c +static void* vsimpleht_get(vsimpleht_t *tbl, vuintptr_t key) +``` +_Searches the hashtable for a value associated with the given key._ + + + + +**Parameters:** + +- `tbl`: address of vsimpleht_t object. +- `key`: key to search for. + + +**Returns:** void* address of the object associated with the given key if exists. + +**Returns:** NULL if there is no value found associated with given key. + + + +## Function `vsimpleht_iter_init` + +```c +static void vsimpleht_iter_init(vsimpleht_t *tbl, vsimpleht_iter_t *iter) +``` +_Initializes the given iterator object_ `iter` _for operating_ `tbl` _object._ + + + + +**Parameters:** + +- `tbl`: address of vsimpleht_t object. +- `iter`: address of vsimpleht_iter_t object to initialize. + + + + +## Function `vsimpleht_iter_next` + +```c +static vbool_t vsimpleht_iter_next(vsimpleht_iter_t *iter, vuintptr_t *key, void **val) +``` +_Reads the current entry (key/value) the iterator has reached._ + + +After the values are read, the iterator advances to the next entry. + + + +**Parameters:** + +- `iter`: address of vsimpleht_iter_t object. +- `key`: output parameter where the current key value is stored. +- `val`: output parameter where the current object value is stored. + + +**Returns:** true the current output of key and val is valid. + +**Returns:** false there are no more objects to iterate. Stored values in key, val parameters should be ignored by the user. + +**Precondition:** iter must be initalized via vsimpleht_iter_init. + +> **Note:** the iterator is mainly meant for sequential setting. When called in parallel the results might not be consistent. + + +### Example: + + + +```c +vuintptr_t key = 0; +data_t *date = NULL; +vsimpleht_iter_t iter; +vsimpleht_iter_init(&g_simpleht, &iter); + while (vsimpleht_iter_next(&iter, &key, (void **)&data)) { + // access key + // access data + } +``` + + + + +## Function `vsimpleht_remove` + +```c +static vsimpleht_ret_t vsimpleht_remove(vsimpleht_t *tbl, vuintptr_t key) +``` +_Removes the node associated with the given key from the hashtable._ + + + + +**Parameters:** + +- `tbl`: address of vsimpleht_t object. +- `key`: key to remove the value associated with. + + +**Returns:** VSIMPLEHT_RET_OK key exists, node was removed. + +**Returns:** VSIMPLEHT_RET_KEY_DNE key does not exist. + +> **Note:** this operation is not lock-free, if the the removal is successful. It can be super-slow, and the associated value can be freed via the destroy callback registered at `vsimpleht_init`. + + + +--- diff --git a/doc/api/vsync/map/treeset_bst_coarse.h.md b/doc/api/vsync/map/treeset_bst_coarse.h.md new file mode 100644 index 0000000..ce3834b --- /dev/null +++ b/doc/api/vsync/map/treeset_bst_coarse.h.md @@ -0,0 +1,277 @@ +# [vsync](../README.md) / [map](README.md) / treeset_bst_coarse.h +_This implementation of treeset uses unbalanced binary search tree (BST) and coarse-grained locking._ + +**Groups:** [Linearizable](../GROUP_linearizable.md) + +A treeset is a binary search tree with nodes that are ordered by unique keys. + +Available implementations share the same interface, but differ in three orthogonal aspects: +- balancing scheme: unbalanced binary search tree (BST) or balanced red-black tree (RB), +- synchronisation scheme: so far only coarse-grained locking and fine-grained locking, +- allocation scheme: so far only malloc-like allocation. + + +All implementations use external trees, where values are stored in external nodes (leaves) of the tree, and internal nodes are used only for routing. + +For lock-based implementations one need to choose desired lock implementation: +- `-DTREESET_LOCK_PTHREAD` (for pthread mutex), +- `-DTREESET_LOCK_TTAS` (for libvsync TTAS lock), +- `-DTREESET_LOCK_RW` (for libvsync readers-writer lock). +### Example: + + + + +```c + +#define TREESET_LOCK_RW +#include + +#include +#include +#include + +#define N 4 +#define MIN_KEY 0 +#define MAX_KEY 3 + +typedef vuintptr_t value_t; + +treeset_t tree; + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)args; + + for (treeset_key_t key = MIN_KEY; key <= MAX_KEY; key++) { + value_t value = tid; + value_t old_value; + + // insert + vbool_t res = + treeset_add(&tree, key, (void *)value, (void *)&old_value); + + if (res) { + // insert succeeded + printf("[%lu] key %lu inserted\n", tid, key); + } else { + printf( + "[%lu] key %lu not inserted, already in tree with value %lu\n", + tid, key, old_value); + } + + // search + res = treeset_contains(&tree, key, (void *)&old_value); + + if (res) { + // search successful + printf("[%lu] key %lu in tree with value %lu\n", tid, key, + old_value); + } else { + printf("[%lu] key %lu not in tree\n", tid, key); + } + + // remove + res = treeset_remove(&tree, key, (void *)&old_value); + + if (res) { + // remove succeeded + printf("[%lu] key %lu removed, old value was %lu\n", tid, key, + old_value); + } else { + printf("[%lu] key %lu not removed\n", tid, key); + } + } + + return NULL; +} + +void * +malloc_cb(vsize_t sz, void *arg) +{ + (void)arg; + return malloc(sz); +} + +void +free_cb(void *ptr, void *arg) +{ + (void)arg; + free(ptr); +} + +void +free_visitor(treeset_key_t key, void *value, void *arg) +{ + (void)key; + (void)arg; + free((value_t *)value); +} + +int +main(void) +{ + pthread_t threads[N]; + + vmem_lib_t mem_lib = {.free_fun = free_cb, + .malloc_fun = malloc_cb, + .arg = NULL}; + + treeset_init(&tree, mem_lib); + + for (vsize_t i = 0; i < N; ++i) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; ++i) { + pthread_join(threads[i], NULL); + } + + treeset_visit(&tree, free_visitor, NULL); + treeset_destroy(&tree); + + return 0; +} +``` + + + +--- +# Functions + +| Function | Description | +|---|---| +| [treeset_init](treeset_bst_coarse.h.md#function-treeset_init) | Initializes the treeset. | +| [treeset_destroy](treeset_bst_coarse.h.md#function-treeset_destroy) | Destroys all the remaining nodes in the treeset. | +| [treeset_add](treeset_bst_coarse.h.md#function-treeset_add) | Attempts to insert an element with a given key and value into the treeset. | +| [treeset_remove](treeset_bst_coarse.h.md#function-treeset_remove) | Attempts to remove an element with a given key from the treeset. | +| [treeset_contains](treeset_bst_coarse.h.md#function-treeset_contains) | Searches the treeset for an element with a given key. | +| [treeset_visit](treeset_bst_coarse.h.md#function-treeset_visit) | Visits all elements in the treeset. | + +## Function `treeset_init` + +```c +static void treeset_init(treeset_t *tree, vmem_lib_t mem_lib) +``` +_Initializes the treeset._ + + +> **Note:** must be called before threads access the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `mem_lib`: object of type `vmem_lib_t` containing malloc/free functions to allocate/free internal nodes. + + + + +## Function `treeset_destroy` + +```c +static void treeset_destroy(treeset_t *tree) +``` +_Destroys all the remaining nodes in the treeset._ + + +> **Note:** call only after thread join, or after all threads finished accessing the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. + + + + +## Function `treeset_add` + +```c +static vbool_t treeset_add(treeset_t *tree, treeset_key_t key, void *value, void **out_value) +``` +_Attempts to insert an element with a given key and value into the treeset._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be inserted. +- `value`: value to be associated with inserted key. +- `out_value`: out parameter for the previous value associated with the key. + + +**Returns:** true operation succeeded. + +**Returns:** false operation failed, since the given key was already in the treeset, in the `out_value` the value of this element is returned. + + + +## Function `treeset_remove` + +```c +static vbool_t treeset_remove(treeset_t *tree, treeset_key_t key, void **out_value) +``` +_Attempts to remove an element with a given key from the treeset._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be removed. +- `out_value`: out parameter for the value associated with the key. + + +**Returns:** true operation succeeded, in the `out_value` the value of the removed element is returned. + +**Returns:** false operation failed, there is no element with the given key. + + + +## Function `treeset_contains` + +```c +static vbool_t treeset_contains(treeset_t *tree, treeset_key_t key, void **out_value) +``` +_Searches the treeset for an element with a given key._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be searched for. +- `out_value`: out parameter for the value associated with the key. + + +**Returns:** true operation succeeded, in the `out_value` the value of the found element is returned. + +**Returns:** false operation failed, there is no element with the given key. + + + +## Function `treeset_visit` + +```c +static void treeset_visit(treeset_t *tree, treeset_visitor visitor, void *arg) +``` +_Visits all elements in the treeset._ + + +> **Note:** call only after thread join, or after all threads finished accessing the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `visitor`: address of the function to call on each element. +- `arg`: the third argument to the visitor function. + + + + + +--- diff --git a/doc/api/vsync/map/treeset_bst_fine.h.md b/doc/api/vsync/map/treeset_bst_fine.h.md new file mode 100644 index 0000000..bb18a03 --- /dev/null +++ b/doc/api/vsync/map/treeset_bst_fine.h.md @@ -0,0 +1,265 @@ +# [vsync](../README.md) / [map](README.md) / treeset_bst_fine.h +_This implementation of treeset uses unbalanced binary search tree (BST) and fine-grained locking._ + +**Groups:** [Linearizable](../GROUP_linearizable.md) + +Refer to [treeset_bst_coarse.h](treeset_bst_coarse.h.md) for more general information about treeset. + + +### Example: + + + +```c + +#define TREESET_LOCK_RW +#include + +#include +#include +#include + +#define N 4 +#define MIN_KEY 0 +#define MAX_KEY 3 + +typedef vuintptr_t value_t; + +treeset_t tree; + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)args; + + for (treeset_key_t key = MIN_KEY; key <= MAX_KEY; key++) { + value_t value = tid; + value_t old_value; + + // insert + vbool_t res = + treeset_add(&tree, key, (void *)value, (void *)&old_value); + + if (res) { + // insert succeeded + printf("[%lu] key %lu inserted\n", tid, key); + } else { + printf( + "[%lu] key %lu not inserted, already in tree with value %lu\n", + tid, key, old_value); + } + + // search + res = treeset_contains(&tree, key, (void *)&old_value); + + if (res) { + // search successful + printf("[%lu] key %lu in tree with value %lu\n", tid, key, + old_value); + } else { + printf("[%lu] key %lu not in tree\n", tid, key); + } + + // remove + res = treeset_remove(&tree, key, (void *)&old_value); + + if (res) { + // remove succeeded + printf("[%lu] key %lu removed, old value was %lu\n", tid, key, + old_value); + } else { + printf("[%lu] key %lu not removed\n", tid, key); + } + } + + return NULL; +} + +void * +malloc_cb(vsize_t sz, void *arg) +{ + (void)arg; + return malloc(sz); +} + +void +free_cb(void *ptr, void *arg) +{ + (void)arg; + free(ptr); +} + +void +free_visitor(treeset_key_t key, void *value, void *arg) +{ + (void)key; + (void)arg; + free((value_t *)value); +} + +int +main(void) +{ + pthread_t threads[N]; + + vmem_lib_t mem_lib = {.free_fun = free_cb, + .malloc_fun = malloc_cb, + .arg = NULL}; + + treeset_init(&tree, mem_lib); + + for (vsize_t i = 0; i < N; ++i) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; ++i) { + pthread_join(threads[i], NULL); + } + + treeset_visit(&tree, free_visitor, NULL); + treeset_destroy(&tree); + + return 0; +} +``` + + + +--- +# Functions + +| Function | Description | +|---|---| +| [treeset_init](treeset_bst_fine.h.md#function-treeset_init) | Initializes the treeset. | +| [treeset_destroy](treeset_bst_fine.h.md#function-treeset_destroy) | Destroys all the remaining nodes in the treeset. | +| [treeset_add](treeset_bst_fine.h.md#function-treeset_add) | Attempts to insert an element with a given key and value into the treeset. | +| [treeset_remove](treeset_bst_fine.h.md#function-treeset_remove) | Attempts to remove an element with a given key from the treeset. | +| [treeset_contains](treeset_bst_fine.h.md#function-treeset_contains) | Searches the treeset for an element with a given key. | +| [treeset_visit](treeset_bst_fine.h.md#function-treeset_visit) | Visits all elements in the treeset. | + +## Function `treeset_init` + +```c +static void treeset_init(treeset_t *tree, vmem_lib_t mem_lib) +``` +_Initializes the treeset._ + + +> **Note:** must be called before threads access the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `mem_lib`: object of type `vmem_lib_t` containing malloc/free functions to allocate/free internal nodes. + + + + +## Function `treeset_destroy` + +```c +static void treeset_destroy(treeset_t *tree) +``` +_Destroys all the remaining nodes in the treeset._ + + +> **Note:** call only after thread join, or after all threads finished accessing the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. + + + + +## Function `treeset_add` + +```c +static vbool_t treeset_add(treeset_t *tree, treeset_key_t key, void *value, void **out_value) +``` +_Attempts to insert an element with a given key and value into the treeset._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be inserted. +- `value`: value to be associated with inserted key. +- `out_value`: out parameter for the previous value associated with the key. + + +**Returns:** true operation succeeded. + +**Returns:** false operation failed, since the given key was already in the treeset, in the `out_value` the value of this element is returned. + + + +## Function `treeset_remove` + +```c +static vbool_t treeset_remove(treeset_t *tree, treeset_key_t key, void **out_value) +``` +_Attempts to remove an element with a given key from the treeset._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be removed. +- `out_value`: out parameter for the value associated with the key. + + +**Returns:** true operation succeeded, in the `out_value` the value of the removed element is returned. + +**Returns:** false operation failed, there is no element with the given key. + + + +## Function `treeset_contains` + +```c +static vbool_t treeset_contains(treeset_t *tree, treeset_key_t key, void **out_value) +``` +_Searches the treeset for an element with a given key._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be searched for. +- `out_value`: out parameter for the value associated with the key. + + +**Returns:** true operation succeeded, in the `out_value` the value of the found element is returned. + +**Returns:** false operation failed, there is no element with the given key. + + + +## Function `treeset_visit` + +```c +static void treeset_visit(treeset_t *tree, treeset_visitor visitor, void *arg) +``` +_Visits all elements in the treeset._ + + +> **Note:** call only after thread join, or after all threads finished accessing the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `visitor`: address of the function to call on each element. +- `arg`: the third argument to the visitor function. + + + + + +--- diff --git a/doc/api/vsync/map/treeset_rb_coarse.h.md b/doc/api/vsync/map/treeset_rb_coarse.h.md new file mode 100644 index 0000000..f05ebeb --- /dev/null +++ b/doc/api/vsync/map/treeset_rb_coarse.h.md @@ -0,0 +1,265 @@ +# [vsync](../README.md) / [map](README.md) / treeset_rb_coarse.h +_This implementation of treeset uses balanced red-black tree (RB) and coarse-grained locking._ + +**Groups:** [Linearizable](../GROUP_linearizable.md) + +Refer to [treeset_bst_coarse.h](treeset_bst_coarse.h.md) for more general information about treeset. + + +### Example: + + + +```c + +#define TREESET_LOCK_RW +#include + +#include +#include +#include + +#define N 4 +#define MIN_KEY 0 +#define MAX_KEY 3 + +typedef vuintptr_t value_t; + +treeset_t tree; + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)args; + + for (treeset_key_t key = MIN_KEY; key <= MAX_KEY; key++) { + value_t value = tid; + value_t old_value; + + // insert + vbool_t res = + treeset_add(&tree, key, (void *)value, (void *)&old_value); + + if (res) { + // insert succeeded + printf("[%lu] key %lu inserted\n", tid, key); + } else { + printf( + "[%lu] key %lu not inserted, already in tree with value %lu\n", + tid, key, old_value); + } + + // search + res = treeset_contains(&tree, key, (void *)&old_value); + + if (res) { + // search successful + printf("[%lu] key %lu in tree with value %lu\n", tid, key, + old_value); + } else { + printf("[%lu] key %lu not in tree\n", tid, key); + } + + // remove + res = treeset_remove(&tree, key, (void *)&old_value); + + if (res) { + // remove succeeded + printf("[%lu] key %lu removed, old value was %lu\n", tid, key, + old_value); + } else { + printf("[%lu] key %lu not removed\n", tid, key); + } + } + + return NULL; +} + +void * +malloc_cb(vsize_t sz, void *arg) +{ + (void)arg; + return malloc(sz); +} + +void +free_cb(void *ptr, void *arg) +{ + (void)arg; + free(ptr); +} + +void +free_visitor(treeset_key_t key, void *value, void *arg) +{ + (void)key; + (void)arg; + free((value_t *)value); +} + +int +main(void) +{ + pthread_t threads[N]; + + vmem_lib_t mem_lib = {.free_fun = free_cb, + .malloc_fun = malloc_cb, + .arg = NULL}; + + treeset_init(&tree, mem_lib); + + for (vsize_t i = 0; i < N; ++i) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; ++i) { + pthread_join(threads[i], NULL); + } + + treeset_visit(&tree, free_visitor, NULL); + treeset_destroy(&tree); + + return 0; +} +``` + + + +--- +# Functions + +| Function | Description | +|---|---| +| [treeset_init](treeset_rb_coarse.h.md#function-treeset_init) | Initializes the treeset. | +| [treeset_destroy](treeset_rb_coarse.h.md#function-treeset_destroy) | Destroys all the remaining nodes in the treeset. | +| [treeset_add](treeset_rb_coarse.h.md#function-treeset_add) | Attempts to insert an element with a given key and value into the treeset. | +| [treeset_remove](treeset_rb_coarse.h.md#function-treeset_remove) | Attempts to remove an element with a given key from the treeset. | +| [treeset_contains](treeset_rb_coarse.h.md#function-treeset_contains) | Searches the treeset for an element with a given key. | +| [treeset_visit](treeset_rb_coarse.h.md#function-treeset_visit) | Visits all elements in the treeset. | + +## Function `treeset_init` + +```c +static void treeset_init(treeset_t *tree, vmem_lib_t mem_lib) +``` +_Initializes the treeset._ + + +> **Note:** must be called before threads access the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `mem_lib`: object of type `vmem_lib_t` containing malloc/free functions to allocate/free internal nodes. + + + + +## Function `treeset_destroy` + +```c +static void treeset_destroy(treeset_t *tree) +``` +_Destroys all the remaining nodes in the treeset._ + + +> **Note:** call only after thread join, or after all threads finished accessing the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. + + + + +## Function `treeset_add` + +```c +static vbool_t treeset_add(treeset_t *tree, treeset_key_t key, void *value, void **out_value) +``` +_Attempts to insert an element with a given key and value into the treeset._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be inserted. +- `value`: value to be associated with inserted key. +- `out_value`: out parameter for the previous value associated with the key. + + +**Returns:** true operation succeeded. + +**Returns:** false operation failed, since the given key was already in the treeset, in the `out_value` the value of this element is returned. + + + +## Function `treeset_remove` + +```c +static vbool_t treeset_remove(treeset_t *tree, treeset_key_t key, void **out_value) +``` +_Attempts to remove an element with a given key from the treeset._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be removed. +- `out_value`: out parameter for the value associated with the key. + + +**Returns:** true operation succeeded, in the `out_value` the value of the removed element is returned. + +**Returns:** false operation failed, there is no element with the given key. + + + +## Function `treeset_contains` + +```c +static vbool_t treeset_contains(treeset_t *tree, treeset_key_t key, void **out_value) +``` +_Searches the treeset for an element with a given key._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be searched for. +- `out_value`: out parameter for the value associated with the key. + + +**Returns:** true operation succeeded, in the `out_value` the value of the found element is returned. + +**Returns:** false operation failed, there is no element with the given key. + + + +## Function `treeset_visit` + +```c +static void treeset_visit(treeset_t *tree, treeset_visitor visitor, void *arg) +``` +_Visits all elements in the treeset._ + + +> **Note:** call only after thread join, or after all threads finished accessing the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `visitor`: address of the function to call on each element. +- `arg`: the third argument to the visitor function. + + + + + +--- diff --git a/doc/api/vsync/map/treeset_rb_fine.h.md b/doc/api/vsync/map/treeset_rb_fine.h.md new file mode 100644 index 0000000..96873b5 --- /dev/null +++ b/doc/api/vsync/map/treeset_rb_fine.h.md @@ -0,0 +1,265 @@ +# [vsync](../README.md) / [map](README.md) / treeset_rb_fine.h +_This implementation of treeset uses balanced red-black tree (RB) and fine-grained locking._ + +**Groups:** [Linearizable](../GROUP_linearizable.md) + +Refer to [treeset_bst_coarse.h](treeset_bst_coarse.h.md) for more general information about treeset. + + +### Example: + + + +```c + +#define TREESET_LOCK_RW +#include + +#include +#include +#include + +#define N 4 +#define MIN_KEY 0 +#define MAX_KEY 3 + +typedef vuintptr_t value_t; + +treeset_t tree; + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)args; + + for (treeset_key_t key = MIN_KEY; key <= MAX_KEY; key++) { + value_t value = tid; + value_t old_value; + + // insert + vbool_t res = + treeset_add(&tree, key, (void *)value, (void *)&old_value); + + if (res) { + // insert succeeded + printf("[%lu] key %lu inserted\n", tid, key); + } else { + printf( + "[%lu] key %lu not inserted, already in tree with value %lu\n", + tid, key, old_value); + } + + // search + res = treeset_contains(&tree, key, (void *)&old_value); + + if (res) { + // search successful + printf("[%lu] key %lu in tree with value %lu\n", tid, key, + old_value); + } else { + printf("[%lu] key %lu not in tree\n", tid, key); + } + + // remove + res = treeset_remove(&tree, key, (void *)&old_value); + + if (res) { + // remove succeeded + printf("[%lu] key %lu removed, old value was %lu\n", tid, key, + old_value); + } else { + printf("[%lu] key %lu not removed\n", tid, key); + } + } + + return NULL; +} + +void * +malloc_cb(vsize_t sz, void *arg) +{ + (void)arg; + return malloc(sz); +} + +void +free_cb(void *ptr, void *arg) +{ + (void)arg; + free(ptr); +} + +void +free_visitor(treeset_key_t key, void *value, void *arg) +{ + (void)key; + (void)arg; + free((value_t *)value); +} + +int +main(void) +{ + pthread_t threads[N]; + + vmem_lib_t mem_lib = {.free_fun = free_cb, + .malloc_fun = malloc_cb, + .arg = NULL}; + + treeset_init(&tree, mem_lib); + + for (vsize_t i = 0; i < N; ++i) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; ++i) { + pthread_join(threads[i], NULL); + } + + treeset_visit(&tree, free_visitor, NULL); + treeset_destroy(&tree); + + return 0; +} +``` + + + +--- +# Functions + +| Function | Description | +|---|---| +| [treeset_init](treeset_rb_fine.h.md#function-treeset_init) | Initializes the treeset. | +| [treeset_destroy](treeset_rb_fine.h.md#function-treeset_destroy) | Destroys all the remaining nodes in the treeset. | +| [treeset_add](treeset_rb_fine.h.md#function-treeset_add) | Attempts to insert an element with a given key and value into the treeset. | +| [treeset_remove](treeset_rb_fine.h.md#function-treeset_remove) | Attempts to remove an element with a given key from the treeset. | +| [treeset_contains](treeset_rb_fine.h.md#function-treeset_contains) | Searches the treeset for an element with a given key. | +| [treeset_visit](treeset_rb_fine.h.md#function-treeset_visit) | Visits all elements in the treeset. | + +## Function `treeset_init` + +```c +static void treeset_init(treeset_t *tree, vmem_lib_t mem_lib) +``` +_Initializes the treeset._ + + +> **Note:** must be called before threads access the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `mem_lib`: object of type `vmem_lib_t` containing malloc/free functions to allocate/free internal nodes. + + + + +## Function `treeset_destroy` + +```c +static void treeset_destroy(treeset_t *tree) +``` +_Destroys all the remaining nodes in the treeset._ + + +> **Note:** call only after thread join, or after all threads finished accessing the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. + + + + +## Function `treeset_add` + +```c +static vbool_t treeset_add(treeset_t *tree, treeset_key_t key, void *value, void **out_value) +``` +_Attempts to insert an element with a given key and value into the treeset._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be inserted. +- `value`: value to be associated with inserted key. +- `out_value`: out parameter for the previous value associated with the key. + + +**Returns:** true operation succeeded. + +**Returns:** false operation failed, since the given key was already in the treeset, in the `out_value` the value of this element is returned. + + + +## Function `treeset_remove` + +```c +static vbool_t treeset_remove(treeset_t *tree, treeset_key_t key, void **out_value) +``` +_Attempts to remove an element with a given key from the treeset._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be removed. +- `out_value`: out parameter for the value associated with the key. + + +**Returns:** true operation succeeded, in the `out_value` the value of the removed element is returned. + +**Returns:** false operation failed, there is no element with the given key. + + + +## Function `treeset_contains` + +```c +static vbool_t treeset_contains(treeset_t *tree, treeset_key_t key, void **out_value) +``` +_Searches the treeset for an element with a given key._ + + + + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `key`: the key to be searched for. +- `out_value`: out parameter for the value associated with the key. + + +**Returns:** true operation succeeded, in the `out_value` the value of the found element is returned. + +**Returns:** false operation failed, there is no element with the given key. + + + +## Function `treeset_visit` + +```c +static void treeset_visit(treeset_t *tree, treeset_visitor visitor, void *arg) +``` +_Visits all elements in the treeset._ + + +> **Note:** call only after thread join, or after all threads finished accessing the treeset. + +**Parameters:** + +- `tree`: address of the treeset_t object. +- `visitor`: address of the function to call on each element. +- `arg`: the third argument to the visitor function. + + + + + +--- diff --git a/doc/api/vsync/spinlock/GROUP_numa_aware.md b/doc/api/vsync/spinlock/GROUP_numa_aware.md new file mode 100644 index 0000000..300c7e6 --- /dev/null +++ b/doc/api/vsync/spinlock/GROUP_numa_aware.md @@ -0,0 +1,15 @@ +# Numa-aware +_Group of numa-aware algorithms._ + +--- +## File Index + + +| File|Description|Numa-aware| +| --- | --- | --- | +| [vsync/spinlock/cnalock.h](cnalock.h.md)|Compact NUMA-aware Lock. | ✔ | +| [vsync/spinlock/hclhlock.h](hclhlock.h.md)|Hierarchical CLH Queue Lock. | ✔ | +| [vsync/spinlock/hmcslock.h](hmcslock.h.md)|Hierarchical MCS lock for systems with NUMA Hierarchies. | ✔ | + + +--- diff --git a/doc/api/vsync/spinlock/README.md b/doc/api/vsync/spinlock/README.md index c0cf99d..ba642fe 100644 --- a/doc/api/vsync/spinlock/README.md +++ b/doc/api/vsync/spinlock/README.md @@ -5,19 +5,27 @@ _Spinlocks for kernel and userspace._ ## File Index -| File|Description| -| --- | --- | -| [vsync/spinlock/caslock.h](caslock.h.md)|Simple spinlock based on compare-and-swap (CAS). | -| [vsync/spinlock/mcslock.h](mcslock.h.md)|Mellor-Crummey Scott Lock - the well-known scalable lock. | -| [vsync/spinlock/rec_mcslock.h](rec_mcslock.h.md)|Recursive MCS lock implementation using recursive.h. | -| [vsync/spinlock/rec_spinlock.h](rec_spinlock.h.md)|Recursive spinlock implementation using recursive.h. | -| [vsync/spinlock/rec_ticketlock.h](rec_ticketlock.h.md)|Recursive ticketlock implementation using recursive.h. | -| [vsync/spinlock/rwlock.h](rwlock.h.md)|Write-preferring rwlock based on semaphore and an atomic vbool_t. | -| [vsync/spinlock/semaphore.h](semaphore.h.md)|Counting Semaphore. | -| [vsync/spinlock/seqcount.h](seqcount.h.md)|Lightweight single-writer multi-reader optimistic lock. | -| [vsync/spinlock/seqlock.h](seqlock.h.md)|Multi-writer multi-reader optimistic lock. | -| [vsync/spinlock/ticketlock.h](ticketlock.h.md)|Classic ticketlock with two 32-bit variables (owner and next). | -| [vsync/spinlock/ttaslock.h](ttaslock.h.md)|Test, Test and Set lock. | +| File|Description|Numa-aware| +| --- | --- | --- | +| [vsync/spinlock/arraylock.h](arraylock.h.md)|Simple array-based queue lock. | ❌ | +| [vsync/spinlock/caslock.h](caslock.h.md)|Simple spinlock based on compare-and-swap (CAS). | ❌ | +| [vsync/spinlock/clhlock.h](clhlock.h.md)|List-based lock attributed to Craig, Landin and Hagersten. | ❌ | +| [vsync/spinlock/cnalock.h](cnalock.h.md)|Compact NUMA-aware Lock. | ✔ | +| [vsync/spinlock/hclhlock.h](hclhlock.h.md)|Hierarchical CLH Queue Lock. | ✔ | +| [vsync/spinlock/hemlock.h](hemlock.h.md)|Hemlock by Dice and Kogan. | ❌ | +| [vsync/spinlock/hmcslock.h](hmcslock.h.md)|Hierarchical MCS lock for systems with NUMA Hierarchies. | ✔ | +| [vsync/spinlock/mcslock.h](mcslock.h.md)|Mellor-Crummey Scott Lock - the well-known scalable lock. | ❌ | +| [vsync/spinlock/rec_mcslock.h](rec_mcslock.h.md)|Recursive MCS lock implementation using recursive.h. | ❌ | +| [vsync/spinlock/rec_seqlock.h](rec_seqlock.h.md)|Recursive seqlock implementation using recursive.h. | ❌ | +| [vsync/spinlock/rec_spinlock.h](rec_spinlock.h.md)|Recursive spinlock implementation using recursive.h. | ❌ | +| [vsync/spinlock/rec_ticketlock.h](rec_ticketlock.h.md)|Recursive ticketlock implementation using recursive.h. | ❌ | +| [vsync/spinlock/rwlock.h](rwlock.h.md)|Write-preferring rwlock based on semaphore and an atomic vbool_t. | ❌ | +| [vsync/spinlock/semaphore.h](semaphore.h.md)|Counting Semaphore. | ❌ | +| [vsync/spinlock/seqcount.h](seqcount.h.md)|Lightweight single-writer multi-reader optimistic lock. | ❌ | +| [vsync/spinlock/seqlock.h](seqlock.h.md)|Multi-writer multi-reader optimistic lock. | ❌ | +| [vsync/spinlock/ticketlock.h](ticketlock.h.md)|Classic ticketlock with two 32-bit variables (owner and next). | ❌ | +| [vsync/spinlock/ttaslock.h](ttaslock.h.md)|Test, Test and Set lock. | ❌ | +| [vsync/spinlock/twalock.h](twalock.h.md)|Ticketlock with waiting array (TWA). | ❌ | --- diff --git a/doc/api/vsync/spinlock/arraylock.h.md b/doc/api/vsync/spinlock/arraylock.h.md new file mode 100644 index 0000000..7000972 --- /dev/null +++ b/doc/api/vsync/spinlock/arraylock.h.md @@ -0,0 +1,141 @@ +# [vsync](../README.md) / [spinlock](README.md) / arraylock.h +_Simple array-based queue lock._ + +The array lock has an array of flags, each slot is associated with a thread. If we have `N` threads then the array has `N` slots. Each slot represents a boolean flag indicating the associated thread's permission to acquire the lock. The thread waits on its flag to become `true` to acquire the lock. The thread releases the lock, by giving away its permission to the next thread, i.e. sets its own flag to `false` and the flag of the one next in line to `true`. + +Initially the first flag is set to `true` and the rest to `false`, and the tail holds the index of the first slot. + + +### Example: + + + +```c + +#include +#include +#include +#include + +#define N 12 +#define EXPECTED_VAL N + +#define NEXT_POWER_2 16U + +arraylock_flag_t g_flags[NEXT_POWER_2]; +arraylock_t g_lock; + +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +void * +run(void *args) +{ + vuint32_t slot = 0; + + arraylock_acquire(&g_lock, &slot); + g_x++; + g_y++; + arraylock_release(&g_lock, slot); + + (void)args; + return NULL; +} + +int +main(void) +{ + arraylock_init(&g_lock, g_flags, NEXT_POWER_2); + + pthread_t threads[N]; + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + ASSERT(g_x == EXPECTED_VAL); + ASSERT(g_x == g_y); + printf("Final value %u\n", g_x); + return 0; +} +``` + + + + +### References: + +Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 7.5.1](https://dl.acm.org/doi/pdf/10.5555/2385452) + +--- +# Functions + +| Function | Description | +|---|---| +| [arraylock_init](arraylock.h.md#function-arraylock_init) | Initializes the given lock object. | +| [arraylock_acquire](arraylock.h.md#function-arraylock_acquire) | Acquires the given lock. | +| [arraylock_release](arraylock.h.md#function-arraylock_release) | Releases the given lock. | + +## Function `arraylock_init` + +```c +static void arraylock_init(arraylock_t *lock, arraylock_flag_t *flags, const vuint32_t len) +``` +_Initializes the given lock object._ + + + + +**Parameters:** + +- `lock`: address of arraylock_t object. +- `flags`: address of arraylock_flag_t object. +- `len`: next power of two greater or equal to the maximum number of threads. Length of `flags`array. + + +> **Note:** `len` MUST BE A POWER OF TWO. This is required because we use a circular array and take advantage of unsigned integers "overflow". + + +## Function `arraylock_acquire` + +```c +static void arraylock_acquire(arraylock_t *lock, vuint32_t *slot) +``` +_Acquires the given lock._ + + + + +**Parameters:** + +- `lock`: address of arraylock_t object. +- `slot`: output parameter. Stores the index of the slot associates with the calling thread. + + +> **Note:** the same value returned in `slot` must be passed intact to `arraylock_release`. + + +## Function `arraylock_release` + +```c +static void arraylock_release(arraylock_t *lock, vuint32_t slot) +``` +_Releases the given lock._ + + + + +**Parameters:** + +- `lock`: address of arraylock_t object. +- `slot`: index of the slot associates with the calling thread. + + + + + +--- diff --git a/doc/api/vsync/spinlock/clhlock.h.md b/doc/api/vsync/spinlock/clhlock.h.md new file mode 100644 index 0000000..66cc535 --- /dev/null +++ b/doc/api/vsync/spinlock/clhlock.h.md @@ -0,0 +1,179 @@ +# [vsync](../README.md) / [spinlock](README.md) / clhlock.h +_List-based lock attributed to Craig, Landin and Hagersten._ + + +### Example: + + + +```c +#include +#include +#include +#include + +#define N 12 +#define MAX_THREADS N +#define EXPECTED_VAL N + +clhlock_t g_lock; +clh_node_t g_nodes[N]; + +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)args; + + clhlock_node_init(&g_nodes[tid]); + + clhlock_acquire(&g_lock, &g_nodes[tid]); + g_x++; + g_y++; + clhlock_release(&g_lock, &g_nodes[tid]); + + (void)args; + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + clhlock_init(&g_lock); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + ASSERT(g_x == EXPECTED_VAL); + ASSERT(g_x == g_y); + printf("Final value %u\n", g_x); + return 0; +} +``` + + + + +### References: + +Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 7.5](https://dl.acm.org/doi/pdf/10.5555/2385452) + +--- +# Functions + +| Function | Description | +|---|---| +| [clhlock_node_init](clhlock.h.md#function-clhlock_node_init) | Initializes the given node. | +| [clhlock_init](clhlock.h.md#function-clhlock_init) | Initializes the given lock. | +| [clhlock_acquire](clhlock.h.md#function-clhlock_acquire) | Acquires the lock. | +| [clhlock_release](clhlock.h.md#function-clhlock_release) | Releases the lock. | +| [clhlock_has_waiters](clhlock.h.md#function-clhlock_has_waiters) | Returns whether there is a thread waiting to acquire the clhlock. | + +## Function `clhlock_node_init` + +```c +static void clhlock_node_init(clh_node_t *node) +``` +_Initializes the given node._ + + + + +**Parameters:** + +- `node`: address of clh_node_t object. + + + + +## Function `clhlock_init` + +```c +static void clhlock_init(clhlock_t *lock) +``` +_Initializes the given lock._ + + + + +**Parameters:** + +- `lock`: address of clhlock_t object. + + + + +## Function `clhlock_acquire` + +```c +static void clhlock_acquire(clhlock_t *lock, clh_node_t *node) +``` +_Acquires the lock._ + + + + +**Parameters:** + +- `lock`: address of clhlock_t object. +- `node`: address of clh_node_t object associated with the calling thread. + + +> **Note:** `node` has to continue to exist even if the thread dies. + + +## Function `clhlock_release` + +```c +static void clhlock_release(clhlock_t *lock, clh_node_t *node) +``` +_Releases the lock._ + + + + +**Parameters:** + +- `lock`: address of clhlock_t object. +- `node`: address of clh_node_t object associated with the calling thread. + + +> **Note:** It hijacks its predecessor's queue node as its own. + + +## Function `clhlock_has_waiters` + +```c +static vbool_t clhlock_has_waiters(clhlock_t *lock, clh_node_t *node) +``` +_Returns whether there is a thread waiting to acquire the clhlock._ + + +This function should only be called by the current owner of the lock. + + + +**Parameters:** + +- `lock`: address of clhlock_t object. +- `node`: address of clh_node_t object associated with the calling thread. + + +**Returns:** true if there are waiters. + +**Returns:** false if there are no waiters. + +> **Note:** this function is not part of the standard lock API. + + + +--- diff --git a/doc/api/vsync/spinlock/cnalock.h.md b/doc/api/vsync/spinlock/cnalock.h.md new file mode 100644 index 0000000..be3e878 --- /dev/null +++ b/doc/api/vsync/spinlock/cnalock.h.md @@ -0,0 +1,165 @@ +# [vsync](../README.md) / [spinlock](README.md) / cnalock.h +_Compact NUMA-aware Lock._ + +**Groups:** [Numa-aware](GROUP_numa_aware.md) + +The CNA is an efficient variant of the MCS locks, which adds NUMA-awareness without a hierarchical approach. + + +### Example: + + + +```c + +#include +#include +#include +#include + +#define N 12U +#define EXPECTED_VAL N + +cnalock_t g_lock = CNALOCK_INIT(); +cna_node_t g_nodes[N] = {0}; + +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +typedef struct eg_args { + vsize_t tid; + vuint32_t numa_node; +} eg_args_t; + +void * +run(void *args) +{ + eg_args_t *t_args = (eg_args_t *)args; + // Bind the thread to run only on the specified numa node, e.g. + // with `sched_setaffinity`. + + // an example critical section. + cnalock_acquire(&g_lock, &g_nodes[t_args->tid], t_args->numa_node); + g_x++; + g_y++; + cnalock_release(&g_lock, &g_nodes[t_args->tid], t_args->numa_node); + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + eg_args_t args[N]; + + for (vsize_t i = 0; i < N; i++) { + args[i].tid = i; + // In a real-world program you would probably use `libnuma` to detect + // how many numa nodes there are, and have a more elaborate strategy + // for binding threads to certain cores / nodes. + args[i].numa_node = i % 2; + pthread_create(&threads[i], NULL, run, (void *)&args[i]); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + ASSERT(g_x == EXPECTED_VAL); + ASSERT(g_x == g_y); + printf("Final value %u\n", g_x); + return 0; +} +``` + + + + +### References: + Dave Dice and Alex Kogan - [Compact NUMA-aware locks](https://dl.acm.org/doi/10.1145/3302424.3303984) + +--- +# Macros + +| Macro | Description | +|---|---| +| [CNALOCK_INIT](cnalock.h.md#macro-cnalock_init) | Initializer of `cnalock_t`. | + +## Macro `CNALOCK_INIT` + +```c +CNALOCK_INIT() +``` + + +_Initializer of_ `cnalock_t`_._ + + + +--- +# Functions + +| Function | Description | +|---|---| +| [cnalock_init](cnalock.h.md#function-cnalock_init) | Initializes the CNA lock. | +| [cnalock_acquire](cnalock.h.md#function-cnalock_acquire) | Acquires the CNA lock. | +| [cnalock_release](cnalock.h.md#function-cnalock_release) | Releases the CNA lock. | + +## Function `cnalock_init` + +```c +static void cnalock_init(cnalock_t *lock) +``` +_Initializes the CNA lock._ + + + + +**Parameters:** + +- `lock`: address of cnalock_t object. + + +> **Note:** alternatively use CNALOCK_INIT + + +## Function `cnalock_acquire` + +```c +static void cnalock_acquire(cnalock_t *lock, cna_node_t *me, vuint32_t numa_node) +``` +_Acquires the CNA lock._ + + + + +**Parameters:** + +- `lock`: address of cnalock_t object. +- `me`: address of cna_node_t object associated with the calling thread. +- `numa_node`: valid id of the NUMA node where the calling thread is hosted + + + + +## Function `cnalock_release` + +```c +static void cnalock_release(cnalock_t *lock, cna_node_t *me, vuint32_t numa_node) +``` +_Releases the CNA lock._ + + + + +**Parameters:** + +- `lock`: address of cnalock_t object. +- `me`: address of cna_node_t object associated with the calling thread. +- `numa_node`: valid id of the NUMA node where the calling thread is hosted. + + + + + +--- diff --git a/doc/api/vsync/spinlock/hclhlock.h.md b/doc/api/vsync/spinlock/hclhlock.h.md new file mode 100644 index 0000000..860f310 --- /dev/null +++ b/doc/api/vsync/spinlock/hclhlock.h.md @@ -0,0 +1,191 @@ +# [vsync](../README.md) / [spinlock](README.md) / hclhlock.h +_Hierarchical CLH Queue Lock._ + +**Groups:** [Numa-aware](GROUP_numa_aware.md) + + +### Example: + + + +```c + +/* you can use the default value or redefine it to indicate how many \ + clusters you need */ +#define HCLH_MAX_CLUSTERS 2 +#include +#include +#include +#include + +#define N 10 +#define IT 10 +#define EXPECTED_VAL (N) * (IT) + +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +hclh_lock_t g_lock; +hclh_qnode_t g_qnode[N] = {0}; +hclh_tnode_t g_tnode[N] = {0}; + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)args; + + /* this should be assigned to either numa group or cache group where the + * thread is running */ + vuint32_t cluster = tid % HCLH_MAX_CLUSTERS; + + hclhlock_init_tnode(&g_tnode[tid], &g_qnode[tid], cluster); + + for (vsize_t i = 0; i < IT; i++) { + hclhlock_acquire(&g_lock, &g_tnode[tid]); + g_x++; + g_y++; + assert(g_x == g_y); + hclhlock_release(&g_tnode[tid]); + } + + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + hclhlock_init(&g_lock); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + printf("Final value %u %u\n", g_x, g_y); + assert(g_x == EXPECTED_VAL); + assert(g_x == g_y); + return 0; +} +``` + + + + +### References: + +Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 7.8.2](https://dl.acm.org/doi/pdf/10.5555/2385452) + +--- +# Macros + +| Macro | Description | +|---|---| +| [HCLH_MAX_CLUSTERS](hclhlock.h.md#macro-hclh_max_clusters) | configures the number of available clusters in HCLH lock. | + +## Macro `HCLH_MAX_CLUSTERS` + + +_configures the number of available clusters in HCLH lock._ + + +Each cluster has a local queue. Each thread must be associated with a cluster value in `[0: HCLH_MAX_CLUSTERS -1]` + +A cluster can be thought of as a NUMA node or cache group. Threads running on cores that belong to the same cache-group or NUMA node are expected to be associated with the same cluster. + +> **Note:** by default 32 clusters are defined. Users can redefine it using: `-DHCLH_MAX_CLUSTERS=N`, where `N` is the number of clusters the user wishes to set. + + +--- +# Functions + +| Function | Description | +|---|---| +| [hclhlock_init](hclhlock.h.md#function-hclhlock_init) | Initializes the lock. | +| [hclhlock_init_tnode](hclhlock.h.md#function-hclhlock_init_tnode) | Initializes the given `tnode`. | +| [hclhlock_acquire](hclhlock.h.md#function-hclhlock_acquire) | Acquires the lock. | +| [hclhlock_release](hclhlock.h.md#function-hclhlock_release) | Releases the lock. | + +## Function `hclhlock_init` + +```c +static void hclhlock_init(hclh_lock_t *hclh_lock) +``` +_Initializes the lock._ + + + + +**Parameters:** + +- `hclh_lock`: address of hclh_lock object to be initialized. + + + + +## Function `hclhlock_init_tnode` + +```c +static void hclhlock_init_tnode(hclh_tnode_t *tnode, hclh_qnode_t *qnode, vuint32_t cluster) +``` +_Initializes the given_ `tnode`_._ + + + + +**Parameters:** + +- `tnode`: address of the hclh_tnode_t object associated with the calling thread. +- `qnode`: address of the hclh_qnode_t object associated with the calling thread. +- `cluster`: the cluster to which the calling thread belongs. + + +> **Note:** Must be called once before the first call of `hclhlock_acquire`. + +> **Note:** `qnode` must live as long as the system is running and not be freed. The reason is that on release qnodes get passed to successor threads. + +> **Note:** `tnode` should live as long as the associated thread lives. + + +## Function `hclhlock_acquire` + +```c +static void hclhlock_acquire(hclh_lock_t *lock, hclh_tnode_t *tnode) +``` +_Acquires the lock._ + + + + +**Parameters:** + +- `lock`: address of the lock to be used. +- `tnode`: address of the hclh_tnode_t object associated with the calling thread. + + +> **Note:** `hclhlock_init_tnode` must be called on the given `tnode`, before `hclhlock_acquire` is called for the first time. + + +## Function `hclhlock_release` + +```c +static void hclhlock_release(hclh_tnode_t *tnode) +``` +_Releases the lock._ + + + + +**Parameters:** + +- `tnode`: address of the hclh_tnode_t object associated with the calling thread. + + + + + +--- diff --git a/doc/api/vsync/spinlock/hemlock.h.md b/doc/api/vsync/spinlock/hemlock.h.md new file mode 100644 index 0000000..e999615 --- /dev/null +++ b/doc/api/vsync/spinlock/hemlock.h.md @@ -0,0 +1,197 @@ +# [vsync](../README.md) / [spinlock](README.md) / hemlock.h +_Hemlock by Dice and Kogan._ + + +### Example: + + + +```c +#include +#include +#include +#include + +#define N 12 +#define MAX_THREADS N +#define EXPECTED_VAL N + +hemlock_t g_lock_a = HEMLOCK_INIT(); +hemlock_t g_lock_b = HEMLOCK_INIT(); + +hem_node_t g_nodes[N]; + +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)args; + // the same thread context object can be used with multiple instances of + // hemlock + hemlock_acquire(&g_lock_a, &g_nodes[tid]); + hemlock_acquire(&g_lock_b, &g_nodes[tid]); + g_x++; + g_y++; + hemlock_release(&g_lock_b, &g_nodes[tid]); + hemlock_release(&g_lock_a, &g_nodes[tid]); + (void)args; + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + // use this function as an alternative to HEMLOCK_INIT + hemlock_init(&g_lock_a); + hemlock_init(&g_lock_b); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + ASSERT(g_x == EXPECTED_VAL); + ASSERT(g_x == g_y); + printf("Final value %u\n", g_x); + return 0; +} +``` + + + + +### References: + +Dice and Kogan - [Hemlock : Compact and Scalable Mutual Exclusion](https://arxiv.org/abs/2102.03863) + +--- +# Macros + +| Macro | Description | +|---|---| +| [HEMLOCK_INIT](hemlock.h.md#macro-hemlock_init) | Initializer of `hemlock_t`. | + +## Macro `HEMLOCK_INIT` + +```c +HEMLOCK_INIT() +``` + + +_Initializer of_ `hemlock_t`_._ + + + +--- +# Functions + +| Function | Description | +|---|---| +| [hemlock_init](hemlock.h.md#function-hemlock_init) | Initializes the given lock object `l`. | +| [hemlock_tryacquire](hemlock.h.md#function-hemlock_tryacquire) | Tries to acquire the Hemlock. | +| [hemlock_acquire](hemlock.h.md#function-hemlock_acquire) | Acquires the Hemlock. | +| [hemlock_release](hemlock.h.md#function-hemlock_release) | Releases the Hemlock. | +| [hemlock_has_waiters](hemlock.h.md#function-hemlock_has_waiters) | Returns whether there is a thread waiting to acquire the Hemlock. | + +## Function `hemlock_init` + +```c +static void hemlock_init(hemlock_t *l) +``` +_Initializes the given lock object_ `l`_._ + + + + +**Parameters:** + +- `l`: address of hemlock_t object. + + + + +## Function `hemlock_tryacquire` + +```c +static int hemlock_tryacquire(hemlock_t *l, hem_node_t *node) +``` +_Tries to acquire the Hemlock._ + + + + +**Parameters:** + +- `l`: address of hemlock_t object. +- `node`: address of hem_node_t object. Associated with the calling thread/core. + + +**Returns:** 1 on success, 0 on failure + + + +## Function `hemlock_acquire` + +```c +static void hemlock_acquire(hemlock_t *l, hem_node_t *node) +``` +_Acquires the Hemlock._ + + + + +**Parameters:** + +- `l`: address of hemlock_t object. +- `node`: address of hem_node_t object. Associated with the calling thread/core. + + + + +## Function `hemlock_release` + +```c +static void hemlock_release(hemlock_t *l, hem_node_t *node) +``` +_Releases the Hemlock._ + + + + +**Parameters:** + +- `l`: address of hemlock_t object. +- `node`: address of hem_node_t object. Associated with the calling thread/core. + + + + +## Function `hemlock_has_waiters` + +```c +static int hemlock_has_waiters(hemlock_t *l, hem_node_t *node) +``` +_Returns whether there is a thread waiting to acquire the Hemlock._ + + +This function should only be called by the current owner of the lock. + + + +**Parameters:** + +- `l`: address of hemlock_t object. +- `node`: address of hem_node_t object. Associated with the calling thread/core. + + + + + +--- diff --git a/doc/api/vsync/spinlock/hmcslock.h.md b/doc/api/vsync/spinlock/hmcslock.h.md new file mode 100644 index 0000000..0e0f287 --- /dev/null +++ b/doc/api/vsync/spinlock/hmcslock.h.md @@ -0,0 +1,224 @@ +# [vsync](../README.md) / [spinlock](README.md) / hmcslock.h +_Hierarchical MCS lock for systems with NUMA Hierarchies._ + +**Groups:** [Numa-aware](GROUP_numa_aware.md) + + +### Example: + + + +```c + +#include +#include +#include +#include + +// To build such a hierarchical tree: // +// R // +// / \ // +// A B // +// / \ / \ // +// C D E F // +// // +// The following specs are needed: // +// level 1, has a root (1 node) // +// level 2, has two nodes connected to R the parent // +// level 3, has two nodes per parent (A/B) // +// // +// number of levels = 3 // +// number of locks = 7 // + +#define NUM_LEVELS 3 + +#define LEVEL_1 1 +#define LEVEL_2 2 +#define LEVEL_3 2 + +#define LEVEL_1_THRESHOLD 1 +#define LEVEL_2_THRESHOLD 1 +#define LEVEL_3_THRESHOLD 1 + +hmcslock_level_spec_t g_level_specs[NUM_LEVELS] = { + {LEVEL_1, LEVEL_1_THRESHOLD}, + {LEVEL_2, LEVEL_2_THRESHOLD}, + {LEVEL_3, LEVEL_3_THRESHOLD}, +}; + +#define NUM_LOCKS \ + ((LEVEL_3 * LEVEL_2 * LEVEL_1) + (LEVEL_1 * LEVEL_2) + (LEVEL_1)) + +#define N 10 +#define IT 10 +#define EXPECTED_VAL (N) * (IT) + +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +hmcslock_t g_locks[NUM_LOCKS]; +hmcs_node_t g_qnode[N] = {0}; + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)args; + + /* the thread should be pinned to the core */ + + /* find the leaf lock associated with this code */ + hmcslock_t *lock = hmcslock_which_lock(g_locks, NUM_LOCKS, g_level_specs, + NUM_LEVELS, LEVEL_3, tid); + + for (vsize_t i = 0; i < IT; i++) { + hmcslock_acquire(lock, &g_qnode[tid], NUM_LEVELS); + g_x++; + g_y++; + assert(g_x == g_y); + hmcslock_release(lock, &g_qnode[tid], NUM_LEVELS); + } + + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + hmcslock_init(g_locks, NUM_LOCKS, g_level_specs, NUM_LEVELS); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + printf("Final value %u %u\n", g_x, g_y); + assert(g_x == EXPECTED_VAL); + assert(g_x == g_y); + return 0; +} +``` + + + + +### References: + +[High Performance Locks for Multi-level NUMA Systems](https://dl.acm.org/doi/pdf/10.1145/2858788.2688503) + + +### References: + +[Verifying and Optimizing the HMCS Lock for Arm Servers](https://link.springer.com/chapter/10.1007/978-3-030-91014-3_17) + +--- +# Functions + +| Function | Description | +|---|---| +| [hmcslock_which_lock](hmcslock.h.md#function-hmcslock_which_lock) | Returns the address of the lock associated with the core id. | +| [hmcslock_init](hmcslock.h.md#function-hmcslock_init) | Initializes the locks array. | +| [hmcslock_acquire](hmcslock.h.md#function-hmcslock_acquire) | Acquires the given lock. | +| [hmcslock_release](hmcslock.h.md#function-hmcslock_release) | Releases the given lock. | + +## Function `hmcslock_which_lock` + +```c +static hmcslock_t* hmcslock_which_lock(hmcslock_t *locks, vsize_t locks_len, hmcslock_level_spec_t *level_specs, vsize_t num_levels, vuint32_t cores_per_node, vuint32_t core_id) +``` +_Returns the address of the lock associated with the core id._ + + + + +**Parameters:** + +- `locks`: array of hmcslock_t objects. +- `locks_len`: `locks` array length. +- `level_specs`: array of hmcslock_level_spec_t objects. +- `num_levels`: number of the levels in heirarchy, same as the length of `level_specs`. +- `cores_per_node`: number of cores per leaf lock node. +- `core_id`: core id of the calling thread. + + +**Returns:** hmcslock_t* address of hmcslock_t object, which the calling thread should use to acquire the lock. + + + +## Function `hmcslock_init` + +```c +static void hmcslock_init(hmcslock_t *locks, vsize_t locks_len, hmcslock_level_spec_t *level_specs, vsize_t num_levels) +``` +_Initializes the locks array._ + + + + +**Parameters:** + +- `locks`: array of hmcslock_t objects. +- `locks_len`: `locks` array length. +- `level_specs`: array of hmcslock_level_spec_t objects. +- `num_levels`: number of the levels in heirarchy, same as the length of `level_specs`. + + +> **Note:** e.g., say you have three levels, the machine, two NUMA nodes, and two caches per NUMA. Then `locks_len = 1(machine) + 2(NUMA) + 4 (2*2 caches) = 7`. `num_levels = 3` (including the machine level) define levels_spec as follows: + +```c +level_specs[num_levels] = { + {1, LEVEL_THRESHOLD}, // 1 machine + {2, LEVEL_THRESHOLD}, // 2 NUMAs per machine + {2, LEVEL_THRESHOLD}, // 2 caches per NUMA +} +``` + + + + +## Function `hmcslock_acquire` + +```c +static void hmcslock_acquire(hmcslock_t *lock, hmcs_node_t *qnode, vsize_t num_levels) +``` +_Acquires the given lock._ + + + + +**Parameters:** + +- `lock`: address of hmcslock_t object. +- `qnode`: address of hmcs_node_t object. +- `num_levels`: number of levels including machine level. + + +> **Note:** `lock` should be what `hmcslock_which_lock` returned. + + +## Function `hmcslock_release` + +```c +static void hmcslock_release(hmcslock_t *lock, hmcs_node_t *qnode, vsize_t num_levels) +``` +_Releases the given lock._ + + + + +**Parameters:** + +- `lock`: address of hmcslock_t object. +- `qnode`: address of hmcs_node_t object. +- `num_levels`: number of levels including machine level. + + +> **Note:** `lock` should be what `hmcslock_which_lock` returned. + + + +--- diff --git a/doc/api/vsync/spinlock/rec_seqlock.h.md b/doc/api/vsync/spinlock/rec_seqlock.h.md new file mode 100644 index 0000000..d000bfd --- /dev/null +++ b/doc/api/vsync/spinlock/rec_seqlock.h.md @@ -0,0 +1,223 @@ +# [vsync](../README.md) / [spinlock](README.md) / rec_seqlock.h +_Recursive seqlock implementation using recursive.h._ + +In a rec_seqlock writers are only blocked by writers, not by readers. Readers optimistically read the shared variables and subsequently check their consistency. If any of the shared variables has been updated while being read, the readers must retry + +This is a reentrant implementation of (see [seqlock.h](seqlock.h.md).) + +### Example: + + + +```c + +#include +#include +#include + +#define N 10 +#define EXPECTED_VAL (N / 2) + +rec_seqlock_t g_lock = REC_SEQLOCK_INIT(); +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +void +writer(vuint32_t tid) +{ + rec_seqlock_acquire(&g_lock, tid); + // writer acquisition of the lock can be nested + rec_seqlock_acquire(&g_lock, tid); + g_x++; + g_y++; + rec_seqlock_release(&g_lock); + rec_seqlock_release(&g_lock); +} + +void +reader(void) +{ + vuint32_t a = 0; + vuint32_t b = 0; + seqvalue_t s = 0; + + do { + s = rec_seqlock_rbegin(&g_lock); + a = g_x; + b = g_y; + } while (!rec_seqlock_rend(&g_lock, s)); + + /* what we read must be consistent */ + ASSERT(a == b); +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)args; + if (tid % 2 == 0) { + reader(); + } else { + writer((vuint32_t)tid); + } + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + ASSERT(g_x == EXPECTED_VAL); + ASSERT(g_x == g_y); + printf("Final value %u\n", g_x); + return 0; +} +``` + + + +--- +# Macros + +| Macro | Description | +|---|---| +| [REC_SEQLOCK_INIT](rec_seqlock.h.md#macro-rec_seqlock_init) | Initializer of `rec_seqlock`. | + +## Macro `REC_SEQLOCK_INIT` + +```c +REC_SEQLOCK_INIT() +``` + + +_Initializer of_ `rec_seqlock`_._ + + + +--- +# Functions + +| Function | Description | +|---|---| +| [rec_seqlock_init](rec_seqlock.h.md#function-rec_seqlock_init) | Initializes the given recursive seqlock. | +| [rec_seqlock_acquire](rec_seqlock.h.md#function-rec_seqlock_acquire) | Acquires the given recursive seqlock. | +| [rec_seqlock_release](rec_seqlock.h.md#function-rec_seqlock_release) | Releases the given recursive seqlock. | +| [rec_seqlock_rbegin](rec_seqlock.h.md#function-rec_seqlock_rbegin) | Marks beginning of reader critical section. | +| [rec_seqlock_rend](rec_seqlock.h.md#function-rec_seqlock_rend) | Ends reader critical section. | + +## Function `rec_seqlock_init` + +```c +static void rec_seqlock_init(rec_seqlock_t *l) +``` +_Initializes the given recursive seqlock._ + + + + +**Parameters:** + +- `l`: address of rec_seqlock_t object. + + + + +## Function `rec_seqlock_acquire` + +```c +static void rec_seqlock_acquire(rec_seqlock_t *l, vuint32_t id) +``` +_Acquires the given recursive seqlock._ + + + + +**Parameters:** + +- `l`: address of rec_seqlock_t object. +- `id`: thread ID or core ID. + + +> **Note:** called by writer threads. + + +## Function `rec_seqlock_release` + +```c +static void rec_seqlock_release(rec_seqlock_t *l) +``` +_Releases the given recursive seqlock._ + + + + +**Parameters:** + +- `l`: address of rec_seqlock_t object. + + +> **Note:** called by writer threads. + + +## Function `rec_seqlock_rbegin` + +```c +static seqvalue_t rec_seqlock_rbegin(rec_seqlock_t *l) +``` +_Marks beginning of reader critical section._ + + +Readers must call this function, before attempting to read. * This function returns a value that must be later passed to `rec_seqlock_rend`. + + +**Postcondition:** readers must call `rec_seqlock_rend` once they are done accessing the critical section. + + +**Parameters:** + +- `l`: address of rec_seqlock_t object. + + +**Returns:** `seqvalue_t` an unsigned integer value. + +> **Note:** called by reader threads. + + +## Function `rec_seqlock_rend` + +```c +static vbool_t rec_seqlock_rend(rec_seqlock_t *l, seqvalue_t sv) +``` +_Ends reader critical section._ + + +Users should rely on the return value to decide if repeating is necessary. + + +**Precondition:** readers must call `rec_seqlock_rbegin` before reading critical section data. + + +**Parameters:** + +- `l`: address of rec_seqlock_t object. +- `sv`: the value `rec_seqlock_rbegin` returned, before the read attempt + + +**Returns:** true read data is consistent. + +**Returns:** false read data is inconsistent, reader must reattempt the read. + +> **Note:** called by reader threads. + + + +--- diff --git a/doc/api/vsync/spinlock/twalock.h.md b/doc/api/vsync/spinlock/twalock.h.md new file mode 100644 index 0000000..ec3c438 --- /dev/null +++ b/doc/api/vsync/spinlock/twalock.h.md @@ -0,0 +1,176 @@ +# [vsync](../README.md) / [spinlock](README.md) / twalock.h +_Ticketlock with waiting array (TWA)._ + +To use the TWA lock, one must declare the global waiting array once in the program. Use `TWALOCK_ARRAY_DECL` to declare the array where convenient. + + +### Example: + + + +```c + +#include +#include +#include +#include + +#define N 12 +#define EXPECTED_VAL N + +TWALOCK_ARRAY_DECL; +twalock_t g_lock = TWALOCK_INIT(); + +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +void * +run(void *args) +{ + twalock_acquire(&g_lock); + g_x++; + g_y++; + twalock_release(&g_lock); + + (void)args; + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + ASSERT(g_x == EXPECTED_VAL); + ASSERT(g_x == g_y); + printf("Final value %u\n", g_x); + return 0; +} +``` + + + + +### References: + +Dice and Kogan - [TWA - Ticket Locks Augmented with a Waiting Array, EuroPar-19] + +--- +# Macros + +| Macro | Description | +|---|---| +| [TWALOCK_ARRAY_DECL](twalock.h.md#macro-twalock_array_decl) | TWALOCK_ARRAY_DECL declares the global __twa_array variable. | +| [TWALOCK_INIT](twalock.h.md#macro-twalock_init) | Initializer of `twalock_t`. | + +## Macro `TWALOCK_ARRAY_DECL` + + +_TWALOCK_ARRAY_DECL declares the global __twa_array variable._ + + + +## Macro `TWALOCK_INIT` + +```c +TWALOCK_INIT() +``` + + +_Initializer of_ `twalock_t`_._ + + + +--- +# Functions + +| Function | Description | +|---|---| +| [twalock_init](twalock.h.md#function-twalock_init) | Initializes the given TWA lock. | +| [twalock_acquire](twalock.h.md#function-twalock_acquire) | Acquires the given TWA lock. | +| [twalock_release](twalock.h.md#function-twalock_release) | Releases the given TWA lock. | +| [twalock_tryacquire](twalock.h.md#function-twalock_tryacquire) | Tries to acquire the given TWA lock. | + +## Function `twalock_init` + +```c +static void twalock_init(twalock_t *l) +``` +_Initializes the given TWA lock._ + + + + +**Parameters:** + +- `l`: address of twalock_t object. + + +> **Note:** alternatively use TWALOCK_INIT + + +## Function `twalock_acquire` + +```c +static void twalock_acquire(twalock_t *l) +``` +_Acquires the given TWA lock._ + + + + +**Parameters:** + +- `l`: address of twalock_t object. + + + + +## Function `twalock_release` + +```c +static void twalock_release(twalock_t *l) +``` +_Releases the given TWA lock._ + + + + +**Parameters:** + +- `l`: address of twalock_t object. + + + + +## Function `twalock_tryacquire` + +```c +static vbool_t twalock_tryacquire(twalock_t *l) +``` +_Tries to acquire the given TWA lock._ + + + + +**Parameters:** + +- `l`: address of twalock_t object. + + +**Returns:** true on success. + +**Returns:** false on failure. + + + + +--- diff --git a/doc/api/vsync/stack/README.md b/doc/api/vsync/stack/README.md index eb690df..04d5db3 100644 --- a/doc/api/vsync/stack/README.md +++ b/doc/api/vsync/stack/README.md @@ -5,9 +5,11 @@ _Concurrent stacks._ ## File Index -| File|Description|Linearizable|Lock-free| -| --- | --- | --- | --- | -| [vsync/stack/quack.h](quack.h.md)|Lockfree concurrent stack/queue (Treiber stack) | ✔ | ✔ | +| File|Description|Linearizable|Lock-free|SMR-required| +| --- | --- | --- | --- | --- | +| [vsync/stack/elimination_stack.h](elimination_stack.h.md)|Unbounded lock-free stack with elimination backoff. | ✔ | ✔ | ✔ | +| [vsync/stack/quack.h](quack.h.md)|Lockfree concurrent stack/queue (Treiber stack) | ✔ | ✔ | ❌ | +| [vsync/stack/xbo_stack.h](xbo_stack.h.md)|Unbounded lock-free stack with exponential backoff. | ✔ | ✔ | ✔ | --- diff --git a/doc/api/vsync/stack/elimination_stack.h.md b/doc/api/vsync/stack/elimination_stack.h.md new file mode 100644 index 0000000..f1f1fdf --- /dev/null +++ b/doc/api/vsync/stack/elimination_stack.h.md @@ -0,0 +1,333 @@ +# [vsync](../README.md) / [stack](README.md) / elimination_stack.h +_Unbounded lock-free stack with elimination backoff._ + +**Groups:** [Linearizable](../GROUP_linearizable.md), [Lock-free](../GROUP_lock_free.md), [SMR-required](../GROUP_requires_smr.md) + +When an attempt to pop/push fails, the calling thread will attempt to rendezvous with another thread. A thread that pushes may exchange the node that is trying to push with a thread that pops. A successful exchange happens when a pusher exchanges its node with a popper within a limited number of trials. If the exchange does not complete within the number of attempts/trials specified in `VSTACK_XCHG_MAX_TRIALS` the normal push/pop operation is reattempted. + +## Configuration + + +- `-DVSTACK_XCHG_SLOTS_COUNT=3` +- `-DVSTACK_XCHG_MAX_TRIALS=10000` + + +Note that the right configuration depends on the contention, number of threads etc. We highly recommend you to benchmark the stack within your application with different configurations and choose the configuration resulting in the best performance. + + +### Example: + + + +```c + +#include +#include +#include +#include +#include +#include +#include + +#define IT 10000 +#define N 24 + +typedef struct data_s { + vstack_node_t stack_node; + smr_node_t smr_node; + vsize_t id; +} data_t; + +gdump_t g_gdump; +vstack_t g_stack; +pthread_rwlock_t g_lock; +vatomic8_t g_stop = VATOMIC_INIT(0); + +static inline void +thread_rw_read_acq(void *arg) +{ + pthread_rwlock_t *lock = (pthread_rwlock_t *)arg; + int ret = pthread_rwlock_rdlock(lock); + assert(ret == 0); +} + +static inline void +thread_rw_read_rel(void *arg) +{ + pthread_rwlock_t *lock = (pthread_rwlock_t *)arg; + int ret = pthread_rwlock_unlock(lock); + assert(ret == 0); +} +static inline void +thread_rw_write_acq(void *arg) +{ + pthread_rwlock_t *lock = (pthread_rwlock_t *)arg; + int ret = pthread_rwlock_wrlock(lock); + assert(ret == 0); +} +static inline void +thread_rw_write_rel(void *arg) +{ + pthread_rwlock_t *lock = (pthread_rwlock_t *)arg; + int ret = pthread_rwlock_unlock(lock); + assert(ret == 0); +} + +smr_rwlock_lib_t g_rwlock_lib = {thread_rw_read_acq, thread_rw_read_rel, + thread_rw_write_acq, thread_rw_write_rel, + &g_lock}; + +void +free_cb(smr_node_t *node, void *args) +{ + data_t *data = V_CONTAINER_OF(node, data_t, smr_node); + free(data); + (void)args; +} + +void +destroy_cb(vstack_node_t *node, void *args) +{ + data_t *data = V_CONTAINER_OF(node, data_t, stack_node); + free(data); + (void)args; +} + +int +usleep_cb(vuint32_t microsecond) +{ + return usleep(microsecond); +} + +static __thread unsigned int g_thread_seed = 0; +vuint32_t +rand_cb(vuint32_t min, vuint32_t max) +{ + if (g_thread_seed == 0) { + g_thread_seed = time(NULL); + } + int r = rand_r(&g_thread_seed); + if (r < 0) { + r *= -1; + } + return (((vuint32_t)r) % (max - min + 1)) + min; +} + +void +reclaim(void) +{ + while (vatomic8_read(&g_stop) == 0) { + vsize_t count = gdump_recycle(&g_gdump, sched_yield, 1); + if (count > 0) { + printf("%zu node(s) were reclaimed\n", count); + } + } +} + +void +consume(vsize_t tid) +{ + gdump_thread_t thread = {0}; + vstack_node_t *node = NULL; + data_t *data = NULL; + + gdump_register(&g_gdump, &thread); + + for (vsize_t i = 0; i < IT; i++) { + gdump_enter(&g_gdump, &thread); + node = vstack_pop(&g_stack); + if (node) { + data = V_CONTAINER_OF(node, data_t, stack_node); + printf("[T%zu] popped %zu\n", tid, data->id); + } + gdump_exit(&g_gdump, &thread); + if (node) { + gdump_retire(&g_gdump, &data->smr_node, free_cb, NULL); + } + } + + gdump_deregister(&g_gdump, &thread); +} + +void +produce(vsize_t tid) +{ + data_t *data = NULL; + for (vsize_t i = 0; i < IT; i++) { + data = malloc(sizeof(data_t)); + if (data) { + data->id = i; + printf("[T%zu] pushing %zu\n", tid, data->id); + vstack_push(&g_stack, &data->stack_node); + } + } +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + if (tid == 0) { + reclaim(); + } else if (tid % 2 == 0) { + produce(tid); + } else { + consume(tid); + } + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + int ret = pthread_rwlock_init(&g_lock, NULL); + assert(ret == 0); + + gdump_init(&g_gdump, g_rwlock_lib); + vstack_init(&g_stack, rand_cb); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 1; i < N; i++) { + pthread_join(threads[i], NULL); + } + // send the signal to stop reclaiming + vatomic8_write(&g_stop, 1); + pthread_join(threads[0], NULL); + + vstack_destroy(&g_stack, destroy_cb, NULL); + gdump_destroy(&g_gdump); + + ret = pthread_rwlock_destroy(&g_lock); + assert(ret == 0); + return 0; +} +``` + + + + +### References: + +Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 11](https://dl.acm.org/doi/pdf/10.5555/2385452) + +--- +# Macros + +| Macro | Description | +|---|---| +| [VSTACK_XCHG_SLOTS_COUNT](elimination_stack.h.md#macro-vstack_xchg_slots_count) | Configures the stack to have the given number of slots. | +| [VSTACK_XCHG_MAX_TRIALS](elimination_stack.h.md#macro-vstack_xchg_max_trials) | Configures the stack to try to exchange for the given number of trials max. | + +## Macro `VSTACK_XCHG_SLOTS_COUNT` + + +_Configures the stack to have the given number of slots._ + + +> **Note:** the default value is `-DVSTACK_XCHG_SLOTS_COUNT=3` + + +## Macro `VSTACK_XCHG_MAX_TRIALS` + + +_Configures the stack to try to exchange for the given number of trials max._ + + +> **Note:** the default value is `-DVSTACK_XCHG_MAX_TRIALS=10000` + + +--- +# Functions + +| Function | Description | +|---|---| +| [vstack_init](elimination_stack.h.md#function-vstack_init) | Initializes the given `stack`. | +| [vstack_push](elimination_stack.h.md#function-vstack_push) | Pushes the given `node` to the top of the given `stack`. | +| [vstack_pop](elimination_stack.h.md#function-vstack_pop) | Pops the top of the given `stack`. | +| [vstack_destroy](elimination_stack.h.md#function-vstack_destroy) | Pops all remaining nodes in the stack and calls `destroy` on them. | + +## Function `vstack_init` + +```c +static void vstack_init(vstack_t *stack, backoff_rand_fun_t rand_fun) +``` +_Initializes the given_ `stack`_._ + + + + +**Parameters:** + +- `stack`: address of vstack_t object. +- `rand_fun`: a function pointer to a function that generates a random number. + + + + +## Function `vstack_push` + +```c +static void vstack_push(vstack_t *stack, vstack_node_t *node) +``` +_Pushes the given_ `node` _to the top of the given_ `stack`_._ + + + + +**Parameters:** + +- `stack`: address of vstack_t object. +- `node`: address of vstack_node_t object. + + +> **Note:** this operation always succeeds. + + +## Function `vstack_pop` + +```c +static vstack_node_t* vstack_pop(vstack_t *stack) +``` +_Pops the top of the given_ `stack`_._ + + + + +**Parameters:** + +- `stack`: address of vstack_t object. + + +**Returns:** address of vstack_node_t object. If the stack is not empty. + +**Returns:** NULL on empty stack. + +> **Note:** must be called inside an SMR critical section. + + +## Function `vstack_destroy` + +```c +static void vstack_destroy(vstack_t *stack, vstack_node_handler_t destroy, void *arg) +``` +_Pops all remaining nodes in the stack and calls_ `destroy` _on them._ + + + + +**Parameters:** + +- `stack`: address of vstack_t object. +- `destroy`: function address of type vstack_node_handler_t. +- `arg`: second argument of `destroy`, can be NULL. + + + + + +--- diff --git a/doc/api/vsync/stack/xbo_stack.h.md b/doc/api/vsync/stack/xbo_stack.h.md new file mode 100644 index 0000000..3b7f790 --- /dev/null +++ b/doc/api/vsync/stack/xbo_stack.h.md @@ -0,0 +1,305 @@ +# [vsync](../README.md) / [stack](README.md) / xbo_stack.h +_Unbounded lock-free stack with exponential backoff._ + +**Groups:** [Linearizable](../GROUP_linearizable.md), [Lock-free](../GROUP_lock_free.md), [SMR-required](../GROUP_requires_smr.md) + +When an attempt to pop/push fails, the calling thread is put to sleep before it reattempts the operation. The sleep duration is determined randomly and it may not exceed the current limit, which doubles on each backoff. The current limit cannot exceed `max_backoff` passed to `vstack_init`. + + +### Example: + + + +```c + +#include +#include +#include +#include +#include +#include +#include + +#define IT 10000 +#define N 12 + +#define VSTACK_MIN_BACKOFF_MICRO_SEC 1 +#define VSTACK_MAX_BACKOFF_MICRO_SEC 1000 + +typedef struct data_s { + vstack_node_t stack_node; + smr_node_t smr_node; + vsize_t id; +} data_t; + +gdump_t g_gdump; +vstack_t g_stack; +pthread_rwlock_t g_lock; +vatomic8_t g_stop = VATOMIC_INIT(0); + +static inline void +thread_rw_read_acq(void *arg) +{ + pthread_rwlock_t *lock = (pthread_rwlock_t *)arg; + int ret = pthread_rwlock_rdlock(lock); + assert(ret == 0); +} + +static inline void +thread_rw_read_rel(void *arg) +{ + pthread_rwlock_t *lock = (pthread_rwlock_t *)arg; + int ret = pthread_rwlock_unlock(lock); + assert(ret == 0); +} +static inline void +thread_rw_write_acq(void *arg) +{ + pthread_rwlock_t *lock = (pthread_rwlock_t *)arg; + int ret = pthread_rwlock_wrlock(lock); + assert(ret == 0); +} +static inline void +thread_rw_write_rel(void *arg) +{ + pthread_rwlock_t *lock = (pthread_rwlock_t *)arg; + int ret = pthread_rwlock_unlock(lock); + assert(ret == 0); +} + +smr_rwlock_lib_t g_rwlock_lib = {thread_rw_read_acq, thread_rw_read_rel, + thread_rw_write_acq, thread_rw_write_rel, + &g_lock}; + +void +free_cb(smr_node_t *node, void *args) +{ + data_t *data = V_CONTAINER_OF(node, data_t, smr_node); + free(data); + (void)args; +} + +void +destroy_cb(vstack_node_t *node, void *args) +{ + data_t *data = V_CONTAINER_OF(node, data_t, stack_node); + free(data); + (void)args; +} + +int +usleep_cb(vuint32_t microsecond) +{ + return usleep(microsecond); +} + +static __thread unsigned int g_thread_seed = 0; +vuint32_t +rand_cb(vuint32_t min, vuint32_t max) +{ + if (g_thread_seed == 0) { + g_thread_seed = time(NULL); + } + int r = rand_r(&g_thread_seed); + if (r < 0) { + r *= -1; + } + return (((vuint32_t)r) % (max - min + 1)) + min; +} + +void +reclaim(void) +{ + while (vatomic8_read(&g_stop) == 0) { + vsize_t count = gdump_recycle(&g_gdump, sched_yield, 1); + if (count > 0) { + printf("%zu node(s) were reclaimed\n", count); + } + } +} + +void +consume(vsize_t tid) +{ + gdump_thread_t thread = {0}; + vstack_node_t *node = NULL; + data_t *data = NULL; + + gdump_register(&g_gdump, &thread); + + for (vsize_t i = 0; i < IT; i++) { + gdump_enter(&g_gdump, &thread); + node = vstack_pop(&g_stack); + if (node) { + data = V_CONTAINER_OF(node, data_t, stack_node); + printf("[T%zu] popped %zu\n", tid, data->id); + } + gdump_exit(&g_gdump, &thread); + if (node) { + gdump_retire(&g_gdump, &data->smr_node, free_cb, NULL); + } + } + + gdump_deregister(&g_gdump, &thread); +} + +void +produce(vsize_t tid) +{ + data_t *data = NULL; + for (vsize_t i = 0; i < IT; i++) { + data = malloc(sizeof(data_t)); + if (data) { + data->id = i; + printf("[T%zu] pushing %zu\n", tid, data->id); + vstack_push(&g_stack, &data->stack_node); + } + } +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + if (tid == 0) { + reclaim(); + } else if (tid % 2 == 0) { + produce(tid); + } else { + consume(tid); + } + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + int ret = pthread_rwlock_init(&g_lock, NULL); + assert(ret == 0); + + gdump_init(&g_gdump, g_rwlock_lib); + vstack_init(&g_stack, usleep_cb, rand_cb, VSTACK_MIN_BACKOFF_MICRO_SEC, + VSTACK_MAX_BACKOFF_MICRO_SEC); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 1; i < N; i++) { + pthread_join(threads[i], NULL); + } + // send the signal to stop reclaiming + vatomic8_write(&g_stop, 1); + pthread_join(threads[0], NULL); + + vstack_destroy(&g_stack, destroy_cb, NULL); + gdump_destroy(&g_gdump); + + ret = pthread_rwlock_destroy(&g_lock); + assert(ret == 0); + return 0; +} +``` + + + + +### References: + +Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 11](https://dl.acm.org/doi/pdf/10.5555/2385452) + +--- +# Functions + +| Function | Description | +|---|---| +| [vstack_init](xbo_stack.h.md#function-vstack_init) | Initializes the given `stack`. | +| [vstack_push](xbo_stack.h.md#function-vstack_push) | Pushes the given `node` to the top of the given `stack`. | +| [vstack_pop](xbo_stack.h.md#function-vstack_pop) | Pops the top of the given `stack`. | +| [vstack_destroy](xbo_stack.h.md#function-vstack_destroy) | Pops all remaining nodes in the stack and calls `destroy` on them. | + +## Function `vstack_init` + +```c +static void vstack_init(vstack_t *stack, backoff_usleep_fun_t vstack_usleep, backoff_rand_fun_t rand_fun, vuint32_t min_backoff, vuint32_t max_backoff) +``` +_Initializes the given_ `stack`_._ + + + + +**Parameters:** + +- `stack`: address of vstack_t object. +- `vstack_usleep`: address of `usleep` like function. +- `rand_fun`: a function pointer to a function that generates a random number. +- `min_backoff`: minimum amount of microseconds a thread is allowed to sleep. +- `max_backoff`: maximum allowed amount of microseconds to sleep. + + + + +## Function `vstack_push` + +```c +static void vstack_push(vstack_t *stack, vstack_node_t *node) +``` +_Pushes the given_ `node` _to the top of the given_ `stack`_._ + + + + +**Parameters:** + +- `stack`: address of vstack_t object. +- `node`: address of vstack_node_t object. + + +> **Note:** this operation always succeeds. + + +## Function `vstack_pop` + +```c +static vstack_node_t* vstack_pop(vstack_t *stack) +``` +_Pops the top of the given_ `stack`_._ + + + + +**Parameters:** + +- `stack`: address of vstack_t object. + + +**Returns:** address of vstack_node_t object. If the stack is not empty. + +**Returns:** NULL on empty stack. + +> **Note:** must be called inside an SMR critical section. + + +## Function `vstack_destroy` + +```c +static void vstack_destroy(vstack_t *stack, vstack_node_handler_t destroy, void *arg) +``` +_Pops all remaining nodes in the stack and calls_ `destroy` _on them._ + + + + +**Parameters:** + +- `stack`: address of vstack_t object. +- `destroy`: function address of type vstack_node_handler_t. +- `arg`: second argument of `destroy`, can be NULL. + + + + + +--- diff --git a/doc/api/vsync/thread/README.md b/doc/api/vsync/thread/README.md new file mode 100644 index 0000000..34c61aa --- /dev/null +++ b/doc/api/vsync/thread/README.md @@ -0,0 +1,23 @@ +# [vsync](../README.md) / thread +_Userspace synchronization primitives._ + +--- +## File Index + + +| File|Description| +| --- | --- | +| [vsync/thread/cond.h](cond.h.md)|Condition variable. | +| [vsync/thread/mutex.h](mutex.h.md)|Futex-based mutex. | +| [vsync/thread/once.h](once.h.md)|One-time initializer. | + +--- +## Directory Index + + +| Directory|Description| +| --- | --- | +| [vsync/thread/mutex](mutex/README.md)|Different implementations of user-space mutex. | + + +--- diff --git a/doc/api/vsync/thread/cond.h.md b/doc/api/vsync/thread/cond.h.md new file mode 100644 index 0000000..bb78859 --- /dev/null +++ b/doc/api/vsync/thread/cond.h.md @@ -0,0 +1,158 @@ +# [vsync](../README.md) / [thread](README.md) / cond.h +_Condition variable._ + +A very simple condition variable. + + +### Example: + + + +```c + +#include +#include +#include +#include +#include + +#define N 12 +#define EXPECTED_VAL N + +vcond_t g_cnd_count_non_zero; +vmutex_t g_mutex_count; +vuint32_t g_count = 0; + +void +dec(void) +{ + for (vsize_t i = 0; i < N - 1; i++) { + vmutex_acquire(&g_mutex_count); + while (g_count == 0) { + vcond_wait(&g_cnd_count_non_zero, &g_mutex_count); + } + g_count--; + vmutex_release(&g_mutex_count); + } +} + +void +inc(void) +{ + vmutex_acquire(&g_mutex_count); + if (g_count == 0) { + vcond_signal(&g_cnd_count_non_zero); + } + g_count++; + vmutex_release(&g_mutex_count); +} + +void * +run(void *args) +{ + vsize_t id = (vsize_t)args; + + if (id == 0) { + dec(); + } else { + inc(); + } + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + vcond_init(&g_cnd_count_non_zero); + vmutex_init(&g_mutex_count); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + ASSERT(g_count == 0); + printf("Final value %u\n", g_count); + return 0; +} +``` + + + +> **Note:** include [mutex.h](mutex.h.md) from libvsync before including [cond.h](cond.h.md). Alternatively, users can implement the same interface with pthread_mutex_t or similar and include that to be used by [cond.h](cond.h.md). + + +### References: + [Condition variable with futex](https://www.remlab.net/op/futex-condvar.shtml) + +--- +# Functions + +| Function | Description | +|---|---| +| [vcond_init](cond.h.md#function-vcond_init) | Initializes the given condition variable. | +| [vcond_wait](cond.h.md#function-vcond_wait) | Waits on the given condition variable. | +| [vcond_signal](cond.h.md#function-vcond_signal) | Signals the condition variable. | + +## Function `vcond_init` + +```c +static void vcond_init(vcond_t *c) +``` +_Initializes the given condition variable._ + + + + +**Parameters:** + +- `c`: address of vcond_t object. + + + + +## Function `vcond_wait` + +```c +static void vcond_wait(vcond_t *c, vmutex_t *m) +``` +_Waits on the given condition variable._ + + +Releases the mutex and waits till the condition variable is signaled, then reacquires the mutex. + + + +**Parameters:** + +- `c`: address of vcond_t object. +- `m`: address of [vmutex_t](structvmutex__t) object. + + + + +## Function `vcond_signal` + +```c +static void vcond_signal(vcond_t *c) +``` +_Signals the condition variable._ + + +Wakes up one sleeping thread waiting on the condition. + + + +**Parameters:** + +- `c`: address of vcond_t object. + + + + + +--- diff --git a/doc/api/vsync/thread/mutex.h.md b/doc/api/vsync/thread/mutex.h.md new file mode 100644 index 0000000..3748b69 --- /dev/null +++ b/doc/api/vsync/thread/mutex.h.md @@ -0,0 +1,62 @@ +# [vsync](../README.md) / [thread](README.md) / mutex.h +_Futex-based mutex._ + +This file includes the default mutex implementation. See [mutex/slim.h](mutex/slim.h.md) for details. + + +### Example: + + + +```c + +#include +#include +#include +#include + +#define N 12 +#define EXPECTED_VAL N + +vmutex_t g_mutex; +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +void * +run(void *args) +{ + vmutex_acquire(&g_mutex); + g_x++; + g_y++; + vmutex_release(&g_mutex); + + (void)args; + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + vmutex_init(&g_mutex); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + ASSERT(g_x == EXPECTED_VAL); + ASSERT(g_x == g_y); + printf("Final value %u\n", g_x); + return 0; +} +``` + + + + +--- diff --git a/doc/api/vsync/thread/mutex/README.md b/doc/api/vsync/thread/mutex/README.md new file mode 100644 index 0000000..14239e0 --- /dev/null +++ b/doc/api/vsync/thread/mutex/README.md @@ -0,0 +1,15 @@ +# [vsync](../../README.md) / [thread](../README.md) / mutex +_Different implementations of user-space mutex._ + +--- +## File Index + + +| File|Description| +| --- | --- | +| [vsync/thread/mutex/musl.h](musl.h.md)|A simplified version of the mutex algorithm in musl libc. | +| [vsync/thread/mutex/slim.h](slim.h.md)|Slim 3-state futex. | +| [vsync/thread/mutex/tristate.h](tristate.h.md)|3-state mutex. | + + +--- diff --git a/doc/api/vsync/thread/mutex/musl.h.md b/doc/api/vsync/thread/mutex/musl.h.md new file mode 100644 index 0000000..8c0e0fe --- /dev/null +++ b/doc/api/vsync/thread/mutex/musl.h.md @@ -0,0 +1,146 @@ +# [vsync](../../README.md) / [thread](../README.md) / [mutex](README.md) / musl.h +_A simplified version of the mutex algorithm in musl libc._ + + +### Example: + + + +```c + +#include +#include +#include +#include + +#define N 12 +#define EXPECTED_VAL N + +vmutex_t g_mutex; +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +void * +run(void *args) +{ + vmutex_acquire(&g_mutex); + g_x++; + g_y++; + vmutex_release(&g_mutex); + + (void)args; + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + vmutex_init(&g_mutex); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + ASSERT(g_x == EXPECTED_VAL); + ASSERT(g_x == g_y); + printf("Final value %u\n", g_x); + return 0; +} +``` + + + +> **Note:** replace `#include <`[`vsync/thread/mutex.h`](../mutex.h.md)`>` with `#include <`[`vsync/thread/mutex/musl.h`](musl.h.md)`>` in the example above. + + +### References: + [Check mutex implementation in libc](http://musl.libc.org) + +> **Note:** It is the normal mutex without support for reentrance. + +--- +# Macros + +| Macro | Description | +|---|---| +| [MUSL_MAX_SPIN](musl.h.md#macro-musl_max_spin) | times of spinning before going to the macro. | + +## Macro `MUSL_MAX_SPIN` + + +_times of spinning before going to the macro._ + + +default value is 100, compile with -DMUSL_MAX_SPIN=N to overwrite the default. + +> **Note:** spinning is deactivated on verification. + + +--- +# Functions + +| Function | Description | +|---|---| +| [vmutex_init](musl.h.md#function-vmutex_init) | Initializes the mutex `m`. | +| [vmutex_acquire](musl.h.md#function-vmutex_acquire) | Acquires the mutex `m`. | +| [vmutex_release](musl.h.md#function-vmutex_release) | Releases the mutex `m`. | + +## Function `vmutex_init` + +```c +static void vmutex_init(vmutex_t *m) +``` +_Initializes the mutex_ `m`_._ + + + + +**Parameters:** + +- `m`: address of [vmutex_t](structvmutex__t) object. + + + + +## Function `vmutex_acquire` + +```c +static void vmutex_acquire(vmutex_t *m) +``` +_Acquires the mutex_ `m`_._ + + + + +**Parameters:** + +- `m`: address of [vmutex_t](structvmutex__t) object. + + + + +## Function `vmutex_release` + +```c +static void vmutex_release(vmutex_t *m) +``` +_Releases the mutex_ `m`_._ + + + + +**Parameters:** + +- `m`: address of [vmutex_t](structvmutex__t) object. + + + + + +--- diff --git a/doc/api/vsync/thread/mutex/slim.h.md b/doc/api/vsync/thread/mutex/slim.h.md new file mode 100644 index 0000000..bbef64e --- /dev/null +++ b/doc/api/vsync/thread/mutex/slim.h.md @@ -0,0 +1,122 @@ +# [vsync](../../README.md) / [thread](../README.md) / [mutex](README.md) / slim.h +_Slim 3-state futex._ + + +### Example: + + + +```c + +#include +#include +#include +#include + +#define N 12 +#define EXPECTED_VAL N + +vmutex_t g_mutex; +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +void * +run(void *args) +{ + vmutex_acquire(&g_mutex); + g_x++; + g_y++; + vmutex_release(&g_mutex); + + (void)args; + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + vmutex_init(&g_mutex); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + ASSERT(g_x == EXPECTED_VAL); + ASSERT(g_x == g_y); + printf("Final value %u\n", g_x); + return 0; +} +``` + + + +> **Note:** replace `#include <`[`vsync/thread/mutex.h`](../mutex.h.md)`>` with `#include <`[`vsync/thread/mutex/slim.h`](slim.h.md)`>` in the example above. + +--- +# Functions + +| Function | Description | +|---|---| +| [vmutex_init](slim.h.md#function-vmutex_init) | Initializes the mutex `m`. | +| [vmutex_acquire](slim.h.md#function-vmutex_acquire) | Acquires the mutex `m`. | +| [vmutex_release](slim.h.md#function-vmutex_release) | Releases the mutex `m`. | + +## Function `vmutex_init` + +```c +static void vmutex_init(vmutex_t *m) +``` +_Initializes the mutex_ `m`_._ + + + + +**Parameters:** + +- `m`: address of [vmutex_t](structvmutex__t) object. + + + + +## Function `vmutex_acquire` + +```c +static void vmutex_acquire(vmutex_t *m) +``` +_Acquires the mutex_ `m`_._ + + + + +**Parameters:** + +- `m`: address of [vmutex_t](structvmutex__t) object. + + + + +## Function `vmutex_release` + +```c +static void vmutex_release(vmutex_t *m) +``` +_Releases the mutex_ `m`_._ + + + + +**Parameters:** + +- `m`: address of [vmutex_t](structvmutex__t) object. + + + + + +--- diff --git a/doc/api/vsync/thread/mutex/tristate.h.md b/doc/api/vsync/thread/mutex/tristate.h.md new file mode 100644 index 0000000..22a08bc --- /dev/null +++ b/doc/api/vsync/thread/mutex/tristate.h.md @@ -0,0 +1,126 @@ +# [vsync](../../README.md) / [thread](../README.md) / [mutex](README.md) / tristate.h +_3-state mutex._ + + +### Example: + + + +```c + +#include +#include +#include +#include + +#define N 12 +#define EXPECTED_VAL N + +vmutex_t g_mutex; +vuint32_t g_x = 0; +vuint32_t g_y = 0; + +void * +run(void *args) +{ + vmutex_acquire(&g_mutex); + g_x++; + g_y++; + vmutex_release(&g_mutex); + + (void)args; + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + vmutex_init(&g_mutex); + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)i); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + + ASSERT(g_x == EXPECTED_VAL); + ASSERT(g_x == g_y); + printf("Final value %u\n", g_x); + return 0; +} +``` + + + +> **Note:** replace `#include <`[`vsync/thread/mutex.h`](../mutex.h.md)`>` with `#include <`[`vsync/thread/mutex/tristate.h`](tristate.h.md)`>` in the example above. + + +### References: + [Ulrich Drepper - Futexes Are Tricky](https://cis.temple.edu/~ingargio/old/cis307s07/readings/futex.pdf) + +--- +# Functions + +| Function | Description | +|---|---| +| [vmutex_init](tristate.h.md#function-vmutex_init) | Initializes the mutex `m`. | +| [vmutex_acquire](tristate.h.md#function-vmutex_acquire) | Acquires the mutex `m`. | +| [vmutex_release](tristate.h.md#function-vmutex_release) | Releases the mutex `m`. | + +## Function `vmutex_init` + +```c +static void vmutex_init(vmutex_t *m) +``` +_Initializes the mutex_ `m`_._ + + + + +**Parameters:** + +- `m`: address of [vmutex_t](structvmutex__t) object. + + + + +## Function `vmutex_acquire` + +```c +static void vmutex_acquire(vmutex_t *m) +``` +_Acquires the mutex_ `m`_._ + + + + +**Parameters:** + +- `m`: address of [vmutex_t](structvmutex__t) object. + + + + +## Function `vmutex_release` + +```c +static void vmutex_release(vmutex_t *m) +``` +_Releases the mutex_ `m`_._ + + + + +**Parameters:** + +- `m`: address of [vmutex_t](structvmutex__t) object. + + + + + +--- diff --git a/doc/api/vsync/thread/once.h.md b/doc/api/vsync/thread/once.h.md new file mode 100644 index 0000000..e849352 --- /dev/null +++ b/doc/api/vsync/thread/once.h.md @@ -0,0 +1,109 @@ +# [vsync](../README.md) / [thread](README.md) / once.h +_One-time initializer._ + +Calls a callback exactly once even if concurrently called. Callback happens before any thread returns [vonce_call()](once.h.md#function-vonce_call). + + +### Example: + + + +```c +#include +#include +#include +#include + +#define N 12 + +vonce_t g_once = VONCE_INIT(); +vuint64_t g_cnt = 0; +vuint64_t g_cnt_ret = 0; +vsize_t g_winner = 0; +vsize_t g_notified_winner = 0; + +void * +callback(void *arg) +{ + vsize_t tid = (vsize_t)(vuintptr_t)arg; + printf("Thread %zu called me\n", tid); + // initialize resources + g_winner = tid; + g_cnt++; + return arg; +} + +void * +run(void *arg) +{ + vsize_t tid = (vsize_t)(vuintptr_t)arg; + + void *ret = vonce_call(&g_once, callback, (void *)(vuintptr_t)tid); + if (ret != NULL) { + ASSERT(((vsize_t)(vuintptr_t)ret) == tid); + printf("Thread %zu is the one that called callback\n", tid); + g_notified_winner = tid; + g_cnt_ret++; + } + return NULL; +} + +int +main(void) +{ + pthread_t threads[N]; + + for (vsize_t i = 0; i < N; i++) { + pthread_create(&threads[i], NULL, run, (void *)(vuintptr_t)(i + 1)); + } + + for (vsize_t i = 0; i < N; i++) { + pthread_join(threads[i], NULL); + } + ASSERT(g_cnt_ret == g_cnt); + ASSERT(g_cnt_ret == 1U); + ASSERT(g_notified_winner == g_winner); + return 0; +} +``` + + + + +### References: + [One-time initializer](https://www.remlab.net/op/futex-misc.shtml) + +--- +# Functions + +| Function | Description | +|---|---| +| [vonce_call](once.h.md#function-vonce_call) | Calls `cb(arg)` once. | + +## Function `vonce_call` + +```c +static void* vonce_call(vonce_t *o, vonce_cb cb, void *arg) +``` +_Calls_ `cb(arg)` _once._ + + +The thread that actually executed the callback gets the return value of the callback. Other threads get NULL back. + + + +**Parameters:** + +- `o`: address of once_t object. +- `cb`: address of callback function. +- `arg`: argument of `cb`. + + +**Returns:** void* return value of `cb(arg)` if the thread actually called `cb`. + +**Returns:** NULL if another thread already called `cb`. + + + + +--- diff --git a/include/test/boilerplate/lock.h b/include/test/boilerplate/lock.h index af580f5..f215a2d 100644 --- a/include/test/boilerplate/lock.h +++ b/include/test/boilerplate/lock.h @@ -146,12 +146,14 @@ main(void) init(); + verification_loop_bound(NTHREADS + 1); for (vuintptr_t i = 0; i < NTHREADS; i++) { (void)pthread_create(&t[i], 0, run, (void *)i); } post(); + verification_loop_bound(NTHREADS + 1); for (vuintptr_t i = 0; i < NTHREADS; i++) { (void)pthread_join(t[i], NULL); } diff --git a/include/test/boilerplate/reader_writer.h b/include/test/boilerplate/reader_writer.h index 737790d..39d67fa 100644 --- a/include/test/boilerplate/reader_writer.h +++ b/include/test/boilerplate/reader_writer.h @@ -40,6 +40,7 @@ #include #include #include +#include #define FINAL_COUNT NWRITERS @@ -162,16 +163,19 @@ main(void) init(); + verification_loop_bound(NWRITERS + 1); for (vuintptr_t i = 0; i < NWRITERS; i++) { (void)pthread_create(&t[i], 0, writer, (void *)i); } + verification_loop_bound(NTHREADS + 1); for (vuintptr_t i = NWRITERS; i < NTHREADS; i++) { (void)pthread_create(&t[i], 0, reader, (void *)i); } post(); + verification_loop_bound(NTHREADS + 1); for (vuintptr_t i = 0; i < NTHREADS; i++) { (void)pthread_join(t[i], NULL); } diff --git a/include/test/map/isimple.h b/include/test/map/isimple.h new file mode 100644 index 0000000..4b849ab --- /dev/null +++ b/include/test/map/isimple.h @@ -0,0 +1,203 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef ISIMPLE_H +#define ISIMPLE_H + +/* Macros */ +#define MAIN_TID NTHREADS +#define NTRACES (NTHREADS + 1U) + +#if defined(VSYNC_VERIFICATION) + #define TRACE_CAPACITY 8U +#else + #define TRACE_CAPACITY 1024U +#endif + +#define VSIMPLE_HT_CAPACITY (TRACE_CAPACITY / 2U) + +/* includes */ +#include +#include + + +/* types */ +typedef struct data_s { + vuintptr_t key; + vuint64_t val; +} data_t; + +/* Globals */ +static trace_t g_add[NTRACES] = {0}; +static trace_t g_rem[NTRACES] = {0}; +static vsimpleht_t g_simpleht = {0}; +static void *g_buff = NULL; + + +/* callbacks */ +static inline vint8_t +cb_cmp(vuintptr_t key_a, vuintptr_t key_b) +{ + if (key_a == key_b) { + return 0; + } else if (key_a < key_b) { + return -1; + } else { + return 1; + } +} + +#if defined(VSYNC_VERIFICATION) +static inline vuint64_t +cb_hash(vuintptr_t key) +{ + return (vuint64_t)key; +} +#else +static inline vuint64_t +cb_hash(vuintptr_t key) +{ + vuint64_t h = key; + h ^= h >> 16U; + h *= 0x85ebca6bU; + h ^= h >> 13U; + h *= 0xc2b2ae35U; + h ^= h >> 16U; + return h; +} +#endif +static inline void +cb_destroy(void *data) +{ + free(data); +} + +static inline void +_imap_verify(void) +{ + vuintptr_t key = 0; + data_t *data = NULL; + vsimpleht_iter_t iter; + + trace_t add_trc; + trace_t rem_trc; + trace_t final_state_trc; + + trace_init(&add_trc, TRACE_CAPACITY); + trace_init(&rem_trc, TRACE_CAPACITY); + + /* merge local traces */ + for (vsize_t i = 0; i < NTRACES; i++) { + trace_merge_into(&add_trc, &g_add[i]); + trace_merge_into(&rem_trc, &g_rem[i]); + } + + /* extract final state */ + trace_init(&final_state_trc, TRACE_CAPACITY); + vsimpleht_iter_init(&g_simpleht, &iter); + while (vsimpleht_iter_next(&iter, &key, (void **)&data)) { + trace_add(&final_state_trc, key); + } + + trace_subtract_from(&add_trc, &rem_trc); + vbool_t eq = trace_is_subtrace(&add_trc, &final_state_trc, NULL); + + trace_destroy(&add_trc); + trace_destroy(&rem_trc); + trace_destroy(&final_state_trc); + ASSERT(eq && "the final state is not what is expected"); +} + +/* interface functions */ +static inline void +imap_init(void) +{ + vsize_t sz = vsimpleht_buff_size(VSIMPLE_HT_CAPACITY); + void *g_buff = malloc(sz); + + vsimpleht_init(&g_simpleht, g_buff, VSIMPLE_HT_CAPACITY, cb_cmp, cb_hash, + cb_destroy); + + for (vsize_t i = 0; i < NTRACES; i++) { + trace_init(&g_add[i], TRACE_CAPACITY); + trace_init(&g_rem[i], TRACE_CAPACITY); + } +} +static inline void +imap_destroy(void) +{ + _imap_verify(); + for (vsize_t i = 0; i < NTRACES; i++) { + trace_destroy(&g_add[i]); + trace_destroy(&g_rem[i]); + } + vsimpleht_destroy(&g_simpleht); + free(g_buff); +} +static inline vbool_t +imap_add(vsize_t tid, vuintptr_t key, vuint64_t val) +{ + data_t *data = malloc(sizeof(data_t)); + data->key = key; + data->val = val; + vbool_t added = + vsimpleht_add(&g_simpleht, data->key, data) == VSIMPLEHT_RET_OK; + if (added) { + trace_add(&g_add[tid], data->key); + } else { + free(data); + } + return added; +} + +static inline vbool_t +imap_rem(vsize_t tid, vuintptr_t key) +{ + vbool_t removed = vsimpleht_remove(&g_simpleht, key) == VSIMPLEHT_RET_OK; + if (removed) { + trace_add(&g_rem[tid], key); + } + return removed; +} + +static inline void * +imap_get(vsize_t tid, vuintptr_t key) +{ + V_UNUSED(tid); + data_t *data = vsimpleht_get(&g_simpleht, key); + if (data) { + ASSERT(data->key == key); + } + return data; +} + +static inline void +imap_reg(vsize_t tid) +{ + V_UNUSED(tid); + vsimpleht_thread_register(&g_simpleht); +} + +static inline void +imap_dereg(vsize_t tid) +{ + V_UNUSED(tid); + vsimpleht_thread_deregister(&g_simpleht); +} + +static inline void +imap_print(void) +{ + vuintptr_t key = 0; + data_t *data = NULL; + vsimpleht_iter_t iter; + vsimpleht_iter_init(&g_simpleht, &iter); + while (vsimpleht_iter_next(&iter, &key, (void **)&data)) { + printf("[%lu:%lu],", key, data->val); + } + printf("\n"); +} + +#endif diff --git a/include/test/map/itreeset.h b/include/test/map/itreeset.h new file mode 100644 index 0000000..32bfa5c --- /dev/null +++ b/include/test/map/itreeset.h @@ -0,0 +1,21 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_ITREESET +#define VSYNC_ITREESET + +#if defined TREESET_BST_FINE + #include +#elif defined TREESET_RB_FINE + #include +#elif defined TREESET_BST_COARSE + #include +#elif defined TREESET_RB_COARSE + #include +#else + #error "Choose treeset implementation by setting TREESET_*" +#endif + +#endif diff --git a/include/test/map/treeset_test_interface.h b/include/test/map/treeset_test_interface.h new file mode 100644 index 0000000..f597b4b --- /dev/null +++ b/include/test/map/treeset_test_interface.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_TEST_INTERFACE +#define VSYNC_TREESET_TEST_INTERFACE + +#include + +treeset_t g_tree; + +static inline vbool_t +tr_add(treeset_key_t key) +{ + vbool_t success = treeset_add(&g_tree, key, NULL, NULL); + return success; +} + +static inline vbool_t +tr_rem(treeset_key_t key) +{ + vbool_t success = treeset_remove(&g_tree, key, NULL); + return success; +} + +static inline vbool_t +tr_con(treeset_key_t key) +{ + vbool_t success = treeset_contains(&g_tree, key, NULL); + return success; +} + +static inline void +tr_init(void) +{ + vmem_lib_t mem_lib = VMEM_LIB_DEFAULT(); + treeset_init(&g_tree, mem_lib); +} + +static inline void +tr_destroy(void) +{ + treeset_destroy(&g_tree); +} + +static inline void +tr_verify(void) +{ + _treeset_verify(&g_tree); +} + +#endif diff --git a/include/test/map/treeset_test_interface_traces.h b/include/test/map/treeset_test_interface_traces.h new file mode 100644 index 0000000..18a292d --- /dev/null +++ b/include/test/map/treeset_test_interface_traces.h @@ -0,0 +1,119 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_TEST_INTERFACE_TRACES +#define VSYNC_TREESET_TEST_INTERFACE_TRACES + +#include +#include + +#define NTRACES NTHREADS +#define DEFAULT_TRACE_LEN 10 + +trace_t g_added[NTRACES]; +trace_t g_removed[NTRACES]; +treeset_t g_tree; +vsize_t collected_count; +vsize_t final_count; + + +static inline vbool_t +tr_add_trace(vsize_t tid, treeset_key_t key) +{ + vbool_t success = treeset_add(&g_tree, key, NULL, NULL); + if (success) { + trace_add(&g_added[tid], key); + } + return success; +} + +static inline vbool_t +tr_rem_trace(vsize_t tid, treeset_key_t key) +{ + vbool_t success = treeset_remove(&g_tree, key, NULL); + if (success) { + trace_add(&g_removed[tid], key); + } + return success; +} + +static inline vbool_t +tr_con_trace(vsize_t tid, treeset_key_t key) +{ + V_UNUSED(tid); + vbool_t success = treeset_contains(&g_tree, key, NULL); + return success; +} + +static inline void +tr_init_trace(void) +{ + vmem_lib_t mem_lib = VMEM_LIB_DEFAULT(); + treeset_init(&g_tree, mem_lib); + for (vsize_t i = 0; i < NTRACES; ++i) { + trace_init(&g_added[i], DEFAULT_TRACE_LEN); + trace_init(&g_removed[i], DEFAULT_TRACE_LEN); + } +} + +static inline void +tr_destroy_trace(void) +{ + treeset_destroy(&g_tree); +} + +static inline void +tr_verify_allocs(void) +{ + vuint64_t allocs = vmem_get_alloc_count(); + vuint64_t frees = vmem_get_free_count(); + + printf("Allocs: %" VUINT64_FORMAT "\n", allocs); + printf("Frees: %" VUINT64_FORMAT "\n", frees); + assert(allocs == frees); +} + +vbool_t +verify_trace_unit(trace_unit_t *unit) +{ + if (unit->count > 0) { + ASSERT(unit->count == 1); + ASSERT(treeset_contains(&g_tree, unit->key, NULL)); + collected_count++; + } + return true; +} + +void +visitor(treeset_key_t key, void *a, void *b) +{ + V_UNUSED(key, a, b); + final_count++; +} + +static inline void +tr_verify_traces(void) +{ + trace_t collected; + trace_init(&collected, DEFAULT_TRACE_LEN); + + for (vsize_t i = 0; i < NTRACES; ++i) { + trace_merge_into(&collected, &g_added[i]); + } + for (vsize_t i = 0; i < NTRACES; ++i) { + trace_subtract_from(&collected, &g_removed[i]); + } + + ASSERT(trace_verify(&collected, verify_trace_unit)); + treeset_visit(&g_tree, visitor, NULL); + + printf("Collected: %zu\n", collected_count); + printf("Final: %zu\n", final_count); + assert(collected_count == final_count); + + trace_destroy(&collected); +} + +#endif diff --git a/include/test/rand.h b/include/test/rand.h index 4c8795e..27789f0 100644 --- a/include/test/rand.h +++ b/include/test/rand.h @@ -301,4 +301,44 @@ random_gen_values(vuint32_t arr[], vsize_t len, vuint32_t min, vuint32_t max) arr[i] = random_next_int(min, max); } } + + +/* Deterministic random generator that allows multiple seeding, + * to be used in testing. + */ + +static inline void +deterministic_random_init(unsigned int seed) +{ + srand(seed); +} + +static inline vuint32_t +deterministic_random(vuint32_t range) +{ + vuint32_t r = 0; + do { + r = rand() % v_pow2_round_up(range); + } while (r >= range); + return r; +} + +static inline vuint32_t +deterministic_random_next_int(vuint32_t min, vuint32_t max) +{ + ASSERT(max >= min); + return deterministic_random(max - min + 1) + min; +} + +static inline void +deterministic_random_shuffle(vsize_t arr[], vsize_t len) +{ + for (vsize_t i = 1; i < len; ++i) { + vsize_t index = deterministic_random_next_int(0, i - 1); + vsize_t temp = arr[i]; + arr[i] = arr[index]; + arr[index] = temp; + } +} + #endif diff --git a/include/test/stack/stack_baseline.h b/include/test/stack/stack_baseline.h new file mode 100644 index 0000000..c458d83 --- /dev/null +++ b/include/test/stack/stack_baseline.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSTACK_BASELINE_H +#define VSTACK_BASELINE_H + +#include +#include +#include + +typedef struct vstack_s { + vstack_core_t core; +} vstack_t; + +/** + * Initializes the given `stack`. + * + * @param stack address of vstack_t object. + * @param vstack_usleep address of `usleep` like function. + */ +static inline void +vstack_init(vstack_t *stack) +{ + ASSERT(stack); + /* init stack core */ + vstack_core_init(&stack->core); +} +/** + * Pushes the given `node` to the top of the given `stack`. + * + * @param stack address of vstack_t object. + * @param node address of vstack_node_t object. + * @note this operation always succeeds. + */ +static inline void +vstack_push(vstack_t *stack, vstack_node_t *node) +{ + ASSERT(stack); + ASSERT(node); + + while (!vstack_core_try_push(&stack->core, node)) + ; +} +/** + * Pops the top of the given `stack`. + * + * @param stack address of vstack_t object. + * @return address of vstack_node_t object. If the stack is not empty. + * @return NULL on empty stack. + * @note must be called inside an SMR critical section. + */ +static inline vstack_node_t * +vstack_pop(vstack_t *stack) +{ + vbool_t success = false; + vstack_node_t *node = NULL; + + ASSERT(stack); + + while (true) { + node = vstack_core_try_pop(&stack->core, &success); + if (success) { + return node; + } + } +} +/** + * Pops all remaining nodes in the stack and calls `destroy` on them. + * + * @param stack address of vstack_t object. + * @param destroy function address of type vstack_node_handler_t. + * @param arg second argument of `destroy`, can be NULL. + */ +static inline void +vstack_destroy(vstack_t *stack, vstack_node_handler_t destroy, void *arg) +{ + vstack_core_destroy(&stack->core, destroy, arg); +} +/** + * Calls the given `visitor` function on each stack node. + * + * @param stack address of vstack_t object. + * @param visitor function address of type vstack_node_handler_t. + * @param arg second argument of `visitor`, can be NULL. + */ +static inline void +_vstack_visit(vstack_t *stack, vstack_node_handler_t visitor, void *arg) +{ + vstack_core_visit(&stack->core, visitor, arg); +} +#endif diff --git a/include/test/stack/stack_interface.h b/include/test/stack/stack_interface.h new file mode 100644 index 0000000..055f609 --- /dev/null +++ b/include/test/stack/stack_interface.h @@ -0,0 +1,303 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#if defined(STACK_ELIMINATION_BACKOFF) + #include + #define VSTACK_NAME "elimination" +#elif defined(STACK_XBO_BACKOFF) + #include + #define VSTACK_NAME "backoff" + #include + #ifndef VSTACK_MIN_BACKOFF_MS + #define VSTACK_MIN_BACKOFF_MS 1 + #endif + + #ifndef VSTACK_MAX_BACKOFF_MS + #define VSTACK_MAX_BACKOFF_MS 1000 + #endif +static inline int +_usleep_callback(unsigned int milliseconds) +{ + #if defined(VSYNC_VERIFICATION) + V_UNUSED(milliseconds); + #else + usleep(milliseconds); + #endif + return 0; +} +#elif defined(STACK_BASELINE) + #include + #define VSTACK_NAME "baseline" +/* does not need SMR */ + #define SMR_NONE +#else + #include +#endif + +#include +#include +#include +#include +#include +#include + +#ifndef NTHREADS + #error "undefined number of threads" +#endif + +#define NTRACES (SMR_MAX_NTHREADS + 1) + +#ifndef N_DS + #define N_DS 1 +#endif + +#if !defined(VSYNC_VERIFICATION) + #include +cleaner_t g_cleaner; +vsize_t g_cleaner_tid = 0; +#endif + + +#define DEFAULT_TRACE_LEN 10 + +static vstack_t g_stack[N_DS]; +static trace_t g_added[N_DS][NTRACES]; +static trace_t g_removed[N_DS][NTRACES]; + +trace_t g_joint_adds; +trace_t g_joint_rems; +trace_t g_final_state; + +typedef struct data_node_s { + vstack_node_t stack_node; /* keep as first field DO NOT move it */ + smr_node_t smr_node; + vuint64_t key; + char lbl; +} data_node_t; + + +static inline void +_free_callback(smr_node_t *snode, void *args) +{ + data_node_t *data = V_CONTAINER_OF(snode, data_node_t, smr_node); + vmem_free(data); + V_UNUSED(args); +} + +static inline void +push(vsize_t tid, vsize_t ds, vuintptr_t id) +{ + data_node_t *in_data = NULL; + in_data = vmem_malloc(sizeof(data_node_t)); + in_data->key = id; + DBG_YELLOW("[T%zu] pushing %lx", tid, id); + vstack_push(&g_stack[ds], &in_data->stack_node); + trace_add(&g_added[ds][tid], id); +} + +static inline vuintptr_t +pop(vsize_t tid, vsize_t ds) +{ +#ifdef STACK_HELPER_INTERFACE + vbool_t kernel_thread = tid < (1 << VSTACK_CORE_ID_WIDTH); + vuint32_t core_id = kernel_thread ? tid : tid - (1 << VSTACK_CORE_ID_WIDTH); + + vstack_node_t *node = vstack_pop(&g_stack[ds], core_id, kernel_thread); +#else + vstack_node_t *node = vstack_pop(&g_stack[ds]); +#endif + if (node) { + data_node_t *out_data = V_CONTAINER_OF(node, data_node_t, stack_node); + ismr_retire(tid, &out_data->smr_node, _free_callback, false); + trace_add(&g_removed[ds][tid], out_data->key); + DBG_GREEN("[T%zu] popped %lx", tid, out_data->key); + return out_data->key; + } + /* zero is mapped to no item */ + DBG_GREEN("[T%zu] popped EMPTY ", tid); + return 0; +} + + +static inline void +init(void) +{ + vsize_t i = 0; + vsize_t j = 0; + + ismr_init(); + + for (i = 0; i < N_DS; i++) { +#if defined(STACK_XBO_BACKOFF) + vstack_init(&g_stack[i], _usleep_callback, random_thread_safe_get_next, + VSTACK_MIN_BACKOFF_MS, VSTACK_MAX_BACKOFF_MS); +#elif defined(STACK_ELIMINATION_BACKOFF) + vstack_init(&g_stack[i], random_thread_safe_get_next); +#else + vstack_init(&g_stack[i]); +#endif + + for (j = 0; j < NTRACES; j++) { + trace_init(&g_added[i][j], DEFAULT_TRACE_LEN); + trace_init(&g_removed[i][j], DEFAULT_TRACE_LEN); + } + } + + trace_init(&g_joint_adds, DEFAULT_TRACE_LEN); + trace_init(&g_joint_rems, DEFAULT_TRACE_LEN); + trace_init(&g_final_state, DEFAULT_TRACE_LEN); + + +#if !defined(VSYNC_VERIFICATION) + vcleaner_start(&g_cleaner, g_cleaner_tid, 0, 0); +#endif +} + +static inline void +destroy_data(vstack_node_t *node, void *arg) +{ + data_node_t *data = V_CONTAINER_OF(node, data_node_t, stack_node); + vmem_free(data); + V_UNUSED(arg); +} + +static inline void +destroy(void) +{ + vsize_t i = 0; + vsize_t j = 0; + + for (i = 0; i < N_DS; i++) { + vstack_destroy(&g_stack[i], destroy_data, NULL); + + for (j = 0; j < NTRACES; j++) { + trace_destroy(&g_added[i][j]); + trace_destroy(&g_removed[i][j]); + } + } + + trace_destroy(&g_joint_adds); + trace_destroy(&g_joint_rems); + trace_destroy(&g_final_state); + +#if !defined(VSYNC_VERIFICATION) + vcleaner_stop(&g_cleaner); +#endif + + ismr_destroy(); + +#if !defined(SMR_NONE) + ASSERT(vmem_no_leak() && "#of allocs != #of frees"); +#endif +} + +static inline void +visit_node(vstack_node_t *snode, void *arg) +{ + trace_t *trace = arg; + + ASSERT(snode); + data_node_t *data = V_CONTAINER_OF(snode, data_node_t, stack_node); + + ASSERT(arg); + trace_add(trace, data->key); +} + + +static inline vbool_t +verify_unit(trace_unit_t *unit, vbool_t expect_existence) +{ + vsize_t idx = 0; + if (unit->count) { + if (unit->count > 1) { + DBG_RED("%lu was added %zu times ", unit->key, unit->count); + } + ASSERT(unit->count == 1); + vbool_t exists = trace_find_unit_idx(&g_final_state, unit->key, &idx); + return exists == expect_existence; + } + // ignore removed units + return true; +} + +static inline vbool_t +verify_unit_does_not_exist(trace_unit_t *unit) +{ + return verify_unit(unit, false); +} + +static inline vbool_t +verify_unit_exists(trace_unit_t *unit) +{ + return verify_unit(unit, true); +} + + +static inline void +verify(vsize_t ds_idx) +{ + vsize_t i = 0; + vbool_t trace_is_sound = false; + + ASSERT(ds_idx < N_DS); + + // extract final state + _vstack_visit(&g_stack[ds_idx], visit_node, &g_final_state); + + + for (i = 0; i < NTRACES; i++) { + trace_merge_into(&g_joint_adds, &g_added[ds_idx][i]); + trace_merge_into(&g_joint_rems, &g_removed[ds_idx][i]); + } + + trace_is_sound = trace_verify(&g_joint_rems, verify_unit_does_not_exist); + ASSERT(trace_is_sound && "remove trace is not sound"); + + + trace_subtract_from(&g_joint_adds, &g_joint_rems); + + trace_is_sound = trace_verify(&g_joint_adds, verify_unit_exists); + if (!trace_is_sound) { + trace_print(&g_joint_adds, "remaining add "); + trace_print(&g_final_state, "final state "); + } + ASSERT(trace_is_sound && "remaining add trace is not sound"); + + + vbool_t equal = trace_is_subtrace(&g_joint_adds, &g_final_state, NULL); + ASSERT(equal && "final state is not part of the added nodes"); + + trace_reset(&g_joint_adds); + trace_reset(&g_joint_rems); + trace_reset(&g_final_state); +} + +void +reg(vsize_t tid) +{ + ismr_reg(tid); +} +void +dereg(vsize_t tid) +{ + ismr_dereg(tid); +} +void +stack_enter(vsize_t tid) +{ + ismr_enter(tid); +} +void +stack_exit(vsize_t tid) +{ + ismr_exit(tid); +} + + +static inline void +stack_clean(vsize_t tid) +{ + ismr_recycle(tid); +} diff --git a/include/verify/rwlock.h b/include/verify/rwlock.h new file mode 100644 index 0000000..aaa8e7a --- /dev/null +++ b/include/verify/rwlock.h @@ -0,0 +1,113 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_RWLOCK_WP_H +#define VSYNC_RWLOCK_WP_H + +/******************************************************************************* + * @file rwlock.h + * @brief rwlock simulation for verification usage + * + ******************************************************************************/ + +#include +#include +#include + +#if !defined(RWLOCK_MAX_READERS) + #define RWLOCK_MAX_READERS 3U +#endif + +typedef struct rwlock_s { + pthread_mutex_t lock[RWLOCK_MAX_READERS]; + vatomic8_t writer_active; + vatomic32_t idx; +} rwlock_t; + + +__thread vuint32_t g_tid = RWLOCK_MAX_READERS; +static inline vuint32_t _rwlock_get_tid(rwlock_t *l); + + +static inline void +rwlock_init(rwlock_t *l) +{ + for (vsize_t i = 0; i < RWLOCK_MAX_READERS; i++) { + pthread_mutex_init(&l->lock[i], NULL); + } +} +/** + * Acquires the write lock. + * + * @param l address of rwlock_t object. + */ +static inline void +rwlock_write_acquire(rwlock_t *l) +{ + vatomic8_write_rlx(&l->writer_active, 1U); + for (vsize_t i = 0; i < RWLOCK_MAX_READERS; i++) { + pthread_mutex_lock(&l->lock[i]); + } +} +/** + * Releases the write lock. + * + * @param l address of rwlock_t object. + */ +static inline void +rwlock_write_release(rwlock_t *l) +{ + vatomic8_write_rlx(&l->writer_active, 0U); + for (vsize_t i = 0; i < RWLOCK_MAX_READERS; i++) { + pthread_mutex_unlock(&l->lock[i]); + } +} +/** + * Acquires the read lock. + * + * @param l address of rwlock_t object. + */ +static inline void +rwlock_read_acquire(rwlock_t *l) +{ + vuint32_t idx = _rwlock_get_tid(l); + pthread_mutex_lock(&l->lock[idx]); +} +/** + * Releases the read lock. + * + * @param l address of rwlock_t object. + */ +static inline void +rwlock_read_release(rwlock_t *l) +{ + vuint32_t idx = _rwlock_get_tid(l); + pthread_mutex_unlock(&l->lock[idx]); +} + +static inline vuint32_t +_rwlock_get_tid(rwlock_t *l) +{ + if (g_tid == RWLOCK_MAX_READERS) { + g_tid = vatomic32_get_inc(&l->idx); + ASSERT(g_tid < RWLOCK_MAX_READERS); + } + return g_tid; +} + +static inline vbool_t +rwlock_acquired_by_writer(rwlock_t *l) +{ + return vatomic8_read_rlx(&l->writer_active) == 1U; +} + +static inline vbool_t +rwlock_acquired_by_readers(rwlock_t *l) +{ + V_UNUSED(l); + return true; +} + +#endif diff --git a/include/vsync/atomic/await.h b/include/vsync/atomic/await.h index d1c0774..5e75b82 100644 --- a/include/vsync/atomic/await.h +++ b/include/vsync/atomic/await.h @@ -43,11 +43,10 @@ * * The following example waits for the pointer me->next to be equal to pred. * Once the condition is met, write NULL in me->next. The variable next contains - * the value that satisfied the condition. The operation has an release - * barrier. + * the value that satisfied the condition. The operation has a release barrier. * - * ``` - * node_t *next = vatomicptr_await_eq_set_acq(me->next, pred, NULL); + * ```c + * node_t *next = vatomicptr_await_eq_set_rel(me->next, pred, NULL); * ``` * * ### Return value diff --git a/include/vsync/bitmap/bitmap.h b/include/vsync/bitmap/bitmap.h new file mode 100644 index 0000000..ce506bd --- /dev/null +++ b/include/vsync/bitmap/bitmap.h @@ -0,0 +1,371 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_VBITMAP_H +#define VSYNC_VBITMAP_H +/******************************************************************************* + * @file bitmap.h + * @brief A bitmap implementation. + * + * Bitmap with basic functionality. + * + * @example + * @include eg_bitmap.c + * + * @cite + * The interface is inspired by and similar to [ck_bitmap] + * (https://github.com/concurrencykit/ck/blob/master/include/ck_bitmap.h) + ******************************************************************************/ + +#include +#include +#include + +typedef struct vbitmap_s { + vsize_t maps_cnt; + vsize_t bit_cnt; + vatomic64_t maps[]; +} vbitmap_t; + +typedef struct vbitmap_iter_s { + vbitmap_t *map; + vsize_t idx; + vuint64_t last_val; +} vbitmap_iter_t; + +#define VBITMAP_CALC_SLOT_IDX(_i_) ((_i_) / ((vsize_t)VUINT64_WIDTH)) +#define VBITMAP_CALC_BIT_SLOT_IDX(_i_) ((_i_) % ((vsize_t)VUINT64_WIDTH)) +#define VBITMAP_BIT_MASK(_i_) (((vuint64_t)1) << VBITMAP_CALC_BIT_SLOT_IDX(_i_)) + +/** + * Returns the mask for setting bits in [idx: 63]. + * + * @param idx LSB index. + * @return vuint64_t mask. + */ +static inline vuint64_t +_vbitmap_lsb_mask(vsize_t idx) +{ + vuint64_t mask = 1; + mask = mask << ((vuint64_t)idx); + mask--; + return ~mask; +} +/** + * Returns the mask for setting bits in [0: idx]. + * + * @param idx MSB index. + * @return vuint64_t mask. + */ +static inline vuint64_t +_vbitmap_msb_mask(vsize_t idx) +{ + vuint64_t mask = 1; + mask = mask << ((vuint64_t)idx); + mask = mask + (mask - 1); + return mask; +} +/** + * Returns mask for setting bits in [from, to]. + * + * @param from LSB index. + * @param to MSB index. + * @return vuint64_t mask. + */ +static inline vuint64_t +_vbitmap_range_mask(vsize_t from, vsize_t to) +{ + vuint64_t lsb = _vbitmap_lsb_mask(from); + vuint64_t msb = _vbitmap_msb_mask(to); + vuint64_t mask = msb & lsb; + return mask; +} +/** + * Calculates the size of vbitmap_t object based on the given number of bits. + * + * @param bit_count number of bits. + * @return vsize_t size of vbitmap_t object needed to accommodate `bit_count` of + * bits. + */ +static inline vsize_t +vbitmap_size(vsize_t bit_count) +{ + vsize_t arr_sz = VCEIL_DIV(bit_count, VUINT64_WIDTH) * sizeof(vatomic64_t); + return (sizeof(vbitmap_t) + arr_sz); +} +/** + * Initializes the given `bitmap` object. + * + * All bits are unset initially. + * + * @param bitmap address of vbitmap_t object. + * @param bit_count number of bits. + * @param set true if all bits should be set, false otherwise. + */ +static inline void +vbitmap_init(vbitmap_t *bitmap, vsize_t bit_count, vbool_t set) +{ + vuint64_t init_val = set ? VUINT64_MAX : 0U; + bitmap->bit_cnt = bit_count; + bitmap->maps_cnt = VCEIL_DIV(bit_count, VUINT64_WIDTH); + for (vsize_t i = 0; i < bitmap->maps_cnt; i++) { + vatomic64_init(&bitmap->maps[i], init_val); + } +} +/** + * Clears all bits in the given bitmap. + * + * @param bitmap address of vbitmap_t object. + */ +static inline void +vbitmap_clear(vbitmap_t *bitmap) +{ + for (vsize_t i = 0; i < bitmap->maps_cnt; i++) { + vatomic64_write(&bitmap->maps[i], 0U); + } +} +/** + * Returns the value of the bit at index `bit_idx`. + * + * @param bitmap address of vbitmap_t object. + * @param bit_idx index of the bit to set. + * @return true the bit is set. + * @return false the bit is unset. + */ +static inline vbool_t +vbitmap_get(vbitmap_t *bitmap, vsize_t bit_idx) +{ + ASSERT(bitmap); + ASSERT(bit_idx < bitmap->bit_cnt); + vsize_t idx = VBITMAP_CALC_SLOT_IDX(bit_idx); + ASSERT(idx < bitmap->maps_cnt); + vuint64_t mask = VBITMAP_BIT_MASK(bit_idx); + vuint64_t val = vatomic64_read(&bitmap->maps[idx]); + vbool_t is_set = (val & mask) == mask; + return is_set; +} +/** + * Sets the bit at index `bit_idx`. + * + * @param bitmap address of vbitmap_t object. + * @param bit_idx index of the bit to set. + */ +static inline void +vbitmap_set_bit(vbitmap_t *bitmap, vsize_t bit_idx) +{ + ASSERT(bitmap); + ASSERT(bit_idx < bitmap->bit_cnt); + vsize_t idx = VBITMAP_CALC_SLOT_IDX(bit_idx); + ASSERT(idx < bitmap->maps_cnt); + vuint64_t mask = VBITMAP_BIT_MASK(bit_idx); + vuint64_t val = vatomic64_or_get(&bitmap->maps[idx], mask); + ASSERT((val & mask) == mask); +} +/** + * Sets bits in slot idx using the given mask. + * + * @param bitmap address of vbitmap_t object. + * @param idx slot index. + * @param mask mask of bits to set. + */ +static inline void +_vbitbmap_set_slot(vbitmap_t *bitmap, vsize_t idx, vuint64_t mask) +{ + ASSERT(idx < bitmap->maps_cnt); + vatomic64_or(&bitmap->maps[idx], mask); +} +/** + * Clears bits in slot idx using the given mask. + * + * @param bitmap address of vbitmap_t object. + * @param idx slot index. + * @param mask mask of bits to clear. + */ +static inline void +_vbitbmap_clr_slot(vbitmap_t *bitmap, vsize_t idx, vuint64_t mask) +{ + ASSERT(idx < bitmap->maps_cnt); + vatomic64_and(&bitmap->maps[idx], mask); +} +/** + * Sets all bits with indexes that are in range `[from:to]`. + * + * @note the range is inclusive both bits at index `from` and index `to` will be + * set. + * + * @param bitmap address of vbitmap_t object. + * @param from index of the LSB in the range. + * @param to index of the MSB in the range. + * + * @note This function is not atomic. Bits inside the range might not be + * raised at once. Raising bits occur in multiple steps. + */ +static inline void +vbitmap_set_range(vbitmap_t *bitmap, vsize_t from, vsize_t to) +{ + vsize_t lsb_slot = 0; + vsize_t msb_slot = 0; + vsize_t lsb_idx = 0; + vsize_t msb_idx = 0; + vuint64_t mask = 0; + + ASSERT(bitmap); + ASSERT(from <= to); + ASSERT(to < bitmap->bit_cnt); + + lsb_slot = VBITMAP_CALC_SLOT_IDX(from); + msb_slot = VBITMAP_CALC_SLOT_IDX(to); + lsb_idx = VBITMAP_CALC_BIT_SLOT_IDX(from); + msb_idx = VBITMAP_CALC_BIT_SLOT_IDX(to); + + ASSERT(lsb_slot <= msb_slot); + ASSERT(msb_slot < bitmap->maps_cnt); + + /* LSB and MSB are within the same slot */ + if (lsb_slot == msb_slot) { + mask = _vbitmap_range_mask(lsb_idx, msb_idx); + _vbitbmap_set_slot(bitmap, lsb_slot, mask); + } else { + mask = _vbitmap_lsb_mask(lsb_idx); + _vbitbmap_set_slot(bitmap, lsb_slot, mask); + for (vsize_t i = lsb_slot + 1; i < msb_slot; i++) { + vatomic64_write(&bitmap->maps[i], VUINT64_MAX); + } + mask = _vbitmap_msb_mask(msb_idx); + _vbitbmap_set_slot(bitmap, msb_slot, mask); + } +} +/** + * Clears all bits with indexes that are in range `[from:to]`. + * + * @note the range is inclusive both bits at index `from` and index `to` will be + * set. + * + * @param bitmap address of vbitmap_t object. + * @param from index of the LSB in the range. + * @param to index of the MSB in the range. + * + * @note This function is not atomic. Bits inside the range might not be + * cleared at once. Clearing bits occur in multiple steps. + */ +static inline void +vbitmap_clr_range(vbitmap_t *bitmap, vsize_t from, vsize_t to) +{ + vsize_t lsb_slot = 0; + vsize_t msb_slot = 0; + vsize_t lsb_idx = 0; + vsize_t msb_idx = 0; + vuint64_t mask = 0; + + ASSERT(bitmap); + ASSERT(from <= to); + ASSERT(to < bitmap->bit_cnt); + + lsb_slot = VBITMAP_CALC_SLOT_IDX(from); + msb_slot = VBITMAP_CALC_SLOT_IDX(to); + lsb_idx = VBITMAP_CALC_BIT_SLOT_IDX(from); + msb_idx = VBITMAP_CALC_BIT_SLOT_IDX(to); + + ASSERT(lsb_slot <= msb_slot); + ASSERT(msb_slot < bitmap->maps_cnt); + + /* LSB and MSB are within the same slot */ + if (lsb_slot == msb_slot) { + mask = ~_vbitmap_range_mask(lsb_idx, msb_idx); + _vbitbmap_clr_slot(bitmap, lsb_slot, mask); + } else { + mask = ~_vbitmap_lsb_mask(lsb_idx); + _vbitbmap_clr_slot(bitmap, lsb_slot, mask); + for (vsize_t i = lsb_slot + 1; i < msb_slot; i++) { + vatomic64_write(&bitmap->maps[i], 0UL); + } + mask = ~_vbitmap_msb_mask(msb_idx); + _vbitbmap_clr_slot(bitmap, msb_slot, mask); + } +} +/** + * Clears the bit at index `bit_idx`. + * + * @param bitmap address of vbitmap_t object. + * @param bit_idx index of the bit to unset. + */ +static inline void +vbitmap_clr_bit(vbitmap_t *bitmap, vsize_t bit_idx) +{ + ASSERT(bitmap); + ASSERT(bit_idx < bitmap->bit_cnt); + vsize_t idx = VBITMAP_CALC_SLOT_IDX(bit_idx); + ASSERT(idx < bitmap->maps_cnt); + vuint64_t mask = VBITMAP_BIT_MASK(bit_idx); + vuint64_t val = vatomic64_and_get(&bitmap->maps[idx], ~mask); + ASSERT((val & mask) == 0UL); +} +/** + * Initializes the given vbitmap_iter_t object. + * + * @param bitmap address of vbitmap_t to iterate over. + * @param iter address of vbitmap_iter_t object. + */ +static inline void +vbitmap_iter_init(vbitmap_t *bitmap, vbitmap_iter_t *iter) +{ + ASSERT(bitmap); + ASSERT(iter); + ASSERT(bitmap->maps_cnt > 0); + + iter->map = bitmap; + iter->idx = 0; + iter->last_val = vatomic64_read(&bitmap->maps[0]); +} +/** + * Iterates to the next set bit. + * + * @param iter address of vbitmap_iter_t object. + * @param out_bit_idx index of the next set bit. + * @return true next set bit is found. + * @return false no more set bits are found. + * + * @pre call `vbitmap_iter_init` to initialize `iter` before first use. + */ +static inline vbool_t +vbitmap_iter_next(vbitmap_iter_t *iter, vsize_t *out_bit_idx) +{ + vatomic64_t *maps = iter->map->maps; + vsize_t maps_cnt = iter->map->maps_cnt; + vsize_t bit_idx = 0; + vsize_t idx = 0; + vsize_t out_idx = 0; + + if (iter->last_val == 0) { + /* search for the next slot where there is a bit set */ + for (idx = iter->idx + 1; idx < maps_cnt; idx++) { + iter->last_val = vatomic64_read(&maps[idx]); + if (iter->last_val) { + iter->idx = idx; + goto VBITMAP_LBL_VAL; + } + } + /* no such slot is found */ + return false; + } + + /* process non zero val, find the set bit */ +VBITMAP_LBL_VAL: + /* find the index of the first set bit */ + bit_idx = (vsize_t)__builtin_ctzll(iter->last_val); + out_idx = (VUINT64_WIDTH * iter->idx) + bit_idx; + if (out_idx >= iter->map->bit_cnt) { + return false; + } + /* calc and save the set bit index */ + *out_bit_idx = out_idx; + /* save a copy of the value without the currently detected bit */ + iter->last_val &= ~VBITMAP_BIT_MASK(bit_idx); + return true; +} +#undef VBITMAP_CALC_SLOT_IDX +#undef VBITMAP_CALC_BIT_SLOT_IDX +#undef VBITMAP_BIT_MASK +#endif diff --git a/include/vsync/map/internal/treeset/treeset_alloc.h b/include/vsync/map/internal/treeset/treeset_alloc.h new file mode 100644 index 0000000..8a38697 --- /dev/null +++ b/include/vsync/map/internal/treeset/treeset_alloc.h @@ -0,0 +1,21 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_ALLOC +#define VSYNC_TREESET_ALLOC + +static inline treeset_node_t * +_treeset_get_node(treeset_t *tree) +{ + return tree->mem_lib.malloc_fun(sizeof(treeset_node_t), tree->mem_lib.arg); +} + +static inline void +_treeset_put_node(treeset_t *tree, treeset_node_t *node) +{ + tree->mem_lib.free_fun(node, tree->mem_lib.arg); +} + +#endif diff --git a/include/vsync/map/internal/treeset/treeset_common.h b/include/vsync/map/internal/treeset/treeset_common.h new file mode 100644 index 0000000..aaa21ac --- /dev/null +++ b/include/vsync/map/internal/treeset/treeset_common.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_COMMON +#define VSYNC_TREESET_COMMON + +#include +#include +#include + +typedef vuintptr_t treeset_key_t; + +struct treeset_node_s; + +typedef void (*treeset_visitor)(treeset_key_t key, void *value, void *arg); + +#endif diff --git a/include/vsync/map/internal/treeset/treeset_lock.h b/include/vsync/map/internal/treeset/treeset_lock.h new file mode 100644 index 0000000..e0d7103 --- /dev/null +++ b/include/vsync/map/internal/treeset/treeset_lock.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_LOCK +#define VSYNC_TREESET_LOCK + +#ifdef VSYNC_VERIFICATION + #define TREESET_LOCK_PTHREAD +#endif + +#if defined TREESET_LOCK_PTHREAD + #include +#elif defined TREESET_LOCK_TTAS + #include +#elif defined TREESET_LOCK_RW + #include +#elif defined TREESET_LOCK_FAKE + #include +#else + #error "Choose lock implementation by setting TREESET_LOCK_*" +#endif + +static inline void l_init(lock_t *lock); + +static inline void l_destroy(lock_t *lock); + +static inline void l_acquire(lock_t *lock); + +static inline void l_release(lock_t *lock); + +static inline void l_reader_acquire(lock_t *lock); + +static inline void l_reader_release(lock_t *lock); + +#ifndef TREESET_RW_LOCK_DEFINED + +static inline void +l_reader_acquire(lock_t *lock) +{ + l_acquire(lock); +} + +static inline void +l_reader_release(lock_t *lock) +{ + l_release(lock); +} + +#endif + +#endif diff --git a/include/vsync/map/internal/treeset/treeset_lock_fake.h b/include/vsync/map/internal/treeset/treeset_lock_fake.h new file mode 100644 index 0000000..0b17183 --- /dev/null +++ b/include/vsync/map/internal/treeset/treeset_lock_fake.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_LOCK_FAKE +#define VSYNC_TREESET_LOCK_FAKE + +typedef struct lock_s { + vbool_t locked; +} lock_t; + +static inline void +l_init(lock_t *lock) +{ + lock->locked = false; +} + +static inline void +l_destroy(lock_t *lock) +{ + ASSERT(!lock->locked); +} + +static inline void +l_acquire(lock_t *lock) +{ + ASSERT(!lock->locked); + lock->locked = true; +} + +static inline void +l_release(lock_t *lock) +{ + ASSERT(lock->locked); + lock->locked = false; +} + +#endif diff --git a/include/vsync/map/internal/treeset/treeset_lock_pthread.h b/include/vsync/map/internal/treeset/treeset_lock_pthread.h new file mode 100644 index 0000000..2139b02 --- /dev/null +++ b/include/vsync/map/internal/treeset/treeset_lock_pthread.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_LOCK_PTHREAD +#define VSYNC_TREESET_LOCK_PTHREAD + +#include + +typedef struct lock_s { + pthread_mutex_t lock; +} lock_t; + +static inline void +l_init(lock_t *lock) +{ + pthread_mutex_init(&lock->lock, NULL); +} + +static inline void +l_destroy(lock_t *lock) +{ + pthread_mutex_destroy(&lock->lock); +} + +static inline void +l_acquire(lock_t *lock) +{ + pthread_mutex_lock(&lock->lock); +} + +static inline void +l_release(lock_t *lock) +{ + pthread_mutex_unlock(&lock->lock); +} + +#endif diff --git a/include/vsync/map/internal/treeset/treeset_lock_rw.h b/include/vsync/map/internal/treeset/treeset_lock_rw.h new file mode 100644 index 0000000..e8f481f --- /dev/null +++ b/include/vsync/map/internal/treeset/treeset_lock_rw.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_LOCK_RW +#define VSYNC_TREESET_LOCK_RW + +#include + +#define TREESET_RW_LOCK_DEFINED + +typedef struct lock_s { + rwlock_t lock; +} lock_t; + +static inline void +l_init(lock_t *lock) +{ + rwlock_init(&lock->lock); +} + +static inline void +l_destroy(lock_t *lock) +{ + (void)lock; +} + +static inline void +l_acquire(lock_t *lock) +{ + rwlock_write_acquire(&lock->lock); +} + +static inline void +l_release(lock_t *lock) +{ + rwlock_write_release(&lock->lock); +} + +static inline void +l_reader_acquire(lock_t *lock) +{ + rwlock_read_acquire(&lock->lock); +} + +static inline void +l_reader_release(lock_t *lock) +{ + rwlock_read_release(&lock->lock); +} + +#endif diff --git a/include/vsync/map/internal/treeset/treeset_lock_ttas.h b/include/vsync/map/internal/treeset/treeset_lock_ttas.h new file mode 100644 index 0000000..8049317 --- /dev/null +++ b/include/vsync/map/internal/treeset/treeset_lock_ttas.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_LOCK_TTAS +#define VSYNC_TREESET_LOCK_TTAS + +#include + +typedef struct lock_s { + ttaslock_t lock; +} lock_t; + +static inline void +l_init(lock_t *lock) +{ + ttaslock_init(&lock->lock); +} + +static inline void +l_destroy(lock_t *lock) +{ + (void)lock; +} + +static inline void +l_acquire(lock_t *lock) +{ + ttaslock_acquire(&lock->lock); +} + +static inline void +l_release(lock_t *lock) +{ + ttaslock_release(&lock->lock); +} + +#endif diff --git a/include/vsync/map/simpleht.h b/include/vsync/map/simpleht.h new file mode 100644 index 0000000..14170c3 --- /dev/null +++ b/include/vsync/map/simpleht.h @@ -0,0 +1,599 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSIMPLE_HT_LF_H +#define VSIMPLE_HT_LF_H + +/****************************************************************************** + * @file simpleht.h + * @brief Simple lock-free hashtable + * @ingroup lock_free linearizable + * + * This is an adaptation of the simple lock-free hashtable mentioned in the + * reference. The adaptation allow for remove operation if users need it. + * Users should be aware that enabling the remove incurs overhead on + * add/get operations. The remove operation can be quite slow as it rebalances + * the hashtable and when that happens lock-freedom is compromised. + * + * This hashtable is ideal for when no remove operations are needed, in that + * case users should disable the support of remove by compiling with + * `-DVSIMPLEHT_DISABLE_REMOVE`. Or keep it, and make sure remove is rarely + * called. + * + * # Operating Conditions + * + * The hashtable is supposed to keep gaps for optimal performance. Users + * should create the table with at least a capacity of 2*N, where N + * is the maximum number of entries to be inserted. + * + * As long as the hashtable is not full, every search is guaranteed to + * finish either by locating the desired key, or by locating an entry whose key + * is zero, which means that the desired key does not exist in the hashtable. + * + * There is a tradeoff between performance and the number of empty slots in the + * hashtable. The performance degrades as the hashtable fills. lock-freedom + * of add is compromised if the hashtable becomes full. + * + * @example + * @include eg_simpleht.c + * + * @cite + * [The World's Simplest Lock-Free Hash Table] + * (https://preshing.com/20130605/the-worlds-simplest-lock-free-hash-table/) + * + *****************************************************************************/ +#include +#include +#include +/** + * @def VSIMPLEHT_DISABLE_REMOVE + * @brief defines VSIMPLEHT_DISABLE_REMOVE to use the simple hashtable without + * entry removal support. This makes the insert and get operations faster + */ +#if !defined(VSIMPLEHT_DISABLE_REMOVE) + #if defined(VSYNC_VERIFICATION) + #include + #else + #include + #endif +#endif + +/** + * @def VSIMPLEHT_RELATIVE_THRESHOLD + * @brief Controls when the rebalancing of the hashtable is triggered. + * Everytime the number of removed nodes is equal to + * capacity/VSIMPLEHT_RELATIVE_THRESHOLD. + * + */ +#if !defined(VSIMPLEHT_RELATIVE_THRESHOLD) + #define VSIMPLEHT_RELATIVE_THRESHOLD 4U +#endif + +typedef vint8_t (*vsimpleht_cmp_key_t)(vuintptr_t key_a, vuintptr_t key_b); +typedef vuint64_t (*vsimpleht_hash_key_t)(vuintptr_t key); +typedef void (*vsimpleht_destroy_entry_t)(void *entry); + +typedef struct vsimpleht_entry_s { + vatomicptr(vuintptr_t) key; + vatomicptr(void *) value; +} vsimpleht_entry_t; + +typedef struct vsimpleht_s { + vsize_t capacity; + vsimpleht_entry_t *entries; + vsimpleht_cmp_key_t cmp_key; + vsimpleht_hash_key_t hash_key; + vsimpleht_destroy_entry_t cb_destroy; +#if !defined(VSIMPLEHT_DISABLE_REMOVE) + vsize_t cleaning_threshold; + vatomicsz_t deleted_count; + rwlock_t lock; +#endif +} vsimpleht_t; + +typedef struct vsimpleht_iter_s { + vsimpleht_t *tbl; + vsize_t idx; +} vsimpleht_iter_t; + +typedef enum vsimpleht_ret_e { + VSIMPLEHT_RET_OK, + VSIMPLEHT_RET_TBL_FULL, + VSIMPLEHT_RET_KEY_EXISTS, + VSIMPLEHT_RET_KEY_DNE, +} vsimpleht_ret_t; +/****************************************************************************** + * Prototypes - internal functions + *****************************************************************************/ +static inline void _vsimpleht_cleanup(vsimpleht_t *tbl, void *val); +static inline void _vsimpleht_trigger_cleanup(vsimpleht_t *tbl, void *val); +static inline void _vsimpleht_give_cleanup_a_chance(vsimpleht_t *tbl); +static inline vsimpleht_ret_t _vsimpleht_add(vsimpleht_t *tbl, vuintptr_t key, + void *value); + +/** + * Calculates the size of the buffer needed by the hashtable. + * + * @param capacity capacity of the hashtable. Maximum number of items to be + * inserted in the hashtable. + * @return vsize_t required buffer size to fit `len` items. + * + * @note capacity must be power of two. + */ +static inline vsize_t +vsimpleht_buff_size(vsize_t capacity) +{ + ASSERT(capacity > 0); + ASSERT((capacity & (capacity - 1)) == 0 && "capacity must be power of 2"); + return sizeof(vsimpleht_entry_t) * capacity; +} +/** + * Initializes the hashtable. + * + * @param tbl address of vsimpleht_t object. + * @param buff address of the hashtable buffer. Allocated by the user. + * @param capacity capacity of the hashtable. Same value passed to + * vsimpleht_buff_size. + * @param cmp_key compare key callback function address. + * @param hash_key address of callback function. Used to hash the key. + * @param destroy_cb address of destroy callback function. Used to destroy the + * hashtable stored objects. Called via `vsimpleht_remove` and + * `vsimpleht_destroy`. + */ +static inline void +vsimpleht_init(vsimpleht_t *tbl, void *buff, vsize_t capacity, + vsimpleht_cmp_key_t cmp_fun, vsimpleht_hash_key_t hash_fun, + vsimpleht_destroy_entry_t destroy_cb) +{ + ASSERT(tbl); + ASSERT(buff); + ASSERT(capacity > 0); + ASSERT((capacity & (capacity - 1)) == 0 && "Array size must be power of 2"); + + tbl->capacity = capacity; + tbl->entries = buff; + tbl->cmp_key = cmp_fun; + tbl->hash_key = hash_fun; + tbl->cb_destroy = destroy_cb; + + for (vsize_t i = 0; i < tbl->capacity; i++) { + vatomicptr_init(&tbl->entries[i].key, NULL); + vatomicptr_init(&tbl->entries[i].value, NULL); + } +#if !defined(VSIMPLEHT_DISABLE_REMOVE) + tbl->cleaning_threshold = (capacity / VSIMPLEHT_RELATIVE_THRESHOLD); + vatomicsz_write_rlx(&tbl->deleted_count, 0); + rwlock_init(&tbl->lock); +#endif +} + +/** + * Destroys the stored hashtable values. + * + * @param tbl address of vsimpleht_t object. + */ +static inline void +vsimpleht_destroy(vsimpleht_t *tbl) +{ + vsimpleht_entry_t *entry = NULL; + void *obj = NULL; + ASSERT(tbl); + for (vsize_t i = 0; i < tbl->capacity; i++) { + entry = &tbl->entries[i]; + obj = vatomicptr_read_rlx(&entry->value); + if (obj) { + tbl->cb_destroy(obj); + } + } +} +/** + * Registers the caller thread. + * + * @param tbl address of vsimpleht_t object. + * + * @note should be called once per thread, before the calling thread accesses + * the hashtable for the first time. + */ +static inline void +vsimpleht_thread_register(vsimpleht_t *tbl) +{ +#if defined(VSIMPLEHT_DISABLE_REMOVE) + V_UNUSED(tbl); +#else + rwlock_read_acquire(&tbl->lock); +#endif +} +/** + * Deregisters the caller thread. + * + * @param tbl address of vsimpleht_t object. + * + * @note should be called once per thread, after the calling thread accesses + * the hashtable for the last time. + */ +static inline void +vsimpleht_thread_deregister(vsimpleht_t *tbl) +{ +#if defined(VSIMPLEHT_DISABLE_REMOVE) + V_UNUSED(tbl); +#else + rwlock_read_release(&tbl->lock); +#endif +} +/** + * Inserts the given value into the hashtable. + * + * @param tbl address of vsimpleht_t object. + * @param key key of the value to add. + * @param value address of the object to insert. + * @return VSIMPLEHT_RET_OK key does not exist, value was added. + * @return VSIMPLEHT_RET_KEY_EXISTS key exists, value was not added. + * @return VSIMPLEHT_RET_TBL_FULL table is full. Consider + * - increasing the capacity + * - decreasing VSIMPLEHT_RELATIVE_THRESHOLD + * - removing items from the table + * + * @note neither key can be 0 nor value can be NULL. + */ +static inline vsimpleht_ret_t +vsimpleht_add(vsimpleht_t *tbl, vuintptr_t key, void *value) +{ + ASSERT(key != 0); + ASSERT(value != NULL); + _vsimpleht_give_cleanup_a_chance(tbl); + return _vsimpleht_add(tbl, key, value); +} +/** + * Searches the hashtable for a value associated with the given key. + * + * @param tbl address of vsimpleht_t object. + * @param key key to search for. + * @return void* address of the object associated with the given key if exists. + * @return NULL if there is no value found associated with given key. + */ +static inline void * +vsimpleht_get(vsimpleht_t *tbl, vuintptr_t key) +{ + vsize_t index = 0; + vuintptr_t probed_key = 0; + _vsimpleht_give_cleanup_a_chance(tbl); + for (index = tbl->hash_key(key);; index++) { + index &= tbl->capacity - 1; + ASSERT(index < tbl->capacity); + probed_key = (vuintptr_t)vatomicptr_read(&tbl->entries[index].key); + if (probed_key == 0) { + return NULL; + } else if (tbl->cmp_key(key, probed_key) == 0) { + return vatomicptr_read_acq(&tbl->entries[index].value); + } + } +} +/** + * Initializes the given iterator object `iter` for operating `tbl` object. + * + * @param tbl address of vsimpleht_t object. + * @param iter address of vsimpleht_iter_t object to initialize. + */ +static inline void +vsimpleht_iter_init(vsimpleht_t *tbl, vsimpleht_iter_t *iter) +{ + ASSERT(tbl); + ASSERT(iter); + iter->tbl = tbl; + iter->idx = 0; +} +/** + * Reads the current entry (key/value) the iterator has reached. + * + * After the values are read, the iterator advances to the next entry. + * + * @param iter address of vsimpleht_iter_t object. + * @param key output parameter where the current key value is stored. + * @param val output parameter where the current object value is stored. + * @return true the current output of key and val is valid. + * @return false there are no more objects to iterate. Stored values in key, val + * parameters should be ignored by the user. + * + * @pre iter must be initalized via vsimpleht_iter_init. + * + * @note the iterator is mainly meant for sequential setting. When called in + * parallel the results might not be consistent. + * + * @example + * ```c + * vuintptr_t key = 0; + * data_t *date = NULL; + * vsimpleht_iter_t iter; + * vsimpleht_iter_init(&g_simpleht, &iter); + * while (vsimpleht_iter_next(&iter, &key, (void **)&data)) { + * // access key + * // access data + * } + * ``` + */ +static inline vbool_t +vsimpleht_iter_next(vsimpleht_iter_t *iter, vuintptr_t *key, void **val) +{ + vuintptr_t k = 0; + void *v = NULL; + vsimpleht_entry_t *entries = NULL; + ASSERT(iter); + ASSERT(iter->tbl); + ASSERT(key); + ASSERT(val); + entries = iter->tbl->entries; + ASSERT(entries); + for (vsize_t i = iter->idx; i < iter->tbl->capacity; i++) { + k = (vuintptr_t)vatomicptr_read(&entries[i].key); + v = vatomicptr_read(&entries[i].value); + if (k && v) { + iter->idx = i + 1; + *key = k; + *val = v; + return true; + } + } + return false; +} +/** + * Removes the node associated with the given key from the hashtable. + * + * @param tbl address of vsimpleht_t object. + * @param key key to remove the value associated with. + * @return VSIMPLEHT_RET_OK key exists, node was removed. + * @return VSIMPLEHT_RET_KEY_DNE key does not exist. + * + * @note this operation is not lock-free, if the the removal is successful. It + * can be super-slow, and the associated value can be freed via the destroy + * callback registered at `vsimpleht_init`. + * + */ +static inline vsimpleht_ret_t +vsimpleht_remove(vsimpleht_t *tbl, vuintptr_t key) +{ +#if defined(VSIMPLEHT_DISABLE_REMOVE) + ASSERT(0 && + "vsimpleht_remove has no effect when VSIMPLEHT_DISABLE_REMOVE is " + "defined."); + V_UNUSED(tbl, key); + return VSIMPLEHT_RET_KEY_DNE; +#else + vsize_t index = 0; + vuintptr_t probed_key = 0; + void *probed_value = NULL; + vsize_t start_index = tbl->hash_key(key); + index = start_index & (tbl->capacity - 1); + _vsimpleht_give_cleanup_a_chance(tbl); + do { + probed_key = (vuintptr_t)vatomicptr_read(&tbl->entries[index].key); + if (probed_key == 0) { + // if we reached the end of the bucket + // then the key does not exist. + return VSIMPLEHT_RET_KEY_DNE; + } else if (tbl->cmp_key(key, probed_key) == 0) { + probed_value = vatomicptr_read_acq(&tbl->entries[index].value); + // we found the key + // we check if the value exists + if (probed_value == NULL) { + // the item was there but is logically removed + return VSIMPLEHT_RET_KEY_DNE; + } + /* logically remove the entry by marking the value as NULL */ + if (vatomicptr_cmpxchg_rel(&tbl->entries[index].value, probed_value, + NULL) == probed_value) { + vatomicsz_inc_rlx(&tbl->deleted_count); + /* logical removal is successful, we trigger actual cleanup */ + _vsimpleht_trigger_cleanup(tbl, probed_value); + return VSIMPLEHT_RET_OK; + } + return VSIMPLEHT_RET_KEY_DNE; + } + + index++; + index &= tbl->capacity - 1; + } while (index != start_index); + // if we circled around once the key + // does not exist + return VSIMPLEHT_RET_KEY_DNE; +#endif +} +/** + * Inserts the given value into the hashtable. + * + * @param tbl address of vsimpleht_t object. + * @param key key of the value to add. + * @param value address of the object to insert. + * @return VSIMPLEHT_RET_OK key does not exist, value was added. + * @return VSIMPLEHT_RET_KEY_EXISTS key exists, value was not added. + * @return VSIMPLEHT_RET_TBL_FULL table is full. + * + * @note neither key can be 0 nor value can be NULL. + * @note this API is internal and should not be called by users. + */ +static inline vsimpleht_ret_t +_vsimpleht_add(vsimpleht_t *tbl, vuintptr_t key, void *value) +{ + vsize_t index = 0; + vuintptr_t probed_key = 0; + void *val = NULL; + vsize_t cnt = 0; + + ASSERT(key && "NULL key is not allowed!"); + ASSERT(value && "NULL value is not allowed!"); + // linear search at an index determined by hashing the key. + // scan the array and store the item in the first entry whose + // existing key is either 0, or matches the desired key + for (index = tbl->hash_key(key); cnt < tbl->capacity; cnt++, index++) { + // keep the index within the array boundaries + index &= tbl->capacity - 1; + ASSERT(index < tbl->capacity); + // Load the key that was there. + probed_key = (vuintptr_t)vatomicptr_read(&tbl->entries[index].key); + /* + There is no key in this slot, so we try + to occupy it. + */ + if (probed_key == 0) { + probed_key = (vuintptr_t)vatomicptr_cmpxchg( + &tbl->entries[index].key, NULL, (void *)key); + /* someone stole it, and inserted a different key */ + if (probed_key != 0 && tbl->cmp_key(key, probed_key) != 0) { + continue; + } + } else if (tbl->cmp_key(key, probed_key) != 0) { + /* There exists a key at this slot, but it is not equal + to the one we want to insert, so we move on */ + continue; + } + ASSERT(tbl->cmp_key(key, (vuintptr_t)vatomicptr_read( + &tbl->entries[index].key)) == 0); + /* + we reach here under the following circumstances + 1. There is a key in this slot that is equal to ours. + 2. There was no key and we managed to insert ours. + + so we try to insert the value: + 1. there is already a value we fail + 2. there is no value, entry has either been deleted, so + we just reuse the slot. or someone with the same key is + in the process of inserting its value in that case she fails + and we succeed. + We use cas with expected old value NULL + */ + val = vatomicptr_cmpxchg(&tbl->entries[index].value, NULL, value); + return (val == NULL) ? VSIMPLEHT_RET_OK : VSIMPLEHT_RET_KEY_EXISTS; + } + return VSIMPLEHT_RET_TBL_FULL; +} +/** + * Releases the reader lock briefly if there is a writer waiting to acquire it. + * + * @note this gives the cleaner thread a chance to rebalance the table and free + * the removed node. + * + * @param tbl address of vsimpleht_t object. + */ +static inline void +_vsimpleht_give_cleanup_a_chance(vsimpleht_t *tbl) +{ +#if defined(VSIMPLEHT_DISABLE_REMOVE) + V_UNUSED(tbl); +#else + if (rwlock_acquired_by_writer(&tbl->lock)) { + ASSERT(rwlock_acquired_by_readers(&tbl->lock) && + "You seem to have forgotten to call the " + " thread register function"); + rwlock_read_release(&tbl->lock); + rwlock_read_acquire(&tbl->lock); + } +#endif +} +/** + * Releases the reader-lock, triggers cleaning if needed and then reacquires the + * lock. + * + * @param tbl address of vsimpleht_t object. + * @param val address of the removed object. + */ +static inline void +_vsimpleht_trigger_cleanup(vsimpleht_t *tbl, void *val) +{ +#if defined(VSIMPLEHT_DISABLE_REMOVE) + V_UNUSED(tbl, val); +#else + ASSERT(rwlock_acquired_by_readers(&tbl->lock) && + "You seem to have forgotten to call the " + " thread register function"); + rwlock_read_release(&tbl->lock); + _vsimpleht_cleanup(tbl, val); + rwlock_read_acquire(&tbl->lock); +#endif +} +#if defined(VSIMPLEHT_DISABLE_REMOVE) +static inline void +_vsimpleht_cleanup(vsimpleht_t *tbl, void *val) +{ + V_UNUSED(tbl, val); +} +#else +/** + * Destructs val, and rebalances the hashtable buckets if the threshold is met. + * + * @param tbl address of vsimpleht_t object. + * @param val address of the removed object. + * + * @note this is a blocking operation, acquires the lock in write-mode. + */ +static inline void +_vsimpleht_cleanup(vsimpleht_t *tbl, void *val) +{ + vsize_t e = 0; + vsize_t i = 0; + vuintptr_t key = 0; + void *value = NULL; + vsize_t len = tbl->capacity; + vsimpleht_entry_t *entries = tbl->entries; + vsize_t start_index = len - 1; + uint8_t ret; + + /* we have to go through the lock now */ + rwlock_write_acquire(&tbl->lock); + /* double check if we got lucky in the meantime, someone else did the + * job for us */ + if (vatomicsz_read_rlx(&tbl->deleted_count) < tbl->cleaning_threshold) { + goto CLEANUP_EXIT; + } + /* avoid bug in the next loop in which we first pass an element that + * needs to be resettled, but due to wrap-around of the bucket the + * position into which it should be resettled (logically deleted) has + * not yet been seen by the loop and freed. + * To avoid this we make sure we start the next loop at the beginning of + * a bucket not in the middle. + * Find a beginning of a bucket. + */ + for (e = 0; e < len; e++) { + key = (vuintptr_t)vatomicptr_read_rlx(&entries[e].key); + value = vatomicptr_read_rlx(&entries[e].value); + if (key == 0) { + start_index = e; + break; + } + } + /* scan the whole array starting from the start_index (index of first + * bucket found above), delete logically removed items, and resettle + * existing items. Resettling is important, to avoid having a gap within + * a bucket. The end of the bucket is recognized as a slot with an empty + * key. If a bucket has a gap, we will mistake elements of the bucket + * as not existing when they are placed after the gap. + */ + for (e = 0; e < len; e++) { + i = (e + start_index) & (len - 1); + key = (vuintptr_t)vatomicptr_read_rlx(&entries[i].key); + value = vatomicptr_read_rlx(&entries[i].value); + if (key != 0 && value != NULL) { + /* the addition may fail if the resettled position is equal to + * the current position. In that case, we just leave the entry + * intact. */ + ret = _vsimpleht_add(tbl, key, value); + if (ret == VSIMPLEHT_RET_OK) { + vatomicptr_write_rlx(&entries[i].key, NULL); + vatomicptr_write_rlx(&entries[i].value, NULL); + } + ASSERT(ret != VSIMPLEHT_RET_TBL_FULL && + "since we are inserting what is already in the table, " + "this should never happen"); + } else if (key != 0 && value == NULL) { + vatomicptr_write_rlx(&entries[i].key, NULL); + } + } + vatomicsz_write_rlx(&tbl->deleted_count, 0); + +CLEANUP_EXIT: + tbl->cb_destroy(val); + rwlock_write_release(&tbl->lock); +} +#endif + +#endif diff --git a/include/vsync/map/treeset_bst_coarse.h b/include/vsync/map/treeset_bst_coarse.h new file mode 100644 index 0000000..2719d47 --- /dev/null +++ b/include/vsync/map/treeset_bst_coarse.h @@ -0,0 +1,198 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_BST_COARSE +#define VSYNC_TREESET_BST_COARSE + +/******************************************************************************* + * @file treeset_bst_coarse.h + * @ingroup linearizable + * @brief This implementation of treeset uses unbalanced binary search tree + * (BST) and coarse-grained locking. + * + * A treeset is a binary search tree with nodes that are ordered by unique keys. + * + * Available implementations share the same interface, but differ in three + * orthogonal aspects: + * * balancing scheme: unbalanced binary search tree (BST) or balanced red-black + * tree (RB), + * * synchronisation scheme: so far only coarse-grained locking and fine-grained + * locking, + * * allocation scheme: so far only malloc-like allocation. + * + * All implementations use external trees, where values are stored in external + * nodes (leaves) of the tree, and internal nodes are used only for routing. + * + * For lock-based implementations one need to choose desired lock + * implementation: + * * `-DTREESET_LOCK_PTHREAD` (for pthread mutex), + * * `-DTREESET_LOCK_TTAS` (for libvsync TTAS lock), + * * `-DTREESET_LOCK_RW` (for libvsync readers-writer lock). + * + * @example + * @include eg_treeset_bst_coarse.c + ******************************************************************************/ + +#include +#include +#include +#include +#include + +typedef struct treeset_node_s { + treeset_key_t key; + vbool_t external; + struct treeset_node_s *child[2]; +} treeset_node_t; + +typedef struct treeset_s { + lock_t coarse_lock; + treeset_node_t head_sentinel; + vmem_lib_t mem_lib; +} treeset_t; + +#include + +#define TREESET_REPLACE_HEADER +#define treeset_init treeset_init_fine +#define treeset_destroy treeset_destroy_fine +#define treeset_add treeset_add_fine +#define treeset_remove treeset_remove_fine +#define treeset_contains treeset_contains_fine +#define treeset_visit treeset_visit_fine +#define l_init(l) +#define l_destroy(l) +#define l_acquire(l) +#define l_release(l) +#define l_reader_acquire(l) +#define l_reader_release(l) + +#include + +#undef TREESET_REPLACE_HEADER +#undef treeset_init +#undef treeset_destroy +#undef treeset_add +#undef treeset_remove +#undef treeset_contains +#undef treeset_visit +#undef l_init +#undef l_destroy +#undef l_acquire +#undef l_release +#undef l_reader_acquire +#undef l_reader_release + +/** + * Initializes the treeset. + * + * @note must be called before threads access the treeset. + * @param tree address of the treeset_t object. + * @param mem_lib object of type `vmem_lib_t` containing malloc/free functions + * to allocate/free internal nodes. + */ +static inline void +treeset_init(treeset_t *tree, vmem_lib_t mem_lib) +{ + ASSERT(tree); + ASSERT(vmem_lib_not_null(&mem_lib)); + l_init(&tree->coarse_lock); + treeset_init_fine(tree, mem_lib); +} + +/** + * Destroys all the remaining nodes in the treeset. + * + * @note call only after thread join, or after all threads finished accessing + * the treeset. + * @param tree address of the treeset_t object. + */ +static inline void +treeset_destroy(treeset_t *tree) +{ + ASSERT(tree); + l_destroy(&tree->coarse_lock); + treeset_destroy_fine(tree); +} + +/** + * Attempts to insert an element with a given key and value into the treeset. + * + * @param tree address of the treeset_t object. + * @param key the key to be inserted. + * @param value value to be associated with inserted key. + * @param out_value out parameter for the previous value associated with the + * key. + * @return true operation succeeded. + * @return false operation failed, since the given key was already in the + * treeset, in the `out_value` the value of this element is returned. + */ +static inline vbool_t +treeset_add(treeset_t *tree, treeset_key_t key, void *value, void **out_value) +{ + ASSERT(tree); + l_acquire(&tree->coarse_lock); + vbool_t result = treeset_add_fine(tree, key, value, out_value); + l_release(&tree->coarse_lock); + return result; +} + +/** + * Attempts to remove an element with a given key from the treeset. + * + * @param tree address of the treeset_t object. + * @param key the key to be removed. + * @param out_value out parameter for the value associated with the key. + * @return true operation succeeded, in the `out_value` the value of the removed + * element is returned. + * @return false operation failed, there is no element with the given key. + */ +static inline vbool_t +treeset_remove(treeset_t *tree, treeset_key_t key, void **out_value) +{ + ASSERT(tree); + l_acquire(&tree->coarse_lock); + vbool_t result = treeset_remove_fine(tree, key, out_value); + l_release(&tree->coarse_lock); + return result; +} + +/** + * Searches the treeset for an element with a given key. + * + * @param tree address of the treeset_t object. + * @param key the key to be searched for. + * @param out_value out parameter for the value associated with the key. + * @return true operation succeeded, in the `out_value` the value of the found + * element is returned. + * @return false operation failed, there is no element with the given key. + */ +static inline vbool_t +treeset_contains(treeset_t *tree, treeset_key_t key, void **out_value) +{ + ASSERT(tree); + l_reader_acquire(&tree->coarse_lock); + vbool_t result = treeset_contains_fine(tree, key, out_value); + l_reader_release(&tree->coarse_lock); + return result; +} + +/** + * Visits all elements in the treeset. + * + * @note call only after thread join, or after all threads finished accessing + * the treeset. + * @param tree address of the treeset_t object. + * @param visitor address of the function to call on each element. + * @param arg the third argument to the visitor function. + */ +static inline void +treeset_visit(treeset_t *tree, treeset_visitor visitor, void *arg) +{ + ASSERT(tree); + treeset_visit_fine(tree, visitor, arg); +} + +#endif diff --git a/include/vsync/map/treeset_bst_fine.h b/include/vsync/map/treeset_bst_fine.h new file mode 100644 index 0000000..0cb8a14 --- /dev/null +++ b/include/vsync/map/treeset_bst_fine.h @@ -0,0 +1,401 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_BST_FINE +#define VSYNC_TREESET_BST_FINE + +/******************************************************************************* + * @file treeset_bst_fine.h + * @ingroup linearizable + * @brief This implementation of treeset uses unbalanced binary search tree + * (BST) and fine-grained locking. + * + * Refer to treeset_bst_coarse.h for more general information about treeset. + * + * @example + * @include eg_treeset_bst_fine.c + ******************************************************************************/ + +#ifndef TREESET_REPLACE_HEADER + + #include + #include + #include + #include + #include + +typedef struct treeset_node_s { + lock_t lock; + treeset_key_t key; + vbool_t external; + struct treeset_node_s *child[2]; +} treeset_node_t; + +typedef struct treeset_s { + treeset_node_t head_sentinel; + vmem_lib_t mem_lib; +} treeset_t; + + #include + +#endif + +static inline void _treeset_putall(treeset_t *tree); + +/** + * Initializes the treeset. + * + * @note must be called before threads access the treeset. + * @param tree address of the treeset_t object. + * @param mem_lib object of type `vmem_lib_t` containing malloc/free functions + * to allocate/free internal nodes. + */ +static inline void +treeset_init(treeset_t *tree, vmem_lib_t mem_lib) +{ + ASSERT(tree); + ASSERT(vmem_lib_not_null(&mem_lib)); + tree->head_sentinel.key = 0; // unused + tree->head_sentinel.external = false; + tree->head_sentinel.child[0] = NULL; + tree->head_sentinel.child[1] = NULL; + tree->mem_lib = mem_lib; + l_init(&tree->head_sentinel.lock); +} + +/** + * Destroys all the remaining nodes in the treeset. + * + * @note call only after thread join, or after all threads finished accessing + * the treeset. + * @param tree address of the treeset_t object. + */ +static inline void +treeset_destroy(treeset_t *tree) +{ + ASSERT(tree); + _treeset_putall(tree); + l_destroy(&tree->head_sentinel.lock); +} + +/** + * Attempts to insert an element with a given key and value into the treeset. + * + * @param tree address of the treeset_t object. + * @param key the key to be inserted. + * @param value value to be associated with inserted key. + * @param out_value out parameter for the previous value associated with the + * key. + * @return true operation succeeded. + * @return false operation failed, since the given key was already in the + * treeset, in the `out_value` the value of this element is returned. + */ +static inline vbool_t +treeset_add(treeset_t *tree, treeset_key_t key, void *value, void **out_value) +{ + ASSERT(tree); + + // x = currently processed node + // f = its father + // cX = index of X's child that lies on the path + + treeset_node_t *f = &tree->head_sentinel; + l_acquire(&f->lock); + treeset_node_t *x = f->child[0]; + + if (unlikely(!x)) { + treeset_node_t *ext = _treeset_get_node(tree); + ext->key = key; + ext->external = true; + ext->child[0] = value; + ext->child[1] = NULL; + l_init(&ext->lock); + + f->child[0] = ext; + l_release(&f->lock); + return true; + } + + l_acquire(&x->lock); + + while (!x->external) { + l_release(&f->lock); + const vsize_t cx = x->key <= key; + f = x; + x = x->child[cx]; + l_acquire(&x->lock); + } + + if (x->key == key) { + l_release(&f->lock); + if (out_value) { + *out_value = x->child[0]; + } + l_release(&x->lock); + return false; + } + + /* new */ + treeset_node_t *ext = _treeset_get_node(tree); + ext->key = key; + ext->external = true; + ext->child[0] = value; + ext->child[1] = NULL; + l_init(&ext->lock); + + treeset_node_t *mid = _treeset_get_node(tree); + mid->external = false; + l_init(&mid->lock); + l_acquire(&mid->lock); + + if (x->key < key) { + mid->key = key; + mid->child[0] = x; + mid->child[1] = ext; + } else { + mid->key = x->key; + mid->child[0] = ext; + mid->child[1] = x; + } + + const vsize_t cf = (f != &tree->head_sentinel) && (f->key <= key); + f->child[cf] = mid; + l_release(&f->lock); + l_release(&mid->lock); + l_release(&x->lock); + return true; +} + +/** + * Attempts to remove an element with a given key from the treeset. + * + * @param tree address of the treeset_t object. + * @param key the key to be removed. + * @param out_value out parameter for the value associated with the key. + * @return true operation succeeded, in the `out_value` the value of the removed + * element is returned. + * @return false operation failed, there is no element with the given key. + */ +static inline vbool_t +treeset_remove(treeset_t *tree, treeset_key_t key, void **out_value) +{ + ASSERT(tree); + + // x = currently processed node (black) + // f = its father + // g = its grandfather + // y = its child + // cX = index of X's child that lies on the path + + treeset_node_t *g = NULL; + treeset_node_t *f = &tree->head_sentinel; + l_acquire(&f->lock); + treeset_node_t *x = f->child[0]; + + /* 0 elements */ + if (unlikely(!x)) { + l_release(&f->lock); + return false; + } + l_acquire(&x->lock); + + /* 1 element */ + if (unlikely(x->external)) { + if (x->key != key) { + l_release(&f->lock); + l_release(&x->lock); + return false; + } + f->child[0] = NULL; + l_release(&f->lock); + if (out_value) { + *out_value = x->child[0]; + } + l_release(&x->lock); + l_destroy(&x->lock); + _treeset_put_node(tree, x); + return true; + } + + while (!x->external) { + if (g) { + l_release(&g->lock); + } + const vsize_t cx = x->key <= key; + g = f; + f = x; + x = x->child[cx]; + l_acquire(&x->lock); + } + + /* x is external */ + if (x->key != key) { + l_release(&g->lock); + l_release(&f->lock); + l_release(&x->lock); + return false; + } + + const vsize_t cg = (g != &tree->head_sentinel) && (g->key <= key); + const vsize_t cf = f->key <= key; + + treeset_node_t *y = f->child[!cf]; + l_acquire(&y->lock); + g->child[cg] = y; + + l_release(&g->lock); + l_release(&y->lock); + + l_release(&f->lock); + l_destroy(&f->lock); + if (out_value) { + *out_value = x->child[0]; + } + l_release(&x->lock); + l_destroy(&x->lock); + + _treeset_put_node(tree, f); + _treeset_put_node(tree, x); + + return true; +} + +/** + * Searches the treeset for an element with a given key. + * + * @param tree address of the treeset_t object. + * @param key the key to be searched for. + * @param out_value out parameter for the value associated with the key. + * @return true operation succeeded, in the `out_value` the value of the found + * element is returned. + * @return false operation failed, there is no element with the given key. + */ +static inline vbool_t +treeset_contains(treeset_t *tree, treeset_key_t key, void **out_value) +{ + ASSERT(tree); + + // x = currently processed node (black) + // f = its father + // cX = index of X's child that lies on the path + + treeset_node_t *f = &tree->head_sentinel; + l_reader_acquire(&f->lock); + treeset_node_t *x = f->child[0]; + + if (unlikely(!x)) { + l_reader_release(&f->lock); + return false; + } + + l_reader_acquire(&x->lock); + l_reader_release(&f->lock); + + while (!x->external) { + const vsize_t cx = x->key <= key; + f = x; + x = x->child[cx]; + l_reader_acquire(&x->lock); + l_reader_release(&f->lock); + } + + if (x->key != key) { + l_reader_release(&x->lock); + return false; + } + + if (out_value) { + *out_value = x->child[0]; + } + l_reader_release(&x->lock); + return true; +} + +static inline void +_treeset_visit_recursive(treeset_node_t *node, treeset_visitor visitor, + void *arg) +{ + if (node->external) { + visitor(node->key, node->child[0], arg); + } else { + _treeset_visit_recursive(node->child[0], visitor, arg); + _treeset_visit_recursive(node->child[1], visitor, arg); + } +} + +/** + * Visits all elements in the treeset. + * + * @note call only after thread join, or after all threads finished accessing + * the treeset. + * @param tree address of the treeset_t object. + * @param visitor address of the function to call on each element. + * @param arg the third argument to the visitor function. + */ +static inline void +treeset_visit(treeset_t *tree, treeset_visitor visitor, void *arg) +{ + ASSERT(tree); + ASSERT(visitor); + + treeset_node_t *root = tree->head_sentinel.child[0]; + if (root) { + _treeset_visit_recursive(root, visitor, arg); + } +} + +static inline void +_treeset_putall_recursive(treeset_t *tree, treeset_node_t *node) +{ + if (!node->external) { + _treeset_putall_recursive(tree, node->child[0]); + _treeset_putall_recursive(tree, node->child[1]); + } + l_destroy(&node->lock); + _treeset_put_node(tree, node); +} + +static inline void +_treeset_putall(treeset_t *tree) +{ + ASSERT(tree); + + treeset_node_t *root = tree->head_sentinel.child[0]; + if (root) { + _treeset_putall_recursive(tree, root); + } +} + +static inline void +_treeset_verify_recursive(treeset_node_t *node, vbool_t unlimited_b, + treeset_key_t limit_b, vbool_t unlimited_e, + treeset_key_t limit_e) +{ + vbool_t limit_b_ok = unlimited_b || limit_b <= node->key; + vbool_t limit_e_ok = unlimited_e || node->key < limit_e; + ASSERT(limit_b_ok && "key invariant broken"); + ASSERT(limit_e_ok && "key invariant broken"); + + if (!node->external) { + _treeset_verify_recursive(node->child[0], unlimited_b, limit_b, false, + node->key); + _treeset_verify_recursive(node->child[1], false, node->key, unlimited_e, + limit_e); + } +} + +static inline void +_treeset_verify(treeset_t *tree) +{ + ASSERT(tree); + + treeset_node_t *root = tree->head_sentinel.child[0]; + if (root) { + // if unlimited is true, limit is ignored + _treeset_verify_recursive(root, true, 0, true, 0); + } +} + +#endif diff --git a/include/vsync/map/treeset_rb_coarse.h b/include/vsync/map/treeset_rb_coarse.h new file mode 100644 index 0000000..9052fb9 --- /dev/null +++ b/include/vsync/map/treeset_rb_coarse.h @@ -0,0 +1,182 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_RB_COARSE +#define VSYNC_TREESET_RB_COARSE + +/******************************************************************************* + * @file treeset_rb_coarse.h + * @ingroup linearizable + * @brief This implementation of treeset uses balanced red-black tree + * (RB) and coarse-grained locking. + * + * Refer to treeset_bst_coarse.h for more general information about treeset. + * + * @example + * @include eg_treeset_rb_coarse.c + ******************************************************************************/ + +#include +#include +#include +#include +#include + +typedef struct treeset_node_s { + treeset_key_t key; + vbool_t red; + vbool_t external; + struct treeset_node_s *child[2]; +} treeset_node_t; + +typedef struct treeset_s { + lock_t coarse_lock; + treeset_node_t head_sentinel; + vmem_lib_t mem_lib; +} treeset_t; + +#include + +#define TREESET_REPLACE_HEADER +#define treeset_init treeset_init_fine +#define treeset_destroy treeset_destroy_fine +#define treeset_add treeset_add_fine +#define treeset_remove treeset_remove_fine +#define treeset_contains treeset_contains_fine +#define treeset_visit treeset_visit_fine +#define l_init(l) +#define l_destroy(l) +#define l_acquire(l) +#define l_release(l) +#define l_reader_acquire(l) +#define l_reader_release(l) + +#include + +#undef TREESET_REPLACE_HEADER +#undef treeset_init +#undef treeset_destroy +#undef treeset_add +#undef treeset_remove +#undef treeset_contains +#undef treeset_visit +#undef l_init +#undef l_destroy +#undef l_acquire +#undef l_release +#undef l_reader_acquire +#undef l_reader_release + +/** + * Initializes the treeset. + * + * @note must be called before threads access the treeset. + * @param tree address of the treeset_t object. + * @param mem_lib object of type `vmem_lib_t` containing malloc/free functions + * to allocate/free internal nodes. + */ +static inline void +treeset_init(treeset_t *tree, vmem_lib_t mem_lib) +{ + ASSERT(tree); + ASSERT(vmem_lib_not_null(&mem_lib)); + l_init(&tree->coarse_lock); + treeset_init_fine(tree, mem_lib); +} + +/** + * Destroys all the remaining nodes in the treeset. + * + * @note call only after thread join, or after all threads finished accessing + * the treeset. + * @param tree address of the treeset_t object. + */ +static inline void +treeset_destroy(treeset_t *tree) +{ + ASSERT(tree); + l_destroy(&tree->coarse_lock); + treeset_destroy_fine(tree); +} + +/** + * Attempts to insert an element with a given key and value into the treeset. + * + * @param tree address of the treeset_t object. + * @param key the key to be inserted. + * @param value value to be associated with inserted key. + * @param out_value out parameter for the previous value associated with the + * key. + * @return true operation succeeded. + * @return false operation failed, since the given key was already in the + * treeset, in the `out_value` the value of this element is returned. + */ +static inline vbool_t +treeset_add(treeset_t *tree, treeset_key_t key, void *value, void **out_value) +{ + ASSERT(tree); + l_acquire(&tree->coarse_lock); + vbool_t result = treeset_add_fine(tree, key, value, out_value); + l_release(&tree->coarse_lock); + return result; +} + +/** + * Attempts to remove an element with a given key from the treeset. + * + * @param tree address of the treeset_t object. + * @param key the key to be removed. + * @param out_value out parameter for the value associated with the key. + * @return true operation succeeded, in the `out_value` the value of the removed + * element is returned. + * @return false operation failed, there is no element with the given key. + */ +static inline vbool_t +treeset_remove(treeset_t *tree, treeset_key_t key, void **out_value) +{ + ASSERT(tree); + l_acquire(&tree->coarse_lock); + vbool_t result = treeset_remove_fine(tree, key, out_value); + l_release(&tree->coarse_lock); + return result; +} + +/** + * Searches the treeset for an element with a given key. + * + * @param tree address of the treeset_t object. + * @param key the key to be searched for. + * @param out_value out parameter for the value associated with the key. + * @return true operation succeeded, in the `out_value` the value of the found + * element is returned. + * @return false operation failed, there is no element with the given key. + */ +static inline vbool_t +treeset_contains(treeset_t *tree, treeset_key_t key, void **out_value) +{ + ASSERT(tree); + l_reader_acquire(&tree->coarse_lock); + vbool_t result = treeset_contains_fine(tree, key, out_value); + l_reader_release(&tree->coarse_lock); + return result; +} + +/** + * Visits all elements in the treeset. + * + * @note call only after thread join, or after all threads finished accessing + * the treeset. + * @param tree address of the treeset_t object. + * @param visitor address of the function to call on each element. + * @param arg the third argument to the visitor function. + */ +static inline void +treeset_visit(treeset_t *tree, treeset_visitor visitor, void *arg) +{ + ASSERT(tree); + treeset_visit_fine(tree, visitor, arg); +} + +#endif diff --git a/include/vsync/map/treeset_rb_fine.h b/include/vsync/map/treeset_rb_fine.h new file mode 100644 index 0000000..d301774 --- /dev/null +++ b/include/vsync/map/treeset_rb_fine.h @@ -0,0 +1,652 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TREESET_RB_FINE +#define VSYNC_TREESET_RB_FINE + +/******************************************************************************* + * @file treeset_rb_fine.h + * @ingroup linearizable + * @brief This implementation of treeset uses balanced red-black tree + * (RB) and fine-grained locking. + * + * Refer to treeset_bst_coarse.h for more general information about treeset. + * + * @example + * @include eg_treeset_rb_fine.c + ******************************************************************************/ + +#ifndef TREESET_REPLACE_HEADER + + #include + #include + #include + #include + #include + +typedef struct treeset_node_s { + lock_t lock; + treeset_key_t key; + vbool_t red; + vbool_t external; + struct treeset_node_s *child[2]; +} treeset_node_t; + +typedef struct treeset_s { + treeset_node_t head_sentinel; + vmem_lib_t mem_lib; +} treeset_t; + + #include + +#endif + +static inline treeset_node_t * +_treeset_rebalance(treeset_t *tree, treeset_node_t *gg, treeset_node_t *g, + treeset_node_t *f, treeset_node_t *x, treeset_key_t key); + +static inline void _treeset_putall(treeset_t *tree); + +/** + * Initializes the treeset. + * + * @note must be called before threads access the treeset. + * @param tree address of the treeset_t object. + * @param mem_lib object of type `vmem_lib_t` containing malloc/free functions + * to allocate/free internal nodes. + */ +static inline void +treeset_init(treeset_t *tree, vmem_lib_t mem_lib) +{ + ASSERT(tree); + ASSERT(vmem_lib_not_null(&mem_lib)); + tree->head_sentinel.key = 0; // unused + tree->head_sentinel.red = false; + tree->head_sentinel.external = false; + tree->head_sentinel.child[0] = NULL; + tree->head_sentinel.child[1] = NULL; + tree->mem_lib = mem_lib; + l_init(&tree->head_sentinel.lock); +} + +/** + * Destroys all the remaining nodes in the treeset. + * + * @note call only after thread join, or after all threads finished accessing + * the treeset. + * @param tree address of the treeset_t object. + */ +static inline void +treeset_destroy(treeset_t *tree) +{ + ASSERT(tree); + _treeset_putall(tree); + l_destroy(&tree->head_sentinel.lock); +} + +/** + * Attempts to insert an element with a given key and value into the treeset. + * + * @param tree address of the treeset_t object. + * @param key the key to be inserted. + * @param value value to be associated with inserted key. + * @param out_value out parameter for the previous value associated with the + * key. + * @return true operation succeeded. + * @return false operation failed, since the given key was already in the + * treeset, in the `out_value` the value of this element is returned. + */ +static inline vbool_t +treeset_add(treeset_t *tree, treeset_key_t key, void *value, void **out_value) +{ + ASSERT(tree); + + // x = currently processed node (black) + // f = its father + // g = its grandfather + // gg = its great-grandfather + // y = its child on the path + // cX = index of X's child that lies on the path + + treeset_node_t *gg = NULL; + treeset_node_t *g = NULL; + treeset_node_t *f = &tree->head_sentinel; + l_acquire(&f->lock); + treeset_node_t *x = f->child[0]; + vbool_t f_red = false; + + if (unlikely(!x)) { + treeset_node_t *ext = _treeset_get_node(tree); + ext->key = key; + ext->red = false; + ext->external = true; + ext->child[0] = value; + ext->child[1] = NULL; + l_init(&ext->lock); + + f->child[0] = ext; + l_release(&f->lock); + return true; + } + + l_acquire(&x->lock); + + while (!x->external) { + const vbool_t Red[2] = {x->child[0]->red, x->child[1]->red}; + ASSERT(!x->red); + + if (Red[0] && Red[1]) { + /* 4-node */ + x->child[0]->red = false; + x->child[1]->red = false; + x->red = f != &tree->head_sentinel; + + if (f_red) { + /* Case 4 */ + x = _treeset_rebalance(tree, gg, g, f, x, key); + } else { + /* Case 3 */ + l_release(&f->lock); + const vsize_t cx = x->key <= key; + treeset_node_t *y = x->child[cx]; + l_acquire(&y->lock); + l_release(&x->lock); + x = y; + } + f_red = false; + } else { + if (f_red) { + l_release(&gg->lock); + l_release(&g->lock); + } + const vsize_t cx = x->key <= key; + if (Red[cx]) { + /* Case 2: 3-node bottom */ + gg = f; + g = x; + x = x->child[cx]; + l_acquire(&x->lock); + f_red = true; + } else { + /* Case 1: 3-node top or 2-node */ + l_release(&f->lock); + f_red = false; + } + } + + const vsize_t cx = x->key <= key; + f = x; + x = x->child[cx]; + l_acquire(&x->lock); + } + + if (x->key == key) { + if (f_red) { + l_release(&gg->lock); + l_release(&g->lock); + } + l_release(&f->lock); + if (out_value) { + *out_value = x->child[0]; + } + l_release(&x->lock); + return false; + } + + /* new */ + treeset_node_t *ext = _treeset_get_node(tree); + ext->key = key; + ext->red = false; + ext->external = true; + ext->child[0] = value; + ext->child[1] = NULL; + l_init(&ext->lock); + + treeset_node_t *mid = _treeset_get_node(tree); + mid->red = f != &tree->head_sentinel; + mid->external = false; + l_init(&mid->lock); + l_acquire(&mid->lock); + + if (x->key < key) { + mid->key = key; + mid->child[0] = x; + mid->child[1] = ext; + } else { + mid->key = x->key; + mid->child[0] = ext; + mid->child[1] = x; + } + + const vsize_t cf = (f != &tree->head_sentinel) && (f->key <= key); + f->child[cf] = mid; + l_release(&x->lock); + + if (f_red) { + /* Case 6 */ + _treeset_rebalance(tree, gg, g, f, mid, key); + } else { + /* Case 5 */ + l_release(&f->lock); + l_release(&mid->lock); + } + + return true; +} + +/** + * Attempts to remove an element with a given key from the treeset. + * + * @param tree address of the treeset_t object. + * @param key the key to be removed. + * @param out_value out parameter for the value associated with the key. + * @return true operation succeeded, in the `out_value` the value of the removed + * element is returned. + * @return false operation failed, there is no element with the given key. + */ +static inline vbool_t +treeset_remove(treeset_t *tree, treeset_key_t key, void **out_value) +{ + ASSERT(tree); + + // x = currently processed node (black) + // f = its father + // g = its grandfather + // y = its child + // b = its sibling + // bb = child of its sibling + // cX = index of X's child that lies on the path + + treeset_node_t *g = NULL; + treeset_node_t *f = &tree->head_sentinel; + l_acquire(&f->lock); + treeset_node_t *x = f->child[0]; + + /* 0 elements */ + if (unlikely(!x)) { + l_release(&f->lock); + return false; + } + l_acquire(&x->lock); + + /* 1 element */ + if (unlikely(x->external)) { + if (x->key != key) { + l_release(&f->lock); + l_release(&x->lock); + return false; + } + f->child[0] = NULL; + l_release(&f->lock); + if (out_value) { + *out_value = x->child[0]; + } + l_release(&x->lock); + l_destroy(&x->lock); + _treeset_put_node(tree, x); + return true; + } + + /* 2 or more elements */ + if (!x->child[0]->red && !x->child[1]->red) { + /* root is 2-node */ + /* Assume f->red = true; */ + const vsize_t cx = x->key <= key; + g = f; + f = x; + x = x->child[cx]; + l_acquire(&x->lock); + } + + while (!x->external) { + const vbool_t Red[2] = {x->child[0]->red, x->child[1]->red}; + ASSERT(!x->red); + + const vsize_t cx = x->key <= key; + if (Red[cx]) { + /* Case 1: 4-node or 3-node bottom */ + if (g) { + l_release(&g->lock); + } + l_release(&f->lock); + g = x; + f = x->child[cx]; + l_acquire(&f->lock); + } else if (Red[!cx]) { + /* Case 2: 3-node top */ + if (g) { + l_release(&g->lock); + } + treeset_node_t *s = x->child[!cx]; + l_acquire(&s->lock); + const vsize_t cf = (f != &tree->head_sentinel) && (f->key <= key); + // rotate(x, s, !cx) + x->child[!cx] = s->child[cx]; + s->child[cx] = x; + f->child[cf] = s; + s->red = false; + l_release(&f->lock); + x->red = true; + g = s; + f = x; + } else { + ASSERT(g); + ASSERT(f != &tree->head_sentinel); + /* 2-node */ + const vsize_t cf = f->key <= key; + treeset_node_t *b = f->child[!cf]; + ASSERT(!b->external); + ASSERT(!b->red); + + l_acquire(&b->lock); + const vbool_t b_red[2] = {b->child[0]->red, b->child[1]->red}; + + if (!b_red[0] && !b_red[1]) { + /* Case 3: brother 2-node */ + l_release(&b->lock); + f->red = false; + x->red = true; + b->red = true; + l_release(&g->lock); + g = f; + f = x; + } else if (b_red[cf]) { + /* Case 4b: brother 4-node or same-dir 3-node */ + treeset_node_t *bb = b->child[cf]; + ASSERT(bb->red); + l_acquire(&bb->lock); + + // rotate(b, bb, cf) + b->child[cf] = bb->child[!cf]; + bb->child[!cf] = b; + + l_release(&b->lock); + + // rotate(f, bb, !cf) + f->child[!cf] = bb->child[cf]; + bb->child[cf] = f; + + const vsize_t cg = + (g != &tree->head_sentinel) && (g->key <= key); + g->child[cg] = bb; + f->red = false; + x->red = true; + bb->red = g != &tree->head_sentinel; + + l_release(&g->lock); + l_release(&bb->lock); + g = f; + f = x; + } else { + /* Case 4a: brother opposite-dir 3-node */ + treeset_node_t *bb = b->child[!cf]; + ASSERT(!bb->external); + ASSERT(bb->red); + + // rotate(f, b, !cf) + f->child[!cf] = b->child[cf]; + b->child[cf] = f; + + const vsize_t cg = + (g != &tree->head_sentinel) && (g->key <= key); + g->child[cg] = b; + f->red = false; + x->red = true; + b->red = g != &tree->head_sentinel; + bb->red = false; + + l_release(&g->lock); + l_release(&b->lock); + g = f; + f = x; + } + } + + const vsize_t cf = f->key <= key; + x = f->child[cf]; + l_acquire(&x->lock); + } + + ASSERT(g); + + /* x is external */ + if (x->key != key) { + l_release(&g->lock); + l_release(&f->lock); + l_release(&x->lock); + return false; + } + + const vsize_t cg = (g != &tree->head_sentinel) && (g->key <= key); + const vsize_t cf = f->key <= key; + + treeset_node_t *b = f->child[!cf]; + l_acquire(&b->lock); + g->child[cg] = b; + + l_release(&g->lock); + l_release(&b->lock); + + l_release(&f->lock); + l_destroy(&f->lock); + if (out_value) { + *out_value = x->child[0]; + } + l_release(&x->lock); + l_destroy(&x->lock); + + _treeset_put_node(tree, f); + _treeset_put_node(tree, x); + + return true; +} + +/** + * Searches the treeset for an element with a given key. + * + * @param tree address of the treeset_t object. + * @param key the key to be searched for. + * @param out_value out parameter for the value associated with the key. + * @return true operation succeeded, in the `out_value` the value of the found + * element is returned. + * @return false operation failed, there is no element with the given key. + */ +static inline vbool_t +treeset_contains(treeset_t *tree, treeset_key_t key, void **out_value) +{ + ASSERT(tree); + + // x = currently processed node (black) + // f = its father + // cX = index of X's child that lies on the path + + treeset_node_t *f = &tree->head_sentinel; + l_reader_acquire(&f->lock); + treeset_node_t *x = f->child[0]; + + if (unlikely(!x)) { + l_reader_release(&f->lock); + return false; + } + + l_reader_acquire(&x->lock); + l_reader_release(&f->lock); + + while (!x->external) { + const vsize_t cx = x->key <= key; + f = x; + x = x->child[cx]; + l_reader_acquire(&x->lock); + l_reader_release(&f->lock); + } + + if (x->key != key) { + l_reader_release(&x->lock); + return false; + } + + if (out_value) { + *out_value = x->child[0]; + } + l_reader_release(&x->lock); + return true; +} + +static inline treeset_node_t * +_treeset_rebalance(treeset_t *tree, treeset_node_t *gg, treeset_node_t *g, + treeset_node_t *f, treeset_node_t *x, treeset_key_t key) +{ + // x = currently processed node (black) + // f = its father + // g = its grandfather + // gg = its great-grandfather + // y = its child on the path + // cX = index of X's child that lies on the path + + const vsize_t cgg = (gg != &tree->head_sentinel) && (gg->key <= key); + const vsize_t cg = g->key <= key; + const vsize_t cf = f->key <= key; + const vsize_t cx = x->key <= key; + + treeset_node_t *y = x->child[cx]; + + if (cg == cf) { + /* Case a: single rotation */ + g->child[cg] = f->child[!cg]; + f->child[!cg] = g; + gg->child[cgg] = f; + f->red = false; + l_release(&gg->lock); + g->red = true; + l_release(&f->lock); + l_release(&g->lock); + } else { + /* Case b: double rotation */ + g->child[cg] = x->child[!cg]; + x->child[!cg] = g; + f->child[!cg] = x->child[cg]; + x->child[cg] = f; + gg->child[cgg] = x; + x->red = false; + l_release(&gg->lock); + g->red = true; + l_release(&x->lock); + + if (cx == cg) { + l_release(&g->lock); + x = f; + } else { + l_release(&f->lock); + x = g; + } + } + + if (!y->external) { + /* 4-node */ + l_acquire(&y->lock); + l_release(&x->lock); + x = y; + } else { + /* new */ + l_release(&x->lock); + x = NULL; + } + + return x; +} + +static inline void +_treeset_visit_recursive(treeset_node_t *node, treeset_visitor visitor, + void *arg) +{ + if (node->external) { + visitor(node->key, node->child[0], arg); + } else { + _treeset_visit_recursive(node->child[0], visitor, arg); + _treeset_visit_recursive(node->child[1], visitor, arg); + } +} + +/** + * Visits all elements in the treeset. + * + * @note call only after thread join, or after all threads finished accessing + * the treeset. + * @param tree address of the treeset_t object. + * @param visitor address of the function to call on each element. + * @param arg the third argument to the visitor function. + */ +static inline void +treeset_visit(treeset_t *tree, treeset_visitor visitor, void *arg) +{ + ASSERT(tree); + ASSERT(visitor); + + treeset_node_t *root = tree->head_sentinel.child[0]; + if (root) { + _treeset_visit_recursive(root, visitor, arg); + } +} + +static inline void +_treeset_putall_recursive(treeset_t *tree, treeset_node_t *node) +{ + if (!node->external) { + _treeset_putall_recursive(tree, node->child[0]); + _treeset_putall_recursive(tree, node->child[1]); + } + l_destroy(&node->lock); + _treeset_put_node(tree, node); +} + +static inline void +_treeset_putall(treeset_t *tree) +{ + ASSERT(tree); + + treeset_node_t *root = tree->head_sentinel.child[0]; + if (root) { + _treeset_putall_recursive(tree, root); + } +} + +static inline int +_treeset_verify_recursive(treeset_node_t *node, vbool_t f_red, + vbool_t unlimited_b, treeset_key_t limit_b, + vbool_t unlimited_e, treeset_key_t limit_e) +{ + vbool_t limit_b_ok = unlimited_b || limit_b <= node->key; + vbool_t limit_e_ok = unlimited_e || node->key < limit_e; + ASSERT(limit_b_ok && "key invariant broken"); + ASSERT(limit_e_ok && "key invariant broken"); + + if (node->external) { + ASSERT(!node->red && "external invariant broken"); + return 0; + } + int bh_le = _treeset_verify_recursive( + node->child[0], node->red, unlimited_b, limit_b, false, node->key); + int bh_ri = _treeset_verify_recursive(node->child[1], node->red, false, + node->key, unlimited_e, limit_e); + + ASSERT(bh_le == bh_ri && "black invariant broken"); + ASSERT((!f_red || !node->red) && "red invariant broken"); + + return bh_le + !node->red; +} + +static inline void +_treeset_verify(treeset_t *tree) +{ + ASSERT(tree); + + treeset_node_t *root = tree->head_sentinel.child[0]; + if (root) { + // if unlimited is true, limit is ignored + _treeset_verify_recursive(root, true, true, 0, true, 0); + } +} + +#endif diff --git a/include/vsync/spinlock/arraylock.h b/include/vsync/spinlock/arraylock.h new file mode 100644 index 0000000..a2ec81f --- /dev/null +++ b/include/vsync/spinlock/arraylock.h @@ -0,0 +1,119 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_ARRAYLOCK_H +#define VSYNC_ARRAYLOCK_H +/******************************************************************************* + * @file arraylock.h + * @ingroup fair_lock + * @brief Simple array-based queue lock + * + * The array lock has an array of flags, each slot is associated with a thread. + * If we have `N` threads then the array has `N` slots. Each slot + * represents a boolean flag indicating the associated thread's permission to + * acquire the lock. The thread waits on its flag to become `true` to acquire + * the lock. The thread releases the lock, by giving away its permission to the + * next thread, i.e. sets its own flag to `false` and the flag of the one next + * in line to `true`. + * + * Initially the first flag is set to `true` and the rest to `false`, and the + * tail holds the index of the first slot. + * + * @example + * @include eg_arraylock.c + * + * @cite + * Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 7.5.1] + * (https://dl.acm.org/doi/pdf/10.5555/2385452) + ******************************************************************************/ + +#include +#include +#include + +typedef struct arraylock_flag_s { + vatomic8_t flag; +} VSYNC_CACHEALIGN arraylock_flag_t; + +typedef struct arraylock_s { + vatomic8_t tail; /* The virtual index of the slot that belongs to the + current thread */ + arraylock_flag_t *flags; /* An array that holds a slot for each thread, + indicating its permission */ + vuint32_t flags_len; /* length of flags */ +} arraylock_t; + +/** + * Initializes the given lock object. + * + * @param lock address of arraylock_t object. + * @param flags address of arraylock_flag_t object. + * @param len next power of two greater or equal to the maximum number of + * threads. Length of `flags`array. + * + * @note `len` MUST BE A POWER OF TWO. This is required because we use a + * circular array and take advantage of unsigned integers "overflow". + */ +static inline void +arraylock_init(arraylock_t *lock, arraylock_flag_t *flags, const vuint32_t len) +{ + vuint32_t i = 0; + + vatomic8_init(&lock->tail, 0); + ASSERT(len > 0); + ASSERT(V_IS_POWER_OF_TWO(len)); + lock->flags_len = len; + lock->flags = flags; + + /* first slot init as one */ + vatomic8_init(&lock->flags[0].flag, 1); + + /* rest init as zeros */ + for (i = 1; i < lock->flags_len; i++) { + vatomic8_init(&lock->flags[i].flag, 0); + } +} +/** + * Acquires the given lock. + * + * @param lock address of arraylock_t object. + * @param slot output parameter. Stores the index of the slot associates with + * the calling thread. + * + * @note the same value returned in `slot` must be passed intact to + * `arraylock_release`. + */ +static inline void +arraylock_acquire(arraylock_t *lock, vuint32_t *slot) +{ + // we take the slot with the tail index, and increment the tail by 1 + // note that if we reached the maximum index we will go back to index 0 + // thus slot = tail % lock->size. Because of unsigned overflow, lock->size + // must be a power of two, and this was ensured in the initialization. + + vuint32_t tail = vatomic8_get_inc(&lock->tail); + *slot = tail % lock->flags_len; + + // wait until you have permission + await_while (vatomic8_read_acq(&lock->flags[*slot].flag) == 0U) {} +} +/** + * Releases the given lock. + * + * @param lock address of arraylock_t object. + * @param slot index of the slot associates with the calling thread. + */ +static inline void +arraylock_release(arraylock_t *lock, vuint32_t slot) +{ + vuint32_t next = 0; + // Give away permission + vatomic8_write_rlx(&lock->flags[slot].flag, 0); + // Calculate the index of the next in line thread's slot + next = (slot + 1) % lock->flags_len; + // Grant the permission to the next in line thread + vatomic8_write_rel(&lock->flags[next].flag, 1); +} +#endif diff --git a/include/vsync/spinlock/clhlock.h b/include/vsync/spinlock/clhlock.h new file mode 100644 index 0000000..70a8526 --- /dev/null +++ b/include/vsync/spinlock/clhlock.h @@ -0,0 +1,130 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_CLHLOCK_H +#define VSYNC_CLHLOCK_H +/******************************************************************************* + * @file clhlock.h + * @ingroup fair_lock + * @brief List-based lock attributed to Craig, Landin and Hagersten. + * + * @example + * @include eg_clhlock.c + * + * @cite + * Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 7.5] + * (https://dl.acm.org/doi/pdf/10.5555/2385452) + ******************************************************************************/ +#include +#include + +/** + * Each thread has its own qnode (`clh_qnode_t`). When `locked` is set to 1, the + * thread wants to enter the critical section. + */ +typedef struct clh_qnode_s { + vatomic32_t locked; +} VSYNC_CACHEALIGN clh_qnode_t; + +typedef struct clh_node_s { + clh_qnode_t *pred; /* points to the pred node */ + clh_qnode_t *qnode; /* points to the used qnode by the owner thread */ + clh_qnode_t _qnode; /* inital own qnode, later is used by other threads */ +} clh_node_t; + +/** The lock holds a pointer to the tail of the queue */ +typedef struct clhlock_s { + vatomicptr_t tail; + clh_qnode_t initial; +} clhlock_t; + +/** + * Initializes the given node. + * + * @param node address of clh_node_t object. + */ +static inline void +clhlock_node_init(clh_node_t *node) +{ + vatomic32_init(&node->_qnode.locked, 0); + node->qnode = &node->_qnode; + node->pred = NULL; +} +/** + * Initializes the given lock. + * + * @param lock address of clhlock_t object. + */ +static inline void +clhlock_init(clhlock_t *lock) +{ + vatomic32_init(&lock->initial.locked, 0); + vatomicptr_init(&lock->tail, &lock->initial); +} +/** + * Acquires the lock. + * + * @param lock address of clhlock_t object. + * @param node address of clh_node_t object associated with the calling thread. + * + * @note `node` has to continue to exist even if the thread dies. + */ +static inline void +clhlock_acquire(clhlock_t *lock, clh_node_t *node) +{ + // declare your intention of acquiring the lock + vatomic32_write_rlx(&node->qnode->locked, 1); + + // find out, who has acquired the lock before me + // the tail always points to the thread infront of me + // put me inline and let the tail points to me + node->pred = (clh_qnode_t *)vatomicptr_xchg(&lock->tail, node->qnode); + + // wait on my predecessor to free the lock + vatomic32_await_eq_acq(&node->pred->locked, 0); + + // I have the lock by now +} +/** + * Releases the lock. + * + * @param lock address of clhlock_t object. + * @param node address of clh_node_t object associated with the calling thread. + * + * @note It hijacks its predecessor's queue node as its own. + */ +static inline void +clhlock_release(clhlock_t *lock, clh_node_t *node) +{ + // free the lock + vatomic32_write_rel(&node->qnode->locked, 0); + + // node recycling: use your predecessor node as your own in future runs + node->qnode = node->pred; + + V_UNUSED(lock); +} +/** + * Returns whether there is a thread waiting to acquire the clhlock. + * + * This function should only be called by the current owner of the lock. + * + * @param lock address of clhlock_t object. + * @param node address of clh_node_t object associated with the calling thread. + * + * @return true if there are waiters. + * @return false if there are no waiters. + * + * @note this function is not part of the standard lock API. + */ +static inline vbool_t +clhlock_has_waiters(clhlock_t *lock, clh_node_t *node) +{ + clh_qnode_t *tail = (clh_qnode_t *)vatomicptr_read_rlx(&lock->tail); + + return tail != node->qnode; +} + +#endif /* VSYNC_CLHLOCK_H */ diff --git a/include/vsync/spinlock/cnalock.h b/include/vsync/spinlock/cnalock.h new file mode 100644 index 0000000..ecef3a9 --- /dev/null +++ b/include/vsync/spinlock/cnalock.h @@ -0,0 +1,223 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_CNA_H +#define VSYNC_CNA_H +/******************************************************************************* + * @file cnalock.h + * @ingroup fair_lock numa_aware + * @brief Compact NUMA-aware Lock. + * + * The CNA is an efficient variant of the MCS locks, which adds NUMA-awareness + * without a hierarchical approach. + * + * @example + * @include eg_cna.c + * + * @cite Dave Dice and Alex Kogan - + * [Compact NUMA-aware locks](https://dl.acm.org/doi/10.1145/3302424.3303984) + * + ******************************************************************************/ + +#include +#include +#include + +#define CNALOCK_NODE_UNSET VUINT32_MAX + +typedef struct cna_node_s { + vatomicptr_t spin; + vatomicptr(struct cna_node_s *) next; + vatomic32_t node; +} VSYNC_CACHEALIGN cna_node_t; + +typedef struct cnalock_s { + vatomicptr(struct cna_node_s *) tail; +} cnalock_t; + +/** Initializer of `cnalock_t`. */ +#define CNALOCK_INIT() \ + { \ + .tail = VATOMIC_INIT(0) \ + } + +#ifdef VSYNC_VERIFICATION +vatomic32_t rand = VATOMIC_INIT(0); +#endif + +#ifndef INITIAL_VAL + #define INITIAL_VAL (128 * 128) +#endif + +/** + * Initializes the CNA lock. + * + * @param lock address of cnalock_t object. + * @note alternatively use CNALOCK_INIT + */ +static inline void +cnalock_init(cnalock_t *lock) +{ + vatomicptr_init(&lock->tail, NULL); +} +/** + * Acquires the CNA lock. + * + * @param lock address of cnalock_t object. + * @param me address of cna_node_t object associated with the calling thread. + * @param numa_node valid id of the NUMA node where the calling thread is hosted + */ +static inline void +cnalock_acquire(cnalock_t *lock, cna_node_t *me, vuint32_t numa_node) +{ + vatomicptr_write_rlx(&me->spin, NULL); + vatomicptr_write_rlx(&me->next, 0); + vatomic32_write_rlx(&me->node, CNALOCK_NODE_UNSET); + + // Access to tail + cna_node_t *tail = vatomicptr_xchg(&lock->tail, me); + + if (!tail) { + vatomicptr_write_rlx(&me->spin, (void *)1); + return; + } + + vatomic32_write_rlx(&me->node, numa_node); + + // Access to another threads queue element -- next + vatomicptr_write_rel(&tail->next, me); + + vatomicptr_await_neq_acq(&me->spin, NULL); +} +/** + * Decides if the lock should be handed to the successor of the same NUMA. + * + * @return vuint32_t non-zero if yes. + * @return 0 otherwise. + */ +static inline vuint32_t +_cnalock_keep_lock_local(void) +{ +#ifdef VSYNC_VERIFICATION + return vatomic32_read_rlx(&rand); +#else + static vuint32_t cnt = 0; + return (cnt = (cnt + 1) % INITIAL_VAL); +#endif +} +/** + * Returns the successor of the given node `me`. + * + * @param me address of cna_node_t object. + * @param numa_node id of the NUMA node where the calling thread is hosted. + * @return cna_node_t* address of the successor. + */ +static inline cna_node_t * +_cnalock_find_successor(cna_node_t *me, vuint32_t numa_node) +{ + cna_node_t *next = vatomicptr_read_rlx(&me->next); + vuint32_t my_node = vatomic32_read_rlx(&me->node); + + if (my_node == CNALOCK_NODE_UNSET) { + my_node = numa_node; + } + + cna_node_t *sec_head = next; + cna_node_t *sec_tail = next; + + cna_node_t *cur; + + // Access to another threads queue element -- node, next + for (cur = next; cur && vatomic32_read_rlx(&cur->node) != my_node; + sec_tail = cur, cur = vatomicptr_read_acq(&cur->next)) {} + + if (!cur) { + return NULL; // No one is in the same node as me + } + + if (cur == next) { + return next; + } + + cna_node_t *spin = vatomicptr_read_rlx(&me->spin); + + if (spin > (cna_node_t *)1) { + // There is someone in the secondary queue + + // xchg is not necessary, read and write are enough + // This version was choosen to improve GenMC verification + // Access to another threads queue element -- next + cna_node_t *origSecHead = vatomicptr_xchg_rlx(&spin->next, sec_head); + sec_head = origSecHead; + } + // Access to another threads queue element -- next + vatomicptr_write_rlx(&sec_tail->next, sec_head); + vatomicptr_write_rlx(&me->spin, sec_tail); + + return cur; +} +/** + * Releases the CNA lock. + * + * @param lock address of cnalock_t object. + * @param me address of cna_node_t object associated with the calling thread. + * @param numa_node valid id of the NUMA node where the calling thread is + * hosted. + */ +static inline void +cnalock_release(cnalock_t *lock, cna_node_t *me, vuint32_t numa_node) +{ + cna_node_t *next = vatomicptr_read_acq(&me->next); + cna_node_t *spin = vatomicptr_read_rlx(&me->spin); + + if (!next) { + // There is no one in the main queue + if (spin == (void *)1) { + // There is no one in the secondary queue + // Access to tail + if (vatomicptr_cmpxchg_rel(&lock->tail, me, NULL) == me) { + return; + } + } else { + cna_node_t *sec_tail = spin; + // Access to another threads queue element -- next + cna_node_t *sec_head = vatomicptr_xchg_rlx(&sec_tail->next, NULL); + + // Access to tail + if (vatomicptr_cmpxchg_rel(&lock->tail, me, sec_tail) == me) { + // Access to another thread queue element -- spin + vatomicptr_write_rel(&sec_head->spin, (void *)1); + return; + } + // Access to another threads queue element -- spin + vatomicptr_write_rlx(&sec_tail->next, sec_head); + } + next = vatomicptr_await_neq_acq(&me->next, NULL); + } + cna_node_t *succ = NULL; + void *value = (void *)1; + + vuint32_t keep_lock = _cnalock_keep_lock_local(); + + if (keep_lock) { + succ = _cnalock_find_successor(me, numa_node); + spin = vatomicptr_read_rlx(&me->spin); + // Spin could have changed due to _cnalock_find_successor, so re-read it + } + + if (keep_lock && succ) { + value = spin; + } else if (spin > (cna_node_t *)1) { + // xchg is not necessary, read and write are enough + // This version was choosen to improve GenMC verification + // Access to another threads queue element -- next + succ = vatomicptr_xchg_rlx(&spin->next, next); + } else { + succ = next; + } + // Access to another threads queue element -- spin + vatomicptr_write_rel(&succ->spin, value); +} +#endif diff --git a/include/vsync/spinlock/hclhlock.h b/include/vsync/spinlock/hclhlock.h new file mode 100644 index 0000000..1df3536 --- /dev/null +++ b/include/vsync/spinlock/hclhlock.h @@ -0,0 +1,375 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_HCLHLOCK_H +#define VSYNC_HCLHLOCK_H + +#include +#include +#include + +/******************************************************************************* + * @file hclhlock.h + * @ingroup fair_lock numa_aware + * @brief Hierarchical CLH Queue Lock. + * + * @example + * @include eg_hclh.c + * + * @cite + * Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 7.8.2] + * (https://dl.acm.org/doi/pdf/10.5555/2385452) + ******************************************************************************/ + +/** + * @def HCLH_MAX_CLUSTERS + * @brief configures the number of available clusters in HCLH lock. + * + * Each cluster has a local queue. Each thread must be + * associated with a cluster value in `[0: HCLH_MAX_CLUSTERS -1]` + * + * A cluster can be thought of as a NUMA node or cache group. Threads + * running on cores that belong to the same cache-group or NUMA node are + * expected to be associated with the same cluster. + * + * @note by default 32 clusters are defined. Users can redefine it using: + * `-DHCLH_MAX_CLUSTERS=N`, where `N` is the number of clusters the user wishes + * to set. + * + */ +#ifndef HCLH_MAX_CLUSTERS + #define HCLH_MAX_CLUSTERS 32 +#endif + +/* + * TWS: Tail When Spliced + * MSB bits[31:31] = 1 + */ +#define HCLH_TWS_MASK 0x80000000U +#define HCLH_SET_TWS_TRUE(_s_) ((_s_) |= HCLH_TWS_MASK) +#define HCLH_SET_TWS_FALSE(_s_) ((_s_) &= (~HCLH_TWS_MASK)) +#define HCLH_IS_TWS(_s_) ((((_s_)&HCLH_TWS_MASK) >> 31U) == 1U) + +/* + * SMW: Successor Must Wait + * bits[30:30] = 1 + */ +#define HCLH_SMW_MASK 0x40000000U +#define HCLH_SET_SMW_TRUE(_s_) ((_s_) |= HCLH_SMW_MASK) +#define HCLH_SET_SMW_FALSE(_s_) ((_s_) &= (~HCLH_SMW_MASK)) +#define HCLH_IS_SMW(_s_) ((((_s_)&HCLH_SMW_MASK) >> 30U) == 1U) + +/* + * Cluster value in + * bits [29:00] + */ +#define HCLH_CLUSTER_MASK 0x3FFFFFFFU +#define HCLH_SET_CLUSTER(_s_, _c_) \ + ((_s_) = ((_s_) & (~HCLH_CLUSTER_MASK)) | (_c_)) +#define HCLH_GET_CLUSTER(_s_) ((_s_)&HCLH_CLUSTER_MASK) + +typedef struct hclh_qnode_s { + vatomic32_t + state; /* State encodes the following information + * - state[31:31]: TWS (Tail When Spliced) indicates + * whether the node is the last node in the sequence + * being spliced onto the global queue. + * - state[30:30]: SMW (Successor Must Wait) indicates + * whether the successor thread must wait. + * - state[29:00]: indicates to which cluster the thread belongs + */ +} hclh_qnode_t; + +typedef struct hclh_lock_s { + vatomicptr_t local_queues[HCLH_MAX_CLUSTERS]; /* tails of local queues, one + queue per cluster needed */ + vatomicptr_t global_queue; /* the tail of the global queue */ + hclh_qnode_t head; /* inital node in the global queue */ +} hclh_lock_t; + +typedef struct hclh_tnode_s { + hclh_qnode_t *pred; /* predecessor's qnode */ + hclh_qnode_t *qnode; /* thread's qnode */ +} hclh_tnode_t; + +/** + * Internal functions + */ +static inline void _hclhlock_qnode_hijack(hclh_tnode_t *, vuint32_t); + +static inline vuint32_t _hclh_lock_get_default_state(vuint32_t); + +static inline vbool_t _hclh_await_grant_or_cluster_master(vatomic32_t *, + vuint32_t); + +/** + * Initializes the lock. + * + * @param hclh_lock address of hclh_lock object to be initialized. + */ +static inline void +hclhlock_init(hclh_lock_t *hclh_lock) +{ + vsize_t i = 0; + + vatomic32_write_rlx(&hclh_lock->head.state, 0); + vatomicptr_write_rlx(&hclh_lock->global_queue, &hclh_lock->head); + for (i = 0; i < HCLH_MAX_CLUSTERS; i++) { + vatomicptr_write_rlx(&hclh_lock->local_queues[i], NULL); + } +} +/** + * Initializes the given `tnode`. + * + * @param tnode address of the hclh_tnode_t object associated with the calling + * thread. + * @param qnode address of the hclh_qnode_t object associated with the calling + * thread. + * @param cluster the cluster to which the calling thread belongs. + * + * @note Must be called once before the first call of `hclhlock_acquire`. + * @note `qnode` must live as long as the system is running and not be freed. + * The reason is that on release qnodes get passed to successor threads. + * @note `tnode` should live as long as the associated thread lives. + */ +static inline void +hclhlock_init_tnode(hclh_tnode_t *tnode, hclh_qnode_t *qnode, vuint32_t cluster) +{ + vuint32_t state = 0; + + ASSERT(tnode); + ASSERT(qnode); + ASSERT(cluster < HCLH_MAX_CLUSTERS); + + state = _hclh_lock_get_default_state(cluster); + vatomic32_write_rlx(&qnode->state, state); + // thread's qnode must point to the given node + tnode->qnode = qnode; + tnode->pred = NULL; +} +/** + * Acquires the lock. + * + * @param lock address of the lock to be used. + * @param tnode address of the hclh_tnode_t object associated with the calling + * thread. + * + * @note `hclhlock_init_tnode` must be called on the given `tnode`, before + * `hclhlock_acquire` is called for the first time. + */ +static inline void +hclhlock_acquire(hclh_lock_t *lock, hclh_tnode_t *tnode) +{ + vatomicptr_t *local_queue = NULL; + hclh_qnode_t *pred = NULL; + hclh_qnode_t *local_tail = NULL; + hclh_qnode_t *exp_tail = NULL; + vuint32_t state = 0; + vuint32_t cluster = 0; + vuint32_t succ_state = 0; + vbool_t granted = false; + + ASSERT(lock); + ASSERT(tnode); + + state = vatomic32_read_rlx(&tnode->qnode->state); + cluster = HCLH_GET_CLUSTER(state); + local_queue = &lock->local_queues[cluster]; + + // splice my qnode into my cluster's local queue + do { + exp_tail = pred; + pred = vatomicptr_cmpxchg(local_queue, exp_tail, tnode->qnode); + } while (pred != exp_tail); + + // if I have a predecessor I will wait till its done or I am the cluster + // master + if (pred != NULL) { + granted = _hclh_await_grant_or_cluster_master(&pred->state, cluster); + if (granted) { + tnode->pred = pred; + /* I have been granted the permission to enter the CS */ + return; + } + } + + /* + * I am the head of the queue, I am the cluster master and I am responsible + * for splicing the local queue onto the global queue + * splice my qnode into the global queue + */ + pred = NULL; + do { + local_tail = vatomicptr_read_acq(local_queue); + exp_tail = pred; + pred = vatomicptr_cmpxchg(&lock->global_queue, exp_tail, local_tail); + } while (pred != exp_tail); + + // inform successor it is the new master (the head of the local queue) + succ_state = vatomic32_read_rlx(&local_tail->state); + // set tail when spliced to true + HCLH_SET_TWS_TRUE(succ_state); + vatomic32_write_rlx(&local_tail->state, succ_state); + + /* as long as successor must wait in our pred state is set, we wait */ + state = vatomic32_read_acq(&pred->state); + while (HCLH_IS_SMW(state)) { + /* wait for the state to change */ + state = vatomic32_await_neq_acq(&pred->state, state); + } + tnode->pred = pred; +} +/** + * Releases the lock. + * + * @param tnode address of the hclh_tnode_t object associated with the calling + * thread. + */ +static inline void +hclhlock_release(hclh_tnode_t *tnode) +{ + vatomic32_t *state = NULL; + vuint32_t cur_state = 0; + vuint32_t cluster = 0; + + ASSERT(tnode); + ASSERT(tnode->qnode); + + state = &tnode->qnode->state; + + cur_state = vatomic32_read_rlx(state); + + // extract cluster + cluster = HCLH_GET_CLUSTER(cur_state); + + // set successor must wait to false + HCLH_SET_SMW_FALSE(cur_state); + + vatomic32_write_rel(state, cur_state); + + // Rest the state and take the qnode from pred + _hclhlock_qnode_hijack(tnode, cluster); +} + +/** + * Reinitializes the given state. + * + * Sets the cluster id and resets the flags + * SMW and TWS. + * + * @param state address of the state to reinitialize + * @param cluster cluster of the calling thread. + */ +static inline void +_hclhlock_qnode_hijack(hclh_tnode_t *tnode, vuint32_t cluster) +{ + vuint32_t cur_state = 0; /* current state */ + vuint32_t exp_state = 0; /* expected state */ + vuint32_t new_state = 0; /* new state */ + vatomic32_t *state = NULL; + + ASSERT(tnode); + ASSERT(tnode->pred); + ASSERT(cluster < HCLH_CLUSTER_MASK); + + state = &tnode->pred->state; + + new_state = _hclh_lock_get_default_state(cluster); + + cur_state = vatomic32_read_rlx(state); + do { + exp_state = cur_state; + cur_state = vatomic32_cmpxchg_rlx(state, exp_state, new_state); + } while (cur_state != exp_state); + + /* + * NOTE: GenMC 0.9 has a bug that does not consider the write even of the + * successful cas on hclhlock.c client when REACQUIRE is enabled A + * workaround would be to trick it with adding: + * vatomic32_write_rlx(state, new_state); + * Right now the REACQUIRE is disabled, and should be re-enabled + * after GenMC is fixed. + */ + tnode->qnode = tnode->pred; +} + +/** + * Waits on the given state. + * + * + * A blocking function that waits till either of the following is met: + * - the predecessor node is from the same cluster and TWS=SMW=0, + * - the predecessor node is from a different cluster and TWS=1. + * + * @param state address of the predecessor's state. + * @param state state of the calling thread. + * @return true pred is from same cluster, SMW=false & TWS=false. + * @return false pred is not from the same cluster, or TWS=true. + */ +static inline vbool_t +_hclh_await_grant_or_cluster_master(vatomic32_t *state, vuint32_t cluster) +{ + vuint32_t cur_state = 0; /* curr state */ + vuint32_t required_state = 0; + + ASSERT(state); + ASSERT(cluster < HCLH_CLUSTER_MASK); + /** + * Predecessor state + * Predecessor is in the same cluster and + * It Successor must wait is set to false + * and tail when spliced is set to false + */ + required_state = _hclh_lock_get_default_state(cluster); + HCLH_SET_SMW_FALSE(required_state); + + cur_state = vatomic32_read_acq(state); + + while (true) { + // check same cluster, SMW=TWS=0 + if (cur_state == required_state) { + return true; + } + // check diff cluster or TWS=1 + if (HCLH_GET_CLUSTER(cur_state) != cluster || HCLH_IS_TWS(cur_state)) { + return false; + } + // wait for a new update to occur. + cur_state = vatomic32_await_neq_acq(state, cur_state); + } +} +/** + * Returns default state. + * + * @param cluster cluster of calling thread. + * @return vuint32_t default state value. + */ +static inline vuint32_t +_hclh_lock_get_default_state(vuint32_t cluster) +{ + vuint32_t state = 0; + + ASSERT(cluster < HCLH_CLUSTER_MASK); + /* set cluster */ + HCLH_SET_CLUSTER(state, cluster); + /* set successor must wait to one */ + HCLH_SET_SMW_TRUE(state); + /* set tail when spliced to false */ + HCLH_SET_TWS_FALSE(state); + return state; +} + +#undef HCLH_TWS_MASK +#undef HCLH_SET_TWS_TRUE +#undef HCLH_SET_TWS_FALSE +#undef HCLH_IS_TWS +#undef HCLH_SMW_MASK +#undef HCLH_SET_SMW_TRUE +#undef HCLH_SET_SMW_FALSE +#undef HCLH_IS_SMW +#undef HCLH_CLUSTER_MASK +#undef HCLH_SET_CLUSTER +#undef HCLH_GET_CLUSTER +#endif diff --git a/include/vsync/spinlock/hemlock.h b/include/vsync/spinlock/hemlock.h new file mode 100644 index 0000000..86124dd --- /dev/null +++ b/include/vsync/spinlock/hemlock.h @@ -0,0 +1,136 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_HEMLOCK_H +#define VSYNC_HEMLOCK_H +/******************************************************************************* + * @file hemlock.h + * @ingroup fair_lock + * @brief Hemlock by Dice and Kogan. + * + * @example + * @include eg_hemlock.c + * + * @cite + * Dice and Kogan - [Hemlock : Compact and Scalable Mutual Exclusion] + * (https://arxiv.org/abs/2102.03863) + ******************************************************************************/ + +#include +#include + +/** Node of a thread/core for all Hemlock instances. */ +typedef struct hem_node_s { + vatomicptr(struct hemlock_s *) grant; +} VSYNC_CACHEALIGN hem_node_t; + +/** Hemlock data structure. */ +typedef struct hemlock_s { + vatomicptr(hem_node_t *) tail; +} VSYNC_CACHEALIGN hemlock_t; + +/** Initializer of `hemlock_t`. */ +#define HEMLOCK_INIT() \ + { \ + .tail = VATOMIC_INIT(0) \ + } + +/** + * Initializes the given lock object `l`. + * + * @param l address of hemlock_t object. + */ +static inline void +hemlock_init(hemlock_t *l) +{ + vatomicptr_write(&l->tail, NULL); +} +/** + * Tries to acquire the Hemlock. + * + * @param l address of hemlock_t object. + * @param node address of hem_node_t object. Associated with the calling + * thread/core. + * @return 1 on success, 0 on failure + */ +static inline int +hemlock_tryacquire(hemlock_t *l, hem_node_t *node) +{ + hem_node_t *pred = NULL; + + vatomicptr_write_rlx(&node->grant, NULL); + pred = (hem_node_t *)vatomicptr_cmpxchg_acq(&l->tail, NULL, node); + return pred == NULL; +} +/** + * Acquires the Hemlock. + * + * @param l address of hemlock_t object. + * @param node address of hem_node_t object. Associated with the calling + * thread/core. + */ +static inline void +hemlock_acquire(hemlock_t *l, hem_node_t *node) +{ + hem_node_t *pred = NULL; + + vatomicptr_write_rlx(&node->grant, NULL); + pred = (hem_node_t *)vatomicptr_xchg(&l->tail, node); + if (pred == NULL) { + return; + } + +#if !defined(HEMLOCK_USE_CTR) + vatomicptr_await_eq_acq(&pred->grant, l); + vatomicptr_write_rlx(&pred->grant, NULL); +#else + await_while (vatomicptr_cmpxchg_acq(&pred->grant, l, NULL) != l) + vatomic_cpu_pause(); +#endif +} +/** + * Releases the Hemlock. + * + * @param l address of hemlock_t object. + * @param node address of hem_node_t object. Associated with the calling + * thread/core. + */ +static inline void +hemlock_release(hemlock_t *l, hem_node_t *node) +{ + if (vatomicptr_read_rlx(&l->tail) == node && + vatomicptr_cmpxchg_rel(&l->tail, node, NULL) == node) { + return; + } + vatomicptr_write_rel(&node->grant, l); + +#if !defined(HEMLOCK_USE_CTR) + vatomicptr_await_eq_rlx(&node->grant, NULL); +#elif __SIZEOF_POINTER__ == 8 + vatomic64_t *grant = (vatomic64_t *)&node->grant; + await_while (vatomic64_get_add_rlx(grant, 0) != (vuint64_t)NULL) + vatomic_cpu_pause(); +#else + vatomic32_t *grant = (vatomic32_t *)&node->grant; + await_while (vatomic32_get_add_rlx(&node->grant, 0) != (vuint32_t)NULL) + vatomic_cpu_pause(); +#endif +} +/** + * Returns whether there is a thread waiting to acquire the Hemlock. + * + * This function should only be called by the current owner of the lock. + * + * @param l address of hemlock_t object. + * @param node address of hem_node_t object. Associated with the calling + * thread/core. + */ +static inline int +hemlock_has_waiters(hemlock_t *l, hem_node_t *node) +{ + return vatomicptr_read_rlx(&l->tail) != node; +} + +#endif diff --git a/include/vsync/spinlock/hmcslock.h b/include/vsync/spinlock/hmcslock.h new file mode 100644 index 0000000..5271a9a --- /dev/null +++ b/include/vsync/spinlock/hmcslock.h @@ -0,0 +1,422 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_HMCSLOCK_H +#define VSYNC_HMCSLOCK_H +/****************************************************************************** + * @file hmcslock.h + * @ingroup fair_lock numa_aware + * @brief Hierarchical MCS lock for systems with NUMA Hierarchies. + * + * @example + * @include eg_hmcslock.c + * + * @cite + * [High Performance Locks for Multi-level NUMA Systems] + * (https://dl.acm.org/doi/pdf/10.1145/2858788.2688503) + * + * @cite + * [Verifying and Optimizing the HMCS Lock for Arm Servers] + * (https://link.springer.com/chapter/10.1007/978-3-030-91014-3_17) + *****************************************************************************/ +#include +#include +#include +#include + +/****************************************************************************** + * MACROS/CONSTANTS + *****************************************************************************/ +/** @cond DO_NOT_DOCUMENT */ + +#define HMCLOCK_UNLOCKED 0x0UL +#define HMCLOCK_LOCKED 0x1UL +#define HMCLOCK_COHORT_START 0x1UL +#define HMCLOCK_MAX UINT64_MAX +#define HMCLOCK_ACQUIRE_PARENT (HMCLOCK_MAX - 1) +#define HMCLOCK_WAIT HMCLOCK_MAX + +#if defined(NDEBUG) + /* on release mode make all assertions void */ + #define HMCSLOCK_ASSERT(_c_) +#else + #define HMCS_ENABLE_DEBUG + #include + #define HMCSLOCK_ASSERT(_c_) ASSERT(_c_) +#endif + +/** @endcond */ + +/****************************************************************************** + * Data Types + *****************************************************************************/ +typedef struct hmcs_node_s { + vatomicptr(struct hmcs_node_s *) next; + vatomic64_t status; +} VSYNC_CACHEALIGN hmcs_node_t; + +typedef struct hmcslock_s { + vatomicptr(hmcs_node_t *) lock; + struct hmcslock_s *parent; + hmcs_node_t qnode; + vuint32_t threshold; + vbool_t is_leaf; +} VSYNC_CACHEALIGN hmcslock_t; + +typedef struct hmcs_level_spec_s { + vsize_t num_nodes_per_parent; + vuint32_t threshold; +} hmcslock_level_spec_t; +/* ***************************************************************************** + * Private functions + * ****************************************************************************/ +static inline void _hmcslock_release_real(hmcslock_t *lock, hmcs_node_t *qnode, + vsize_t depth); + +static inline void _hmcslock_acquire_real(hmcslock_t *lock, hmcs_node_t *qnode, + vsize_t depth); + +static void _hmcslock_release_helper(hmcslock_t *lock, hmcs_node_t *qnode, + vuint64_t val); + +static inline void _hmcslock_one_acquire_real(hmcslock_t *lock, + hmcs_node_t *qnode); + +static void _hmcslock_init_level(hmcslock_t *cur_parent, hmcslock_t *locks, + vsize_t *i, vsize_t cur_level, + hmcslock_level_spec_t *level_specs, + vsize_t num_levels); + +#if defined(HMCS_ENABLE_DEBUG) +static vuint32_t _hmcslock_count_nodes(hmcslock_level_spec_t *level_specs, + vsize_t num_levels); +#endif +/** + * Returns the address of the lock associated with the core id. + * + * @param locks array of hmcslock_t objects. + * @param locks_len `locks` array length. + * @param level_specs array of hmcslock_level_spec_t objects. + * @param num_levels number of the levels in heirarchy, same as the length of + * `level_specs`. + * @param cores_per_node number of cores per leaf lock node. + * @param core_id core id of the calling thread. + * + * @return hmcslock_t* address of hmcslock_t object, which the calling + * thread should use to acquire the lock. + */ +static inline hmcslock_t * +hmcslock_which_lock(hmcslock_t *locks, vsize_t locks_len, + hmcslock_level_spec_t *level_specs, vsize_t num_levels, + vuint32_t cores_per_node, vuint32_t core_id) +{ + vsize_t l = 0; + vsize_t i = 0; + vsize_t j = 0; + vsize_t leaf_node_count = 1; + vsize_t node_index = 0; + + for (l = 0; l < num_levels; l++) { + leaf_node_count *= level_specs[l].num_nodes_per_parent; + } + +#ifdef __cplusplus + hmcslock_t *leaf_nodes[HMCS_MAX_THREADS]; +#else + hmcslock_t *leaf_nodes[leaf_node_count]; +#endif + + for (i = 0, j = 0; i < locks_len; i++) { + HMCSLOCK_ASSERT(j < leaf_node_count && + "you got over the boundary of leaf nodes"); + // to map the thread to its leaf parent + if (locks[i].is_leaf) { + leaf_nodes[j++] = &locks[i]; + } + } // for + node_index = (core_id / cores_per_node) % leaf_node_count; + return leaf_nodes[node_index]; +} +/** + * Initializes the locks array. + * + * @param locks array of hmcslock_t objects. + * @param locks_len `locks` array length. + * @param level_specs array of hmcslock_level_spec_t objects. + * @param num_levels number of the levels in heirarchy, same as the length of + * `level_specs`. + * + * @note + * e.g., say you have three levels, the machine, two NUMA nodes, and two caches + * per NUMA. Then `locks_len = 1(machine) + 2(NUMA) + 4 (2*2 caches) = 7`. + * `num_levels = 3` (including the machine level) + * define levels_spec as follows: + * ```c + * level_specs[num_levels] = { + * {1, LEVEL_THRESHOLD}, // 1 machine + * {2, LEVEL_THRESHOLD}, // 2 NUMAs per machine + * {2, LEVEL_THRESHOLD}, // 2 caches per NUMA + * } + * ``` + */ +static inline void +hmcslock_init(hmcslock_t *locks, vsize_t locks_len, + hmcslock_level_spec_t *level_specs, vsize_t num_levels) +{ + vsize_t level = 0; + +#if defined(HMCS_ENABLE_DEBUG) + vuint32_t total_nodes = 0; + + HMCSLOCK_ASSERT(level_specs[0].num_nodes_per_parent == 1 && + "top level should have one node"); + + total_nodes = _hmcslock_count_nodes(level_specs, num_levels); + + HMCSLOCK_ASSERT(locks_len >= total_nodes && + "insufficient nodes, or too many nodes"); +#else + V_UNUSED(locks_len); +#endif + + _hmcslock_init_level(NULL, locks, &level, 0, level_specs, num_levels); + + HMCSLOCK_ASSERT(level == locks_len && + "recursion did not reach the expected depth"); +} +/** + * Acquires the given lock. + * + * @param lock address of hmcslock_t object. + * @param qnode address of hmcs_node_t object. + * @param num_levels number of levels including machine level. + * + * @note `lock` should be what `hmcslock_which_lock` returned. + */ +static inline void +hmcslock_acquire(hmcslock_t *lock, hmcs_node_t *qnode, vsize_t num_levels) +{ + _hmcslock_acquire_real(lock, qnode, num_levels - 1); +} +/** + * Releases the given lock. + * + * @param lock address of hmcslock_t object. + * @param qnode address of hmcs_node_t object. + * @param num_levels number of levels including machine level. + * + * @note `lock` should be what `hmcslock_which_lock` returned. + */ +static inline void +hmcslock_release(hmcslock_t *lock, hmcs_node_t *qnode, vsize_t num_levels) +{ + _hmcslock_release_real(lock, qnode, num_levels - 1); +} +/** + * Helps releasing the lock to the successor. + * + * @param lock address of hmcslock_t object. + * @param qnode address of hmcs_node_t object. + * @param val the new status of the successor. + */ +static void +_hmcslock_release_helper(hmcslock_t *lock, hmcs_node_t *qnode, vuint64_t val) +{ + hmcs_node_t *succ = (hmcs_node_t *)vatomicptr_read_acq(&qnode->next); + + if (succ == NULL) { + if (vatomicptr_cmpxchg(&lock->lock, qnode, NULL) == qnode) { + return; + } + // await till there is a successor + succ = (hmcs_node_t *)vatomicptr_await_neq_rlx(&qnode->next, NULL); + } + // update successor status + vatomic64_write_rel(&succ->status, val); +} +/** + * Releases the lock on depth zero. + * + * @param lock address of hmcslock_t object. + * @param qnode address of hmcs_node_t object. + */ +static inline void +_hmcslock_one_acquire_real(hmcslock_t *lock, hmcs_node_t *qnode) +{ + hmcs_node_t *pred = NULL; + + // Prepare the node for use. + vatomic64_write_rlx(&qnode->status, HMCLOCK_LOCKED); + vatomicptr_write_rlx(&qnode->next, NULL); + pred = (hmcs_node_t *)vatomicptr_xchg(&lock->lock, qnode); + + if (pred) { + // spin as long as it's locked + vatomicptr_write_rel(&pred->next, qnode); + vatomic64_await_neq_acq(&qnode->status, HMCLOCK_LOCKED); + } else { + vatomic64_write_rlx(&qnode->status, HMCLOCK_UNLOCKED); + } +} +/** + * Acquires the lock on `depth > 1`. + * + * Recursive function. + * + * @param lock address of hmcslock_t object. + * @param qnode address of hmcs_node_t object. + * @param depth current recursion depth. + */ +static inline void +_hmcslock_acquire_real(hmcslock_t *lock, hmcs_node_t *qnode, vsize_t depth) +{ + hmcs_node_t *pred = NULL; + vuint64_t cur_status = 0; + + // Prepare the node for use + vatomic64_write_rlx(&qnode->status, HMCLOCK_WAIT); + vatomicptr_write_rlx(&qnode->next, NULL); + pred = (hmcs_node_t *)vatomicptr_xchg(&lock->lock, qnode); + + if (pred) { + vatomicptr_write_rel(&pred->next, qnode); + // spin as long as the status is wait + cur_status = vatomic64_await_neq_acq(&qnode->status, HMCLOCK_WAIT); + + // acquired, enter the CS + if (cur_status < HMCLOCK_ACQUIRE_PARENT) { + return; + } + } + + vatomic64_write_rlx(&qnode->status, HMCLOCK_COHORT_START); + + HMCSLOCK_ASSERT(lock->parent && "parent is null"); + + if (depth == 1) { + _hmcslock_one_acquire_real(lock->parent, &lock->qnode); + } else { + _hmcslock_acquire_real(lock->parent, &lock->qnode, depth - 1); + } +} +/** + * Releases the lock on `depth > 1`. + * + * Recursive function. + * + * @param lock address of hmcslock_t object. + * @param qnode address of hmcs_node_t object. + * @param depth current recursion depth. + */ +static inline void +_hmcslock_release_real(hmcslock_t *lock, hmcs_node_t *qnode, vsize_t depth) +{ + vuint64_t cur_count = 0; + hmcs_node_t *succ = NULL; + + cur_count = vatomic64_read_rlx(&qnode->status); + HMCSLOCK_ASSERT(cur_count <= lock->threshold); + + // Lower level releases + if (cur_count < lock->threshold) { // Not reached threshold + + succ = (hmcs_node_t *)vatomicptr_read_acq(&qnode->next); + if (succ) { // pass within cohorts + vatomic64_write_rel(&succ->status, cur_count + 1); + return; + } // if succ + } // if threshold not reached + + // else: No known successor or reached threshold, release to parent + if (depth == 1) { + _hmcslock_release_helper(lock->parent, &lock->qnode, HMCLOCK_UNLOCKED); + } else { + _hmcslock_release_real(lock->parent, &lock->qnode, depth - 1); + } + + // Ask successor to acquire next level lock + _hmcslock_release_helper(lock, qnode, HMCLOCK_ACQUIRE_PARENT); +} +/** + * Builds and initializes the locks array/tree. + * + * @param cur_parent address of hmcslock_t object. The parent lock of + * `cur_level`. + * @param locks array of hmcslock_t objects. + * @param idx in/out parameters, index of the current lock. + * @param cur_level index of the current level. + * @param level_specs array of hmcslock_level_spec_t objects. + * @param num_levels number of levels including machine level. + * + * @note recursive function. + */ +static void +_hmcslock_init_level(hmcslock_t *cur_parent, hmcslock_t *locks, vsize_t *idx, + vsize_t cur_level, hmcslock_level_spec_t *level_specs, + vsize_t num_levels) +{ + vsize_t p = 0; + vsize_t cur_index = 0; + vbool_t is_leaf = false; + hmcslock_t *cur_child = NULL; + + if (num_levels == cur_level) { + return; + } + + if (cur_level + 1 == num_levels) { + is_leaf = true; + } + + for (p = 0; p < level_specs[cur_level].num_nodes_per_parent; p++) { + cur_index = (*idx)++; + cur_child = &locks[cur_index]; + cur_child->parent = cur_parent; + cur_child->threshold = level_specs[cur_level].threshold; + cur_child->is_leaf = is_leaf; + _hmcslock_init_level(cur_child, locks, idx, cur_level + 1, level_specs, + num_levels); + vatomicptr_write(&cur_child->lock, NULL); + } +} +#if defined(HMCS_ENABLE_DEBUG) +/** + * Returns the number of locks. + * + * @param level_specs array of hmcslock_level_spec_t objects. + * @param num_levels number of levels including machine level. + * @return vuint32_t + * + * @note recursive function + */ +static vuint32_t +_hmcslock_count_nodes(hmcslock_level_spec_t *level_specs, vsize_t num_levels) +{ + vsize_t i = 0; + vuint32_t nodes = 1; + + if (num_levels == 0) { + return 0; + } + + for (i = 0; i < num_levels; i++) { + nodes *= level_specs[i].num_nodes_per_parent; + } + + return nodes + _hmcslock_count_nodes(level_specs, num_levels - 1); +} +#endif + +#undef HMCLOCK_UNLOCKED +#undef HMCLOCK_LOCKED +#undef HMCLOCK_COHORT_START +#undef HMCLOCK_MAX +#undef HMCLOCK_ACQUIRE_PARENT +#undef HMCLOCK_WAIT +#undef HMCSLOCK_ASSERT +#if defined(HMCS_ENABLE_DEBUG) + #undef HMCS_ENABLE_DEBUG +#endif +#endif diff --git a/include/vsync/spinlock/rec_seqlock.h b/include/vsync/spinlock/rec_seqlock.h new file mode 100644 index 0000000..040049b --- /dev/null +++ b/include/vsync/spinlock/rec_seqlock.h @@ -0,0 +1,103 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_REC_SEQLOCK_H +#define VSYNC_REC_SEQLOCK_H +/******************************************************************************* + * @file rec_seqlock.h + * @ingroup reentrant + * @brief Recursive seqlock implementation using recursive.h + * + * In a rec_seqlock writers are only blocked by writers, not by readers. + * Readers optimistically read the shared variables and subsequently + * check their consistency. If any of the shared variables has been updated + * while being read, the readers must retry + * + * This is a reentrant implementation of @see seqlock.h. + * + * + * @example + * @include eg_rec_seqlock.c + * + ******************************************************************************/ + +#include +#include +#include +#include + +/** @cond DO_NOT_DOCUMENT */ +DEF_RECURSIVE_LOCK(rec_seqlock, seqlock_t, seqlock_init, seqlock_acquire, + seqlock_release, WITHOUT_TRYACQUIRE, WITHOUT_CONTEXT) +/** @endcond */ + +/** Initializer of `rec_seqlock`. */ +#define REC_SEQLOCK_INIT() RECURSIVE_LOCK_INIT(SEQ_LOCK_INIT()) + +/** + * Initializes the given recursive seqlock. + * + * @param l address of rec_seqlock_t object. + */ +static inline void rec_seqlock_init(rec_seqlock_t *l); +/** + * Acquires the given recursive seqlock. + * + * @param l address of rec_seqlock_t object. + * @param id thread ID or core ID. + * + * @note called by writer threads. + */ +static inline void rec_seqlock_acquire(rec_seqlock_t *l, vuint32_t id); +/** + * Releases the given recursive seqlock. + * + * @param l address of rec_seqlock_t object. + * + * @note called by writer threads. + */ +static inline void rec_seqlock_release(rec_seqlock_t *l); +/** + * Marks beginning of reader critical section. + * + * Readers must call this function, before attempting to read. * + * This function returns a value that must be later passed + * to `rec_seqlock_rend`. + * + * @post readers must call `rec_seqlock_rend` once they are done accessing the + * critical section. + * + * @param l address of rec_seqlock_t object. + * @return `seqvalue_t` an unsigned integer value. + * + * @note called by reader threads. + */ +static inline seqvalue_t +rec_seqlock_rbegin(rec_seqlock_t *l) +{ + return seqlock_rbegin(&l->lock); +} +/** + * Ends reader critical section. + * + * Users should rely on the return value to decide if repeating is necessary. + * + * @pre readers must call `rec_seqlock_rbegin` before reading critical section + * data. + * + * @param l address of rec_seqlock_t object. + * @param sv the value `rec_seqlock_rbegin` returned, before the read attempt + * @return true read data is consistent. + * @return false read data is inconsistent, reader must reattempt the read. + * + * @note called by reader threads. + */ +static inline vbool_t +rec_seqlock_rend(rec_seqlock_t *l, seqvalue_t sv) +{ + return seqlock_rend(&l->lock, sv); +} + +#endif diff --git a/include/vsync/spinlock/rwlock.h b/include/vsync/spinlock/rwlock.h index 0a1af79..dcff5d5 100644 --- a/include/vsync/spinlock/rwlock.h +++ b/include/vsync/spinlock/rwlock.h @@ -181,6 +181,7 @@ rwlock_acquired_by_writer(rwlock_t *l) static inline vbool_t rwlock_acquired_by_readers(rwlock_t *l) { - return vatomic32_read(&l->rs.s) > 0; + vuint32_t s = vatomic32_read(&l->rs.s); + return s > 0 && s != l->n; } #endif diff --git a/include/vsync/spinlock/twalock.h b/include/vsync/spinlock/twalock.h new file mode 100644 index 0000000..87c7cb2 --- /dev/null +++ b/include/vsync/spinlock/twalock.h @@ -0,0 +1,147 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_TWALOCK_H +#define VSYNC_TWALOCK_H +/******************************************************************************* + * @file twalock.h + * @ingroup fair_lock + * @brief Ticketlock with waiting array (TWA). + * + * To use the TWA lock, one must declare the global waiting array once in + * the program. Use `TWALOCK_ARRAY_DECL` to declare the array where convenient. + * + * @example + * @include eg_twalock.c + * + * @cite + * Dice and Kogan - + * [TWA - Ticket Locks Augmented with a Waiting Array, EuroPar-19] + * + ******************************************************************************/ + +#include +#include + +#define TWA_L 1U +#ifndef TWA_A + #define TWA_A 4096U +#endif +#define TWA_DIFF(a, b) ((a) - (b)) +#define TWA_HASH(t) ((t) % TWA_A) + +typedef struct twa_counter_s { + vatomic32_t val; +} VSYNC_CACHEALIGN twa_counter_t; + +/** @cond DO_NOT_DOCUMENT */ + +/** __twa_array is a waiting array shared among all TWA lock instances. To use + * the TWA lock, one must declare the array once in the program. */ +extern twa_counter_t __twa_array[TWA_A]; + +/** TWA_ARRAY(i) internal macro to access the i-th value of the array. */ +#define TWA_ARRAY(i) &__twa_array[i].val + +/** @endcond */ + +/** TWALOCK_ARRAY_DECL declares the global __twa_array variable. */ +#define TWALOCK_ARRAY_DECL twa_counter_t __twa_array[TWA_A] VSYNC_CACHEALIGN; + +typedef struct twalock_s { + vatomic32_t ticket; + vatomic32_t grant; +} twalock_t; + +/** Initializer of `twalock_t`. */ +#define TWALOCK_INIT() \ + { \ + .ticket = VATOMIC_INIT(0), .grant = VATOMIC_INIT(0) \ + } + +static inline void _twalock_acquire_slowpath(twalock_t *l, vuint32_t t); + +/** + * Initializes the given TWA lock. + * + * @param l address of twalock_t object. + * @note alternatively use TWALOCK_INIT + */ +static inline void +twalock_init(twalock_t *l) +{ + vatomic32_init(&l->ticket, 0); + vatomic32_init(&l->grant, 0); +} +/** + * Acquires the given TWA lock. + * + * @param l address of twalock_t object. + */ +static inline void +twalock_acquire(twalock_t *l) +{ + vuint32_t tx = vatomic32_get_inc_rlx(&l->ticket); + vuint32_t dx = TWA_DIFF(tx, vatomic32_read_acq(&l->grant)); + + if (dx == 0) { + return; + } + + if (dx > TWA_L) { + _twalock_acquire_slowpath(l, tx); + } + vatomic32_await_eq_acq(&l->grant, tx); +} +/** + * Releases the given TWA lock. + * + * @param l address of twalock_t object. + */ +static inline void +twalock_release(twalock_t *l) +{ + vuint32_t k = vatomic32_read_rlx(&l->grant) + 1; + vatomic32_write_rel(&l->grant, k); + vatomic32_inc_rel(TWA_ARRAY(TWA_HASH(k + TWA_L))); +} +/** + * Tries to acquire the given TWA lock. + * + * @param l address of twalock_t object. + * @return true on success. + * @return false on failure. + */ +static inline vbool_t +twalock_tryacquire(twalock_t *l) +{ + vuint32_t k = vatomic32_read_acq(&l->grant); + return vatomic32_cmpxchg_rlx(&l->ticket, k, k + 1) == k; +} +/** + * Acquires the lock via slow path. + * + * @param l address of twalock_t object. + * @param t ticket number. + */ +static inline void +_twalock_acquire_slowpath(twalock_t *l, vuint32_t t) +{ + vuint32_t at = TWA_HASH(t); + vuint32_t u = vatomic32_read_acq(TWA_ARRAY(at)); + vuint32_t k = vatomic32_read_rlx(&l->grant); + vuint32_t dx = TWA_DIFF(t, k); + + while (dx > TWA_L) { + u = vatomic32_await_neq_acq(TWA_ARRAY(at), u); + dx = TWA_DIFF(t, vatomic32_read_rlx(&l->grant)); + } +} + +#undef TWA_L +#undef TWA_DIFF +#undef TWA_HASH +#undef TWA_ARRAY +#endif diff --git a/include/vsync/stack/elimination_stack.h b/include/vsync/stack/elimination_stack.h new file mode 100644 index 0000000..411f5c9 --- /dev/null +++ b/include/vsync/stack/elimination_stack.h @@ -0,0 +1,260 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSTACK_ELIMINATION_H +#define VSTACK_ELIMINATION_H +/****************************************************************************** + * @file elimination_stack.h + * @brief Unbounded lock-free stack with elimination backoff. + * @ingroup requires_smr lock_free linearizable + * + * When an attempt to pop/push fails, the calling thread + * will attempt to rendezvous with another thread. A thread that pushes may + * exchange the node that is trying to push with a thread that pops. A + * successful exchange happens when a pusher exchanges its node with a popper + * within a limited number of trials. If the exchange does not complete within + * the number of attempts/trials specified in `VSTACK_XCHG_MAX_TRIALS` the + * normal push/pop operation is reattempted. + * + * + * ## Configuration + * - `-DVSTACK_XCHG_SLOTS_COUNT=3` + * - `-DVSTACK_XCHG_MAX_TRIALS=10000` + * + * Note that the right configuration depends on the contention, number of + * threads etc. We highly recommend you to benchmark the stack within your + * application with different configurations and choose the configuration + * resulting in the best performance. + * + * @example + * @include eg_stack_elimination.c + * + * @cite + * Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 11] + * (https://dl.acm.org/doi/pdf/10.5555/2385452) + *****************************************************************************/ +#include +#include +#include +#include +#include +#include +#include + +/******************************************************************************* + * @def VSTACK_XCHG_SLOTS_COUNT + * @brief Configures the stack to have the given number of slots. + * + * @note the default value is `-DVSTACK_XCHG_SLOTS_COUNT=3` + ******************************************************************************/ +#ifndef VSTACK_XCHG_SLOTS_COUNT + #define VSTACK_XCHG_SLOTS_COUNT 3U +#endif +/******************************************************************************* + * @def VSTACK_XCHG_MAX_TRIALS + * @brief Configures the stack to try to exchange for the given number of trials + * max. + * + * @note the default value is `-DVSTACK_XCHG_MAX_TRIALS=10000` + ******************************************************************************/ +#ifndef VSTACK_XCHG_MAX_TRIALS + #define VSTACK_XCHG_MAX_TRIALS 10000U +#endif + +/** @cond DO_NOT_DOCUMENT */ +#if defined(VSYNC_VERIFICATION) + #undef VSTACK_XCHG_MAX_TRIALS + #define VSTACK_XCHG_MAX_TRIALS 5U +#endif +/** @endcond */ + +typedef struct vstack_s { + vstack_core_t core; + backoff_rand_fun_t rand_fun; + exchanger_t elimination_array[VSTACK_XCHG_SLOTS_COUNT]; +} vstack_t; + +static inline vstack_node_t *_vstack_rendezvous(vstack_t *stack, + vstack_node_t *node, + vuint32_t max, + vbool_t *out_success); +/** + * Initializes the given `stack`. + * + * @param stack address of vstack_t object. + * @param rand_fun a function pointer to a function that generates a random + * number. + */ +static inline void +vstack_init(vstack_t *stack, backoff_rand_fun_t rand_fun) +{ + vsize_t i = 0; + ASSERT(stack); + ASSERT(rand_fun); + stack->rand_fun = rand_fun; + /* initialize stack core */ + vstack_core_init(&stack->core); + /* initialize exchanger slots */ + for (i = 0; i < VSTACK_XCHG_SLOTS_COUNT; i++) { + exchanger_reset(&stack->elimination_array[i]); + } +} +/** + * Pushes the given `node` to the top of the given `stack`. + * + * @param stack address of vstack_t object. + * @param node address of vstack_node_t object. + * @note this operation always succeeds. + */ +static inline void +vstack_push(vstack_t *stack, vstack_node_t *node) +{ + vbool_t success = false; + vuint32_t max = 0; + void *exchanged_node = NULL; + range_policy_t policy; + + ASSERT(stack); + ASSERT(node); + + /* @note in the book this is a thread local policy, however at the moment we + * use a new instance for each operation, this is because we thread local + * storage might degrade performance, benchmarking required for this. + * Determines the elimination array subrange to be used */ + range_policy_record_init(&policy, VSTACK_XCHG_SLOTS_COUNT - 1); + + while (true) { + /* try the normal path of pushing first */ + if (vstack_core_try_push(&stack->core, node)) { + /* successful push */ + return; + } else { + max = range_policy_get_range(&policy); + /* try to exchange with another some thread */ + exchanged_node = _vstack_rendezvous(stack, node, max, &success); + if (success) { + if (exchanged_node == NULL) { + /* if the returned node is NULL it means we exchanged + * with a popper */ + range_policy_record_success(&policy); + /* successful exchange */ + return; + } + /* if the exchanged node is not NULL, it means we exchanged with + * a pusher, in that case we need to reattempt the operation */ + verification_ignore(); + } else { + // we only record fail on timeout + verification_ignore(); + range_policy_record_fail(&policy); + } + } + } +} +/** + * Pops the top of the given `stack`. + * + * @param stack address of vstack_t object. + * @return address of vstack_node_t object. If the stack is not empty. + * @return NULL on empty stack. + * @note must be called inside an SMR critical section. + */ +static inline vstack_node_t * +vstack_pop(vstack_t *stack) +{ + vstack_node_t *node = NULL; + vbool_t success = false; + vbool_t pop_success = false; + vuint32_t max = 0; + range_policy_t policy; + + ASSERT(stack); + + range_policy_record_init(&policy, VSTACK_XCHG_SLOTS_COUNT - 1); + + while (true) { + node = vstack_core_try_pop(&stack->core, &pop_success); + if (pop_success) { + /* successful pop */ + break; + } else { + max = range_policy_get_range(&policy); + node = _vstack_rendezvous(stack, NULL, max, &success); + if (success) { + /* success means the exchange is complete is successful */ + if (node) { + /* if the returned node is not NULL it means we exchanged + * with a pusher */ + range_policy_record_success(&policy); + /* successful exchange */ + break; + } + /* @note: if exchanged_node is NULL this means we exchanged with + * a pop and thus we need to repeat the operation */ + verification_ignore(); + } else { + ASSERT(!node); + verification_ignore(); + /* timeout we update the policy */ + range_policy_record_fail(&policy); + } + } + } + return node; +} +/** + * Attempts to rendezvous with another thread to exchange the given `node`. + * + * @param stack address of vstack_t object + * @param node address of vstack_node_t object if the calling thread is a + * pusher, NULL otherwise + * @param max maximum index of the exchange slot that can be used in this + * exchange + * @param out_success output parameter indicating if the exchange was successful + * *out_success == true: successful exchange + * *out_success == false: failed exchange + * @return address of vstack_node_t* object. + */ +static inline vstack_node_t * +_vstack_rendezvous(vstack_t *stack, vstack_node_t *node, vuint32_t max, + vbool_t *out_success) +{ + exchanger_t *exchanger = NULL; + vstack_node_t *exchanged_node = NULL; + vsize_t slot_idx = 0; + + /* randomly pick a slot with index in [0, VSTACK_XCHG_SLOTS_COUNT] */ + slot_idx = stack->rand_fun(0, max); + ASSERT(slot_idx < VSTACK_XCHG_SLOTS_COUNT); + exchanger = &stack->elimination_array[slot_idx]; + exchanged_node = + exchanger_xchg(exchanger, node, VSTACK_XCHG_MAX_TRIALS, out_success); + return exchanged_node; +} +/** + * Pops all remaining nodes in the stack and calls `destroy` on them. + * + * @param stack address of vstack_t object. + * @param destroy function address of type vstack_node_handler_t. + * @param arg second argument of `destroy`, can be NULL. + */ +static inline void +vstack_destroy(vstack_t *stack, vstack_node_handler_t destroy, void *arg) +{ + vstack_core_destroy(&stack->core, destroy, arg); +} +/** + * Calls the given `visitor` function on each stack node. + * + * @param stack address of vstack_t object. + * @param visitor function address of type vstack_node_handler_t. + * @param arg second argument of `visitor`, can be NULL. + */ +static inline void +_vstack_visit(vstack_t *stack, vstack_node_handler_t visitor, void *arg) +{ + vstack_core_visit(&stack->core, visitor, arg); +} +#endif diff --git a/include/vsync/stack/internal/range_policy.h b/include/vsync/stack/internal/range_policy.h new file mode 100644 index 0000000..cf242b5 --- /dev/null +++ b/include/vsync/stack/internal/range_policy.h @@ -0,0 +1,77 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_RANGE_POLICY_H +#define VSYNC_RANGE_POLICY_H +#include +#include + +typedef struct range_policy_s { + vuint32_t max; + vuint32_t range; + vuint32_t successful_operations; + vuint32_t failed_operations; +} range_policy_t; + +/** + * Initializes the given `policy` + * + * @param policy + * @param max maximum allowed range + */ +static inline void +range_policy_record_init(range_policy_t *policy, vuint32_t max) +{ + ASSERT(policy); + policy->range = max; + policy->max = max; + policy->successful_operations = 0; + policy->failed_operations = 0; + ASSERT(policy->range <= policy->max); +} +/** + * Returns the current range + * + * @param policy address of `range_policy_t` object + * @return vuint32_t current range + */ +static inline vuint32_t +range_policy_get_range(range_policy_t *policy) +{ + ASSERT(policy); + return policy->range; +} +/** + * Records a successful operation, and expands the range as a result + * + * @param policy address of `range_policy_t` object + */ +static inline void +range_policy_record_success(range_policy_t *policy) +{ + ASSERT(policy); + policy->successful_operations++; + + if (policy->range < policy->max) { + policy->range++; + } +} +/** + * Records a failed operation, and shrinks the range as a result + * + * @param policy address of `range_policy_t` object + */ +static inline void +range_policy_record_fail(range_policy_t *policy) +{ + ASSERT(policy); + policy->failed_operations++; + + if (policy->range > 0) { + policy->range--; + } +} + +#endif diff --git a/include/vsync/stack/internal/stack_core.h b/include/vsync/stack/internal/stack_core.h new file mode 100644 index 0000000..22b9385 --- /dev/null +++ b/include/vsync/stack/internal/stack_core.h @@ -0,0 +1,138 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSTACK_CORE_H +#define VSTACK_CORE_H + +#include +#include +#include +#include + +typedef struct vstack_node_s { + struct vstack_node_s + *next; /* next does not need to be atomic, because it + is never changed in parallel. It is set before insertion! */ +} vstack_node_t; + +typedef struct vstack_core_s { + vatomicptr(vstack_node_t *) top; +} VSYNC_CACHEALIGN vstack_core_t; + +typedef void (*vstack_node_handler_t)(vstack_node_t *node, void *args); + +/** + * Initializes the given `stack` object. + * + * @param stack address of vstack_core_t object. + */ +static inline void +vstack_core_init(vstack_core_t *stack) +{ + ASSERT(stack); + vatomicptr_write(&stack->top, NULL); +} + +/** + * Tries to push the given `node` to the top of the given `stack`. + * + * @param stack address of vstack_core_t object. + * @param node address of vstack_core_t. + * @return true push succeeded. + * @return false push failed. + */ +static inline vbool_t +vstack_core_try_push(vstack_core_t *stack, vstack_node_t *node) +{ + vstack_node_t *top = NULL; + ASSERT(stack); + ASSERT(node); + top = vatomicptr_read(&stack->top); + node->next = top; + return vatomicptr_cmpxchg(&stack->top, top, node) == top; +} +/** + * Tries to pop the top of the given `stack`. + * + * `out_success` indicates if the operation succeeded。 + * + * @param stack address of vstack_core_t object + * @param out_success output parameter, false: pop failed, true: pop succeeded + * @return vstack_node_t* the popped object, can be NULL if the stack is empty. + * This is only valid if the operation succeeded. + */ +static inline vstack_node_t * +vstack_core_try_pop(vstack_core_t *stack, vbool_t *out_success) +{ + vstack_node_t *node = NULL; + vstack_node_t *top = NULL; + vstack_node_t *old_top = NULL; + + ASSERT(stack); + ASSERT(out_success); + top = vatomicptr_read(&stack->top); + if (top) { + node = top->next; + old_top = vatomicptr_cmpxchg(&stack->top, top, node); + *out_success = top == old_top; + return old_top; + } + *out_success = true; + return NULL; +} +/** + * Calls the given `visitor` function on each stack node. + * + * @param stack address of vstack_core_t object. + * @param visitor function address of type vstack_node_handler_t. + * @param arg second argument of `visitor`, can be NULL. + */ +static inline void +vstack_core_visit(vstack_core_t *stack, vstack_node_handler_t visitor, + void *arg) +{ + vstack_node_t *curr = NULL; + vstack_node_t *next = NULL; + + ASSERT(stack); + ASSERT(visitor); + curr = vatomicptr_read(&stack->top); + while (curr) { + next = curr->next; + visitor(curr, arg); + curr = next; + } +} +/** + * Pops all remaining nodes in the stack and calls destroy on them. + * + * @param stack address of vstack_core_t object. + * @param destroy function address of type vstack_node_handler_t. + * @param destroy_arg second argument of `destroy`, can be NULL. + */ +static inline void +vstack_core_destroy(vstack_core_t *stack, vstack_node_handler_t destroy, + void *destroy_arg) +{ + vbool_t success = false; + vstack_node_t *node = NULL; + ASSERT(stack); + ASSERT(destroy); + while (true) { + node = vstack_core_try_pop(stack, &success); + + ASSERT(success && + "are you sure all threads finished before you called " + "vstack_destroy!"); + + if (node == NULL) { + break; + } + + destroy(node, destroy_arg); + } +} + +#endif diff --git a/include/vsync/stack/xbo_stack.h b/include/vsync/stack/xbo_stack.h new file mode 100644 index 0000000..0f1e1dc --- /dev/null +++ b/include/vsync/stack/xbo_stack.h @@ -0,0 +1,137 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSTACK_XBO_H +#define VSTACK_XBO_H +/****************************************************************************** + * @file xbo_stack.h + * @brief Unbounded lock-free stack with exponential backoff. + * @ingroup requires_smr lock_free linearizable + * + * When an attempt to pop/push fails, the calling thread is + * put to sleep before it reattempts the operation. The sleep duration is + * determined randomly and it may not exceed the current limit, which doubles on + * each backoff. The current limit cannot exceed `max_backoff` passed to + * `vstack_init`. + * + * @example + * @include eg_stack_xbo.c + * + * @cite + * Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 11] + * (https://dl.acm.org/doi/pdf/10.5555/2385452) + *****************************************************************************/ +#include +#include +#include +#include +#include + +typedef struct vstack_s { + vstack_core_t core; + backoff_t backoff; +} vstack_t; + +/** + * Initializes the given `stack`. + * + * @param stack address of vstack_t object. + * @param vstack_usleep address of `usleep` like function. + * @param rand_fun a function pointer to a function that generates a random + * number. + * @param min_backoff minimum amount of microseconds a thread is allowed to + * sleep. + * @param max_backoff maximum allowed amount of microseconds to sleep. + */ +static inline void +vstack_init(vstack_t *stack, backoff_usleep_fun_t vstack_usleep, + backoff_rand_fun_t rand_fun, vuint32_t min_backoff, + vuint32_t max_backoff) +{ + ASSERT(stack); + ASSERT(vstack_usleep); + ASSERT(rand_fun); + ASSERT(min_backoff <= max_backoff); + + /* init stack core */ + vstack_core_init(&stack->core); + + /* init exponential backoff object */ + xbobackoff_init(&stack->backoff, min_backoff, max_backoff, vstack_usleep, + rand_fun); +} +/** + * Pushes the given `node` to the top of the given `stack`. + * + * @param stack address of vstack_t object. + * @param node address of vstack_node_t object. + * @note this operation always succeeds. + */ +static inline void +vstack_push(vstack_t *stack, vstack_node_t *node) +{ + ASSERT(stack); + ASSERT(node); + + while (true) { + if (vstack_core_try_push(&stack->core, node)) { + return; + } else { + verification_ignore(); + xbobackoff_backoff(&stack->backoff); + } + } +} +/** + * Pops the top of the given `stack`. + * + * @param stack address of vstack_t object. + * @return address of vstack_node_t object. If the stack is not empty. + * @return NULL on empty stack. + * @note must be called inside an SMR critical section. + */ +static inline vstack_node_t * +vstack_pop(vstack_t *stack) +{ + vbool_t success = false; + vstack_node_t *node = NULL; + + ASSERT(stack); + + while (true) { + node = vstack_core_try_pop(&stack->core, &success); + if (success) { + return node; + } else { + verification_ignore(); + xbobackoff_backoff(&stack->backoff); + } + } +} +/** + * Pops all remaining nodes in the stack and calls `destroy` on them. + * + * @param stack address of vstack_t object. + * @param destroy function address of type vstack_node_handler_t. + * @param arg second argument of `destroy`, can be NULL. + */ +static inline void +vstack_destroy(vstack_t *stack, vstack_node_handler_t destroy, void *arg) +{ + vstack_core_destroy(&stack->core, destroy, arg); +} +/** + * Calls the given `visitor` function on each stack node. + * + * @param stack address of vstack_t object. + * @param visitor function address of type vstack_node_handler_t. + * @param arg second argument of `visitor`, can be NULL. + */ +static inline void +_vstack_visit(vstack_t *stack, vstack_node_handler_t visitor, void *arg) +{ + vstack_core_visit(&stack->core, visitor, arg); +} +#endif diff --git a/include/vsync/thread/cond.h b/include/vsync/thread/cond.h new file mode 100644 index 0000000..3cbd924 --- /dev/null +++ b/include/vsync/thread/cond.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VTHREAD_CND_H +#define VTHREAD_CND_H +/******************************************************************************* + * @file cond.h + * @brief Condition variable + * + * A very simple condition variable. + * + * @example + * @include eg_cond.c + * + * @note include mutex.h from libvsync before including cond.h. Alternatively, + * users can implement the same interface with pthread_mutex_t or similar and + * include that to be used by cond.h. + * + * @cite [Condition variable with futex] + * (https://www.remlab.net/op/futex-condvar.shtml) + * + ******************************************************************************/ + +#include +#include +#include + +typedef struct vcond_s { + vatomic32_t value; +} vcond_t; + +/** + * Initializes the given condition variable. + * + * @param c address of vcond_t object. + */ +static inline void +vcond_init(vcond_t *c) +{ + vatomic32_init(&c->value, 0); +} +/** + * Waits on the given condition variable. + * + * Releases the mutex and waits till the condition variable is signaled, then + * reacquires the mutex. + * + * @param c address of vcond_t object. + * @param m address of vmutex_t object. + */ +static inline void +vcond_wait(vcond_t *c, vmutex_t *m) +{ + vuint32_t val = vatomic32_read_rlx(&c->value); + vmutex_release(m); + vfutex_wait(&c->value, val); + vmutex_acquire(m); +} +/** + * Signals the condition variable. + * + * Wakes up one sleeping thread waiting on the condition. + * + * @param c address of vcond_t object. + */ +static inline void +vcond_signal(vcond_t *c) +{ + vatomic32_inc_rlx(&c->value); + vfutex_wake(&c->value, FUTEX_WAKE_ONE); +} + +#endif diff --git a/include/vsync/thread/internal/futex.h b/include/vsync/thread/internal/futex.h new file mode 100644 index 0000000..32d5b9c --- /dev/null +++ b/include/vsync/thread/internal/futex.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VTHREAD_FUTEX_H +#define VTHREAD_FUTEX_H +/******************************************************************************* + * futex syscalls + * + * If the macros are not defined, we mock the futex syscall with a simple + * spinning mechanism. + ******************************************************************************/ +#include +#include + +#define FUTEX_WAKE_ALL INT_MAX +#define FUTEX_WAKE_ONE 1 + +#if defined(VSYNC_VERIFICATION) || defined(FUTEX_USERSPACE) + +static vatomic32_t signal; + +static inline void +vfutex_wait(vatomic32_t *m, vuint32_t v) +{ + vuint32_t s = vatomic32_read_acq(&signal); + if (vatomic32_read_rlx(m) != v) + return; + vatomic32_await_neq_rlx(&signal, s); +} + +static inline void +vfutex_wake(vatomic32_t *m, vuint32_t v) +{ + vatomic32_inc_rel(&signal); +} + +#elif defined(__linux__) + + #ifndef _GNU_SOURCE + #define _GNU_SOURCE + #endif /* _GNU_SOURCE */ + #include + #include + #include + #include + #include + #include + #include + +static inline long +_vfutex_call(int *uaddr, int futex_op, int val) +{ + return syscall(SYS_futex, uaddr, futex_op, val, NULL, NULL, 0); +} + +static inline void +vfutex_wait(vatomic32_t *m, vuint32_t v) +{ + long s = 0; + s = _vfutex_call((int *)m, FUTEX_WAIT, (int)v); + + if (s == -1 && errno == EAGAIN) { + return; + } + + if (s == -1 && errno != EAGAIN) { + perror("futex_wait failed"); + exit(EXIT_FAILURE); + } +} + +static inline void +vfutex_wake(vatomic32_t *m, vuint32_t nthreads) +{ + long s = 0; + s = _vfutex_call((int *)m, FUTEX_WAKE, (int)nthreads); + + if (s == -1) { + perror("futex_wake failed"); + exit(EXIT_FAILURE); + } +} + +#else /* expect user to provide the interface */ + +void vfutex_wait(vatomic32_t *m, vuint32_t v); +void vfutex_wake(vatomic32_t *m, vuint32_t v); + +#endif + +#endif /* VTHREAD_FUTEX_H */ diff --git a/include/vsync/thread/mutex.h b/include/vsync/thread/mutex.h new file mode 100644 index 0000000..013ba39 --- /dev/null +++ b/include/vsync/thread/mutex.h @@ -0,0 +1,21 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VTHREAD_MUTEX_H +#define VTHREAD_MUTEX_H +/******************************************************************************* + * @file mutex.h + * @brief Futex-based mutex + * + * This file includes the default mutex implementation. See @ref mutex/slim.h + * for details. + * + * @example + * @include eg_mutex.c + ******************************************************************************/ + +#include + +#endif diff --git a/include/vsync/thread/mutex/musl.h b/include/vsync/thread/mutex/musl.h new file mode 100644 index 0000000..3b6670f --- /dev/null +++ b/include/vsync/thread/mutex/musl.h @@ -0,0 +1,101 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VTHREAD_MUTEX_MUSL_H +#define VTHREAD_MUTEX_MUSL_H + +/******************************************************************************* + * @file musl.h + * @brief A simplified version of the mutex algorithm in musl libc. + * + * @example + * @include eg_mutex.c + * + * @note replace `#include ` with + * `#include ` in the example above. + * + * @cite [Check mutex implementation in libc] + * (http://musl.libc.org) + * + * @note It is the normal mutex without support for reentrance. + ******************************************************************************/ + +#include +#include + +/** + * @def MUSL_MAX_SPIN + * @brief times of spinning before going to the macro. + * + * default value is 100, compile with -DMUSL_MAX_SPIN=N to + * overwrite the default. + * + * @note spinning is deactivated on verification. + */ +#ifndef MUSL_MAX_SPIN + #define MUSL_MAX_SPIN 100 +#endif + +typedef struct { + vatomic32_t lock; + vatomic32_t waiters; +} vmutex_t; + +/** + * Initializes the mutex `m`. + * + * @param m address of vmutex_t object. + */ +static inline void +vmutex_init(vmutex_t *m) +{ + vatomic32_init(&m->lock, 0U); + vatomic32_init(&m->waiters, 0U); +} +/** + * Acquires the mutex `m`. + * + * @param m address of vmutex_t object. + */ +static inline void +vmutex_acquire(vmutex_t *m) +{ + if (vatomic32_cmpxchg_acq(&m->lock, 0U, 1U) == 0U) { + return; + } + vatomic_fence_rlx(); // a_barrier of a_cas failure + +#if !defined(VSYNC_VERIFICATION) + vint32_t spin = MUSL_MAX_SPIN; + while (spin-- && vatomic32_read_rlx(&m->lock) && + !vatomic32_read_rlx(&m->waiters)) { + vatomic_cpu_pause(); + } +#endif + + while (vatomic32_cmpxchg_acq(&m->lock, 0U, 1U) != 0U) { + vatomic_fence_rlx(); // a_cas failure + vatomic32_inc_rlx(&m->waiters); + if (vatomic32_cmpxchg_rlx(&m->lock, 1U, 2U) != 1U) { + vatomic_fence_rlx(); // a_cas failure + } + vfutex_wait(&m->lock, 2U); + vatomic32_dec_rlx(&m->waiters); + } +} +/** + * Releases the mutex `m`. + * + * @param m address of vmutex_t object. + */ +static inline void +vmutex_release(vmutex_t *m) +{ + vuint32_t old = vatomic32_xchg_rel(&m->lock, 0U); + if (vatomic32_read_rlx(&m->waiters) > 0U || old != 1U) { + vfutex_wake(&m->lock, FUTEX_WAKE_ONE); + } +} +#endif diff --git a/include/vsync/thread/mutex/slim.h b/include/vsync/thread/mutex/slim.h new file mode 100644 index 0000000..42eb3b4 --- /dev/null +++ b/include/vsync/thread/mutex/slim.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VTHREAD_MUTEX_SLIM_H +#define VTHREAD_MUTEX_SLIM_H +/******************************************************************************* + * @file slim.h + * @brief Slim 3-state futex. + * + * @example + * @include eg_mutex.c + * + * @note replace `#include ` with + * `#include ` in the example above. + * + ******************************************************************************/ + +#include +#include + +typedef vatomic32_t vmutex_t; + +/** + * Initializes the mutex `m`. + * + * @param m address of vmutex_t object. + */ +static inline void +vmutex_init(vmutex_t *m) +{ + vatomic32_init(m, 0); +} +/** + * Acquires the mutex `m`. + * + * @param m address of vmutex_t object. + */ +static inline void +vmutex_acquire(vmutex_t *m) +{ + if (vatomic32_cmpxchg_acq(m, 0U, 1U) == 0U) { + return; + } + + while (vatomic32_xchg_acq(m, 2U) != 0) { + vfutex_wait(m, 2U); + } +} +/** + * Releases the mutex `m`. + * + * @param m address of vmutex_t object. + */ +static inline void +vmutex_release(vmutex_t *m) +{ + if (vatomic32_xchg_rel(m, 0U) == 1U) { + return; + } + vfutex_wake(m, FUTEX_WAKE_ONE); +} +#endif diff --git a/include/vsync/thread/mutex/tristate.h b/include/vsync/thread/mutex/tristate.h new file mode 100644 index 0000000..9dcb685 --- /dev/null +++ b/include/vsync/thread/mutex/tristate.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VTHREAD_MUTEX_3STATE_H +#define VTHREAD_MUTEX_3STATE_H +/******************************************************************************* + * @file tristate.h + * @brief 3-state mutex. + * + * @example + * @include eg_mutex.c + * + * @note replace `#include ` with + * `#include ` in the example above. + * + * @cite [Ulrich Drepper - Futexes Are Tricky] + * (https://cis.temple.edu/~ingargio/old/cis307s07/readings/futex.pdf) + * + ******************************************************************************/ + +#include +#include + +typedef vatomic32_t vmutex_t; + +/** + * Initializes the mutex `m`. + * + * @param m address of vmutex_t object. + */ +static inline void +vmutex_init(vmutex_t *m) +{ + vatomic32_init(m, 0U); +} +/** + * Acquires the mutex `m`. + * + * @param m address of vmutex_t object. + */ +static inline void +vmutex_acquire(vmutex_t *m) +{ + if (vatomic32_cmpxchg_acq(m, 0, 1U) == 0U) { + return; + } + + do { + vatomic32_cmpxchg_rlx(m, 1U, 2U); + vfutex_wait(m, 2U); + } while (vatomic32_cmpxchg_acq(m, 0U, 2U) != 0U); +} +/** + * Releases the mutex `m`. + * + * @param m address of vmutex_t object. + */ +static inline void +vmutex_release(vmutex_t *m) +{ + if (vatomic32_cmpxchg_rel(m, 1U, 0U) == 1U) { + return; + } + + vatomic32_write_rel(m, 0U); + vfutex_wake(m, FUTEX_WAKE_ONE); +} + +#endif diff --git a/include/vsync/thread/once.h b/include/vsync/thread/once.h new file mode 100644 index 0000000..47dcde3 --- /dev/null +++ b/include/vsync/thread/once.h @@ -0,0 +1,82 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VTHREAD_ONCE_H +#define VTHREAD_ONCE_H +/******************************************************************************* + * @file once.h + * @brief One-time initializer. + * + * Calls a callback exactly once even if concurrently called. Callback happens + * before any thread returns vonce_call(). + * + * @example + * @include eg_once.c + * + * @cite [One-time initializer](https://www.remlab.net/op/futex-misc.shtml) + * + ******************************************************************************/ +#include +#include + +typedef vatomic32_t vonce_t; + +#define VONCE_INIT() VATOMIC_INIT(0) +#define VONCE_ZERO 0U +#define VONCE_CALL 1U +#define VONCE_DONE 2U + +// vonce_cb takes some argument and returns a pointer +typedef void *(*vonce_cb)(void *arg); + +#ifdef VSYNC_VERIFICATION +void *__once_verification_cb(void *); +#endif + +/** + * Calls `cb(arg)` once. + * + * The thread that actually executed the callback gets the return value of the + * callback. Other threads get NULL back. + * + * @param o address of once_t object. + * @param cb address of callback function. + * @param arg argument of `cb`. + * @return void* return value of `cb(arg)` if the thread actually called `cb`. + * @return NULL if another thread already called `cb`. + */ +static inline void * +vonce_call(vonce_t *o, vonce_cb cb, void *arg) +{ + void *ret = NULL; + vuint32_t state = VONCE_ZERO; + + if (vatomic32_read_acq(o) == VONCE_DONE) { + return NULL; + } + + state = vatomic32_cmpxchg_acq(o, VONCE_ZERO, VONCE_CALL); + if (state == VONCE_ZERO) { +#ifdef VSYNC_VERIFICATION + ret = __once_verification_cb(arg); +#else + ret = cb(arg); +#endif + vatomic32_write_rel(o, VONCE_DONE); + vfutex_wake(o, FUTEX_WAKE_ALL); + return ret; + } + + while (state == VONCE_CALL) { + vfutex_wait(o, 1); + state = vatomic32_read_acq(o); + } + return ret; +} + +#undef VONCE_ZERO +#undef VONCE_CALL +#undef VONCE_DONE +#endif diff --git a/include/vsync/utils/backoff.h b/include/vsync/utils/backoff.h new file mode 100644 index 0000000..1ff11df --- /dev/null +++ b/include/vsync/utils/backoff.h @@ -0,0 +1,100 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_BACKOFF_H +#define VSYNC_BACKOFF_H + +/******************************************************************************* + * @file backoff.h + * @brief Implements exponential backoff + * + * @cite + * Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 7.4] + * (https://dl.acm.org/doi/pdf/10.5555/2385452) + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include + +/** + * A function pointer to a function that sleeps for the given microseconds. + * + * @param microseconds amount of sleep in microseconds. + * @return 0, on no error. + * @return -1, on error. + * + * @note Use usleep on platforms where `usleep` is available. + */ +typedef int (*backoff_usleep_fun_t)(vuint32_t microseconds); +/** + * A function pointer to a function that generates a random number. + * + * The generated random number should be in `[min, max]`. + * + * @param min lower bound of the random number. + * @param max upper bound of the random number. + * @return the generated random number in `[min, max]`. + */ +typedef vuint32_t (*backoff_rand_fun_t)(vuint32_t min, vuint32_t max); + +typedef struct backoff_s { + vuint32_t max_delay; + /* has to be atomic to avoid data-race and undefined behavior + * @note in java defining this as atomic is not necessary + */ + vatomic32_t limit; + backoff_usleep_fun_t usleep_fun; + backoff_rand_fun_t rand_fun; +} backoff_t; +/** + * Initializes the given exponential `backoff` object. + * + * @param backoff address of backoff_t object. + * @param min minimum amount of microseconds a thread is allowed to + * sleep. + * @param max maximum allowed amount of microseconds to sleep. + * @param usleep_fun function pointer to a function that sleeps for the given + * microseconds. + * @param rand_fun function pointer to a function that generates a random + * number. + */ +static inline void +xbobackoff_init(backoff_t *backoff, vuint32_t min, vuint32_t max, + backoff_usleep_fun_t usleep_fun, backoff_rand_fun_t rand_fun) +{ + ASSERT(backoff); + ASSERT(usleep_fun); + ASSERT(min < max); + + backoff->max_delay = max; + backoff->usleep_fun = usleep_fun; + backoff->rand_fun = rand_fun; + + vatomic32_write_rlx(&backoff->limit, min); +} +/** + * Puts the calling thread to sleep for a random number of microseconds. + * + * @note the sleep cannot exceed the given `max` in `xbobackoff_init`. + * @param backoff address of backoff_t object + */ +static inline void +xbobackoff_backoff(backoff_t *backoff) +{ + vuint32_t limit = vatomic32_read_rlx(&backoff->limit); + vuint32_t delay = backoff->rand_fun(0, limit); + vuint32_t new_limit = VMIN(backoff->max_delay, (2 * limit)); + + /* Update limit */ + vatomic32_write_rlx(&backoff->limit, new_limit); + + /* sleep in microseconds */ + backoff->usleep_fun(delay); +} +#endif diff --git a/include/vsync/utils/exchanger.h b/include/vsync/utils/exchanger.h new file mode 100644 index 0000000..87c4887 --- /dev/null +++ b/include/vsync/utils/exchanger.h @@ -0,0 +1,249 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef VSYNC_EXCHANGER_H +#define VSYNC_EXCHANGER_H +#include +#include +#include +#include +/******************************************************************************* + * @file exchanger.h + * @brief Lock-free object exchanger + * + * Using the lock-free exchanger two threads can rendezvous to exchange objects + * The exchange protocol is lock-free and it is deployed in lock-free stack + * implementations. + * + * @cite + * Maurice Herlihy, Nir Shavit - [The Art of Multiprocessor Programming 11.4.1] + * (https://dl.acm.org/doi/pdf/10.5555/2385452) + ******************************************************************************/ +typedef struct exchanger_s { + vatomicptr_t slot; +} exchanger_t; + +typedef enum exchanger_slot_state_s { + SLOT_STATE_EMPTY = 1, + SLOT_STATE_WAITING = 2, + SLOT_STATE_BUSY = 3 +} exchanger_slot_state_t; + +/** + * The following code is used only for debugging purposes and is not part of + * production code. + * + */ +#ifdef VEXCHANGER_DBG + #include +#else +static inline void +ghost_set_check_point(vsize_t ck) +{ + V_UNUSED(ck); +} +#endif + +#define XCHG_CK_0 0U +#define XCHG_CK_1 1U +#define XCHG_CK_2 2U +#define XCHG_CK_3 3U +#define XCHG_CK_4 4U + +/** + * Private functions + */ +static inline void *_exchanger_slot_get(exchanger_t *exchanger, + exchanger_slot_state_t *out_slot_state); +static inline vbool_t +_exchanger_slot_cmpxchg(exchanger_t *exchanger, void *expected_addr, + exchanger_slot_state_t expected_state, void *new_addr, + exchanger_slot_state_t new_state); +/** + * Resets the given exchanger object to initial state. + * + * @note use it to initialize the object. + * @param exchanger address of exchanger_t object. + */ +static inline void +exchanger_reset(exchanger_t *exchanger) +{ + vatomicptr_write(&exchanger->slot, (void *)(vuintptr_t)SLOT_STATE_EMPTY); +} +/** + * Attempts to exchange the given object for a specified number of trials. + * + * When Two threads rendezvous via this function, they will atomically exchange + * the provided pointers/objects. To increase the probability of two threads + * meeting at the same time, this thread will repeatedly try to find another + * thread for a specified number of trials. + * + * @param exchanger address of exchanger_t object. + * @param my_item address of the object to exchange. Can be NULL，if the calling + * thread wants to only consume. + * @param max_trials the maximum number of exchange trials. + * @param success output parameter. + * true: exchange was successful. + * false: exchange failed within the given number of trials. + * @return void* address of exchanged object on successful exchange. + * @return NULL if no exchange could be conducted within `max_trials` bound. + */ +static inline void * +exchanger_xchg(exchanger_t *exchanger, void *my_item, vuint32_t max_trials, + vbool_t *success) +{ + exchanger_slot_state_t slot_state = SLOT_STATE_EMPTY; + void *your_item = NULL; + vuint32_t trial = 0; + + while (true) { + if (trial >= max_trials) { + /* exceeded the allowed exchange duration, exchange failed! */ + *success = false; + ghost_set_check_point(XCHG_CK_0); + verification_ignore(); + return NULL; + } + trial++; + /* Read the slot state and the object address residing in it */ + your_item = _exchanger_slot_get(exchanger, &slot_state); + /* Inspect the slot state */ + switch (slot_state) { + case SLOT_STATE_EMPTY: + /* the slot is empty no one using it, attempt to place your + * object and switch the status to waiting */ + if (_exchanger_slot_cmpxchg(exchanger, your_item, + SLOT_STATE_EMPTY, my_item, + SLOT_STATE_WAITING)) { + /* spin as long as within time bound waiting for time to + * elapse or a thread to consume the object you placed */ + while (trial < max_trials) { + /* as long as within time bound pull the slot_state */ + your_item = _exchanger_slot_get(exchanger, &slot_state); + /* if state is busy it means another thread consumed our + * object and placed his */ + if (slot_state == SLOT_STATE_BUSY) { + /* exchange succeeded, we can reset the slot now for + * others to use */ + exchanger_reset(exchanger); + *success = true; + ghost_set_check_point(XCHG_CK_1); + return your_item; + } + trial++; + } + /* time bound exceeded no one showed up, we try to reset the + * state, we need to use cas because someone could show up + * this second and switch the state to busy */ + if (_exchanger_slot_cmpxchg(exchanger, my_item, + SLOT_STATE_WAITING, NULL, + SLOT_STATE_EMPTY)) { + /* we managed to abort, no one showed up. The exchange + * failed */ + *success = false; + ghost_set_check_point(XCHG_CK_2); + return NULL; + } else { + /* luckily someone showed up last minute and completed + * the exchange, we consume his object */ + your_item = _exchanger_slot_get(exchanger, &slot_state); + ASSERT(slot_state == SLOT_STATE_BUSY); + + /* exchange succeeded, we reset the slot for someone + * else to use */ + exchanger_reset(exchanger); + *success = true; + ghost_set_check_point(XCHG_CK_3); + return your_item; + } + } + break; + /* we found the slot in waiting state, someone already placed an + * object for an exchange */ + case SLOT_STATE_WAITING: + /* we try to consume the object by setting the state to BUSY, + * because someone else can be competing we use cas. */ + if (_exchanger_slot_cmpxchg(exchanger, your_item, + SLOT_STATE_WAITING, my_item, + SLOT_STATE_BUSY)) { + /* we were the fasted it is all ours, exchange succeeded */ + *success = true; + ghost_set_check_point(XCHG_CK_4); + return your_item; + } + /* hard luck! someone else beat us to it, we have to try again + */ + break; + case SLOT_STATE_BUSY: + /* exchange between two threads is ongoing, we try again */ + break; + default: + ASSERT(0 && "Unknown STATE!"); + } + } +} +/** + * Returns the object and the state of the given `exchanger`'s slot. + * + * @param exchanger address of exchanger_t object. + * @param out_slot_state output parameter the state of the slot. + * @return void* the object address occupying the slot. + */ +static inline void * +_exchanger_slot_get(exchanger_t *exchanger, + exchanger_slot_state_t *out_slot_state) +{ + void *ptr = vatomicptr_read(&exchanger->slot); + + *out_slot_state = + (exchanger_slot_state_t)(((vuintptr_t)ptr) & ((vuintptr_t)0x3)); + + return V_ATOMICPTR_MARKABLE_GET_ADDRESS(ptr); +} +/** + * Compares and exchanges the slot state and object. + * + * @note this is an atomic operation. + * + * @param exchanger address of exchanger_t object. + * @param expected_addr expected address of object currently occupying the slot. + * @param expected_state expected slot state. + * @param new_addr address of the new object to place. + * @param new_state the new state of the slot. + * + * @return true compare exchange succeeded. + * @return false compare exchange failed. + */ +static inline vbool_t +_exchanger_slot_cmpxchg(exchanger_t *exchanger, void *expected_addr, + exchanger_slot_state_t expected_state, void *new_addr, + exchanger_slot_state_t new_state) +{ + vbool_t success = false; + + ASSERT(V_ATOMICPTR_MARKABLE_IS_ALIGNED(expected_addr)); + ASSERT(V_ATOMICPTR_MARKABLE_IS_ALIGNED(new_addr)); + + void *exp_val = V_ATOMICPTR_MARKABLE_COMBINE_ADDRESS_MARK(expected_addr, + expected_state); + void *new_val = + V_ATOMICPTR_MARKABLE_COMBINE_ADDRESS_MARK(new_addr, new_state); + + void *old_val = vatomicptr_cmpxchg(&exchanger->slot, exp_val, new_val); + + success = exp_val == old_val; + + DBG_BLUE("Exchange %u to %u %s", expected_state, new_state, + success ? "succeeded" : "failed"); + + return success; +} + +#undef XCHG_CK_0 +#undef XCHG_CK_1 +#undef XCHG_CK_2 +#undef XCHG_CK_3 +#undef XCHG_CK_4 +#endif diff --git a/include/vsync/utils/math.h b/include/vsync/utils/math.h index 36c8d74..63492e1 100644 --- a/include/vsync/utils/math.h +++ b/include/vsync/utils/math.h @@ -82,7 +82,12 @@ v_pow2_round_down(vuint32_t v) * */ #ifndef VMAX - #define VMAX(_a_, _b_) ((_a_) > (_b_) ? (_a_) : (_b_)) + #define VMAX(_a_, _b_) \ + ({ \ + __typeof__(_a_) _a = (_a_); \ + __typeof__(_b_) _b = (_b_); \ + _a > _b ? _a : _b; \ + }) #endif /** diff --git a/scripts/cmake-format.sh b/scripts/cmake-format.sh new file mode 100755 index 0000000..9b062ad --- /dev/null +++ b/scripts/cmake-format.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +set -e +set -x + +if [ $# -lt 1 ]; then + echo "Usage: $0 [PATH ...]" + echo "" + echo " PATH one or more directories (or source files) to recursively run " + echo " cmake-format on." + echo "Environment variables:" + echo " SILENT=true disable git diff and error code" + echo " STYLE=FILE FILE is a configuration file, default = .cmake-format" + echo " " + echo "" + exit 1 +fi + +if [ -z "${STYLE}" ]; then + STYLE=".cmake-format" +fi + +if [ -z "${IGNORE}" ]; then + IGNORE=.cmake-format.ignore +fi + +# check if the ignore file really exists otherwise do not use it +if [ -f $IGNORE ]; then + # Apply cmake-format to all committed CMake files in the repo. + git ls-files "$@" | + grep -E 'CMakeLists.txt$|.*\.cmake(\.in)?$' | + grep -E --invert-match -f "${IGNORE}" | + xargs cmake-format -c "${STYLE}" -i +else + echo -e "\e[33mWarning: $IGNORE does not exist!\e[0m" + git ls-files "$@" | + grep -E 'CMakeLists.txt$|.*\.cmake(\.in)?$' | + xargs cmake-format -c "${STYLE}" -i +fi + +if [ "${SILENT}" != "true" ]; then + # Display changed files and exit with 1 if there were differences. + git --no-pager diff --exit-code +fi diff --git a/template/include/vsync/atomic/await.h.in b/template/include/vsync/atomic/await.h.in index 32cf43a..4fc5011 100644 --- a/template/include/vsync/atomic/await.h.in +++ b/template/include/vsync/atomic/await.h.in @@ -38,11 +38,10 @@ * * The following example waits for the pointer me->next to be equal to pred. * Once the condition is met, write NULL in me->next. The variable next contains - * the value that satisfied the condition. The operation has an release - * barrier. + * the value that satisfied the condition. The operation has a release barrier. * - * ``` - * node_t *next = vatomicptr_await_eq_set_acq(me->next, pred, NULL); + * ```c + * node_t *next = vatomicptr_await_eq_set_rel(me->next, pred, NULL); * ``` * * ### Return value diff --git a/test/bitmap/CMakeLists.txt b/test/bitmap/CMakeLists.txt new file mode 100644 index 0000000..fffc833 --- /dev/null +++ b/test/bitmap/CMakeLists.txt @@ -0,0 +1,7 @@ +file(GLOB SRCS *.c) +foreach(SRC ${SRCS}) + get_filename_component(TEST ${SRC} NAME_WE) + add_executable(${TEST} ${SRC}) + target_link_libraries(${TEST} vsync pthread) + v_add_bin_test(NAME ${TEST} COMMAND ${TEST}) +endforeach() diff --git a/test/bitmap/test_mt_bitmap.c b/test/bitmap/test_mt_bitmap.c new file mode 100644 index 0000000..332f65b --- /dev/null +++ b/test/bitmap/test_mt_bitmap.c @@ -0,0 +1,71 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +#define N 12U +#define IT (VUINT64_WIDTH + 1U) +#define BIT_COUNT (N * IT) + +vbitmap_t *g_bitmap; + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + vsize_t bit_idx = 0; + vbool_t set = false; + + for (vsize_t i = 0; i < IT; i++) { + bit_idx = (tid * IT) + i; + set = vbitmap_get(g_bitmap, bit_idx); + ASSERT(set == false); + if (VIS_EVEN(bit_idx)) { + vbitmap_set_bit(g_bitmap, bit_idx); + set = vbitmap_get(g_bitmap, bit_idx); + ASSERT(set == true); + } else { + vbitmap_clr_bit(g_bitmap, bit_idx); + set = vbitmap_get(g_bitmap, bit_idx); + ASSERT(set == false); + } + } + + return 0; +} + +int +main(void) +{ + vbool_t set = false; + vsize_t size = 0; + vbitmap_iter_t iter = {0}; + vsize_t idx = 0; + vsize_t i = 0; + + size = vbitmap_size(BIT_COUNT); + g_bitmap = malloc(size); + vbitmap_init(g_bitmap, BIT_COUNT, false); + launch_threads(N, run); + + for (i = 0; i < BIT_COUNT; i++) { + set = vbitmap_get(g_bitmap, i); + ASSERT(set == VIS_EVEN(i)); + } + + vbitmap_iter_init(g_bitmap, &iter); + + i = 0; + while (vbitmap_iter_next(&iter, &idx)) { + ASSERT(i == idx); + i += 2; + } + ASSERT(i == BIT_COUNT); + + free(g_bitmap); + return 0; +} diff --git a/test/bitmap/test_mt_ranges_bitmap.c b/test/bitmap/test_mt_ranges_bitmap.c new file mode 100644 index 0000000..d84144e --- /dev/null +++ b/test/bitmap/test_mt_ranges_bitmap.c @@ -0,0 +1,90 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +#define N 12U +#define BIT_COUNT (92274688UL) +#define PER_THREAD (BIT_COUNT / N) +#define REMAINDER (BIT_COUNT % N) + +#define RANGES_LEN 3 +vsize_t g_clr_ranges[RANGES_LEN][2] = {{5, 8}, + {64, 67}, + {BIT_COUNT - 1, BIT_COUNT - 1}}; + +vbitmap_t *g_bitmap; + +vbool_t +_in_clr_range(vsize_t idx) +{ + for (vsize_t i = 0; i < RANGES_LEN; i++) { + if (idx >= g_clr_ranges[i][0] && idx <= g_clr_ranges[i][1]) { + return true; + } + } + return false; +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + vsize_t num_bits = PER_THREAD; + + vsize_t from = tid * num_bits; + vsize_t to = from + num_bits + (tid == (N - 1) ? REMAINDER - 1 : 0); + + if (tid != N - 1) { + to += 1; // let ranges intersect + } + + DBG_BLUE("T%zu: handles [%zu, %zu]", tid, from, to); + + if (to >= BIT_COUNT) { + DBG_RED("T%zu, %zu %lu", tid, to, BIT_COUNT); + ASSERT(to < BIT_COUNT); + } + + vbitmap_set_range(g_bitmap, from, to); + for (vsize_t i = from; i <= to; i++) { + vbool_t set = vbitmap_get(g_bitmap, i); + if (!_in_clr_range(i)) { + ASSERT(set); + } + } + + for (vsize_t i = 0; i < RANGES_LEN; i++) { + vbitmap_clr_range(g_bitmap, g_clr_ranges[i][0], g_clr_ranges[i][1]); + } + + + return 0; +} + +int +main(void) +{ + vsize_t size = 0; + size = vbitmap_size(BIT_COUNT); + g_bitmap = malloc(size); + vbitmap_init(g_bitmap, BIT_COUNT, false); + + launch_threads(N, run); + + for (vsize_t i = 0; i < BIT_COUNT; i++) { + vbool_t set = vbitmap_get(g_bitmap, i); + if (_in_clr_range(i)) { + ASSERT(!set); + } else { + ASSERT(set); + } + } + + free(g_bitmap); + return 0; +} diff --git a/test/bitmap/test_ut_bitmap.c b/test/bitmap/test_ut_bitmap.c new file mode 100644 index 0000000..d5dd0d9 --- /dev/null +++ b/test/bitmap/test_ut_bitmap.c @@ -0,0 +1,328 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include + +#define VBITMAP_ENTRY_COUNT 1000U +#define VBITMAP_ENTRY_COUNT_MAX 92274688U + +vbitmap_t * +_create(vsize_t len) +{ + vsize_t size = vbitmap_size(len); + vbitmap_t *bitmap = (vbitmap_t *)malloc(size); + vbitmap_init(bitmap, len, false); + return bitmap; +} + +void +_destroy(vbitmap_t *bitmap) +{ + free(bitmap); +} + +void +ut_iter_set(void) +{ + vbitmap_iter_t iter = {0}; + vsize_t len = (VUINT64_WIDTH * 2) + 2; + vsize_t size = vbitmap_size(len); + vbitmap_t *bitmap = (vbitmap_t *)malloc(size); + vsize_t idx = 0; + vsize_t expected = 0; + + vbitmap_init(bitmap, len, true); + vbitmap_iter_init(bitmap, &iter); + + + while (vbitmap_iter_next(&iter, &idx)) { + ASSERT(expected++ == idx); + ASSERT(idx < len); + } + vbool_t set = vbitmap_iter_next(&iter, &idx); + ASSERT(!set); + ASSERT(idx == len - 1); + free(bitmap); +} + +void +ut_set_range_funs(void) +{ + vsize_t len = VBITMAP_ENTRY_COUNT_MAX; + vbitmap_t *bitmap = NULL; + vsize_t from_idx = 1; + vsize_t to_idx = len - 2; + vuint64_t val = 0; + + bitmap = _create(len); + vbitmap_set_range(bitmap, from_idx, to_idx); + + val = vatomic64_read(&bitmap->maps[0]); + + ASSERT(val == VUINT64_MAX - 1); + + for (vsize_t i = 1; i < bitmap->maps_cnt - 1; i++) { + vuint64_t val = vatomic64_read(&bitmap->maps[i]); + ASSERT(val == VUINT64_MAX); + } + + ASSERT(vatomic64_read(&bitmap->maps[bitmap->maps_cnt - 1]) == + ((VUINT64_MAX << 1U) >> 1U)); + + _destroy(bitmap); +} + +void +ut_range_1slot_funs(void) +{ + vsize_t len = VUINT64_WIDTH; + vbitmap_t *bitmap = NULL; + bitmap = _create(len); + +#define PAIRS 8U + vuint64_t indexes[PAIRS][3] = { + {0, 63, VUINT64_MAX}, {1, 63, 0xFFFFFFFFFFFFFFFEUL}, + {0, 62, 0x7FFFFFFFFFFFFFFFUL}, {1, 62, 0x7FFFFFFFFFFFFFFEUL}, + {31, 32, 0x180000000UL}, {4, 60, 0x1FFFFFFFFFFFFFF0UL}, + {63, 63, 0x8000000000000000UL}, {0, 0, 0x0000000000000001UL}}; + + for (vsize_t i = 0; i < PAIRS; i++) { + vsize_t from = indexes[i][0]; + vsize_t to = indexes[i][1]; + vuint64_t expected = indexes[i][2]; + + vatomic64_write(&bitmap->maps[0], 0); + + vbitmap_set_range(bitmap, from, to); + + vuint64_t val = vatomic64_read(&bitmap->maps[0]); + + DBG_GRAY("[TC-%zu] expected: %lx == real %lx", i, expected, val); + ASSERT(expected == val); + } + + for (vsize_t i = 0; i < PAIRS; i++) { + vsize_t from = indexes[i][0]; + vsize_t to = indexes[i][1]; + vuint64_t expected = ~indexes[i][2]; + + vatomic64_write(&bitmap->maps[0], VUINT64_MAX); + + vbitmap_clr_range(bitmap, from, to); + + vuint64_t val = vatomic64_read(&bitmap->maps[0]); + + DBG_GRAY("[TC-%zu] expected: %lx == real %lx", i, expected, val); + ASSERT(expected == val); + } + + _destroy(bitmap); +} + +void +ut_range_multi_slot_funs(vsize_t len, vsize_t from, vsize_t to) +{ + vbitmap_t *bitmap = NULL; + bitmap = _create(len); + vbool_t set = false; + + vbitmap_set_range(bitmap, from, to); + + for (vsize_t i = 0; i < len; i++) { + set = vbitmap_get(bitmap, i); + if (i >= from && i <= to) { + ASSERT(set); + } else { + ASSERT(!set); + } + } + + vbitmap_set_range(bitmap, 0, from); + vbitmap_set_range(bitmap, from, len - 1); + vbitmap_clr_range(bitmap, from, to); + + for (vsize_t i = 0; i < len; i++) { + set = vbitmap_get(bitmap, i); + if (i >= from && i <= to) { + ASSERT(!set); + } else { + ASSERT(set); + } + } + + + _destroy(bitmap); +} + +void +ut_ranges(void) +{ + ut_range_multi_slot_funs(1, 0, 0); + ut_range_multi_slot_funs(VUINT64_WIDTH, 1, VUINT64_WIDTH - 1); + ut_range_multi_slot_funs((VUINT64_WIDTH * 3) + 1, 24, (VUINT64_WIDTH * 3)); +} + +void +ut_basic_funs(void) +{ + bool is_set = false; + vbitmap_t *bitmap = _create(VBITMAP_ENTRY_COUNT); + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + is_set = vbitmap_get(bitmap, i); + ASSERT(is_set == false); + } + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + vbitmap_set_bit(bitmap, i); + is_set = vbitmap_get(bitmap, i); + ASSERT(is_set == true); + } + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + is_set = vbitmap_get(bitmap, i); + ASSERT(is_set == true); + } + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + vbitmap_clr_bit(bitmap, i); + is_set = vbitmap_get(bitmap, i); + ASSERT(is_set == false); + } + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + is_set = vbitmap_get(bitmap, i); + ASSERT(is_set == false); + } + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + if (VIS_EVEN(i)) { + vbitmap_set_bit(bitmap, i); + } + } + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + is_set = vbitmap_get(bitmap, i); + ASSERT(is_set == VIS_EVEN(i)); + } + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + if (VIS_EVEN(i)) { + vbitmap_clr_bit(bitmap, i); + } else { + vbitmap_set_bit(bitmap, i); + } + } + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + is_set = vbitmap_get(bitmap, i); + ASSERT(is_set == VIS_ODD(i)); + } + + _destroy(bitmap); +} + +void +ut_test_iterator_all_set(void) +{ + vbitmap_iter_t iter; + size_t bit_idx = 0; + size_t idx = 0; + + vbitmap_t *bitmap = _create(VBITMAP_ENTRY_COUNT); + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + vbitmap_set_bit(bitmap, i); + } + + vbitmap_iter_init(bitmap, &iter); + + while (vbitmap_iter_next(&iter, &bit_idx)) { + ASSERT(bit_idx == idx); + idx++; + } + + _destroy(bitmap); +} + +void +ut_test_iterator_all_clr(void) +{ + vbitmap_iter_t iter; + size_t bit_idx = 0; + + vbitmap_t *bitmap = _create(VBITMAP_ENTRY_COUNT); + + vbitmap_iter_init(bitmap, &iter); + + ASSERT(vbitmap_iter_next(&iter, &bit_idx) == false); + + _destroy(bitmap); +} + +void +ut_test_iterator_partial_set_even(void) +{ + vbitmap_iter_t iter; + size_t bit_idx = 0; + size_t idx = 0; + + vbitmap_t *bitmap = _create(VBITMAP_ENTRY_COUNT); + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + if (VIS_EVEN(i)) { + vbitmap_set_bit(bitmap, i); + } + } + + vbitmap_iter_init(bitmap, &iter); + + while (vbitmap_iter_next(&iter, &bit_idx)) { + ASSERT(bit_idx == idx); + idx += 2; + } + _destroy(bitmap); +} + +void +ut_test_iterator_partial_set_odd(void) +{ + vbitmap_iter_t iter; + size_t bit_idx = 0; + size_t idx = 1; + + vbitmap_t *bitmap = _create(VBITMAP_ENTRY_COUNT); + + for (size_t i = 0; i < VBITMAP_ENTRY_COUNT; i++) { + if (VIS_ODD(i)) { + vbitmap_set_bit(bitmap, i); + } + } + + vbitmap_iter_init(bitmap, &iter); + + while (vbitmap_iter_next(&iter, &bit_idx)) { + ASSERT(bit_idx == idx); + idx += 2; + } + _destroy(bitmap); +} +int +main(void) +{ + ut_basic_funs(); + ut_test_iterator_all_set(); + ut_test_iterator_all_clr(); + ut_test_iterator_partial_set_even(); + ut_test_iterator_partial_set_odd(); + ut_set_range_funs(); + ut_range_1slot_funs(); + ut_ranges(); + ut_iter_set(); +} diff --git a/test/simpleht/CMakeLists.txt b/test/simpleht/CMakeLists.txt new file mode 100644 index 0000000..bf7a90d --- /dev/null +++ b/test/simpleht/CMakeLists.txt @@ -0,0 +1,43 @@ +ProcessorCount(PCOUNT) + +include_directories(include) + +file(GLOB TEST_FILES *.c) + +set(NTHREADS ${PCOUNT}) + +set(TEST_DEFS TST_IT=10000) + +set(ALGOS simple) + +foreach(test_path IN ITEMS ${TEST_FILES}) + foreach(algo IN ITEMS ${ALGOS}) + # extract test_name with extension + get_filename_component(test_name ${test_path} NAME) + + # name without extension + get_filename_component(test_case ${test_path} NAME_WE) + + set(TEST_CASE ${test_case}_${algo}) + + string(TOLOWER ${TEST_CASE} TEST_CASE) + + add_executable(${TEST_CASE} ${test_name}) + + target_link_libraries(${TEST_CASE} vsync pthread) + + # add defines + target_compile_definitions(${TEST_CASE} PUBLIC ${TEST_DEFS} ${algo}) + + v_add_bin_test_advanced( + NAME + ${TEST_CASE} + COMMAND + ${TEST_CASE} + TIMEOUT + 3600 + PROCESSORS + 4) + + endforeach() +endforeach() diff --git a/test/simpleht/mt_test.c b/test/simpleht/mt_test.c new file mode 100644 index 0000000..ef2a6f5 --- /dev/null +++ b/test/simpleht/mt_test.c @@ -0,0 +1,52 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#define NTHREADS 24U + +#include +#include +#include + +#define MIN_KEY 1U +#define MAX_KEY VSIMPLE_HT_CAPACITY +#define TID MAIN_TID + + +void +run_operation(vsize_t tid, vuintptr_t key) +{ + data_t *data = NULL; + data = imap_get(tid, key); + if (data) { + imap_rem(tid, key); + } else { + imap_add(tid, key, key); + } +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + + imap_reg(tid); + + for (vuintptr_t key = MIN_KEY; key <= MAX_KEY; key++) { + run_operation(tid, key); + } + + imap_dereg(tid); + + return NULL; +} + +int +main(void) +{ + imap_init(); + launch_threads(NTHREADS, run); + imap_destroy(); + return 0; +} diff --git a/test/simpleht/mt_test_no_rem.c b/test/simpleht/mt_test_no_rem.c new file mode 100644 index 0000000..1b634ce --- /dev/null +++ b/test/simpleht/mt_test_no_rem.c @@ -0,0 +1,53 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +#define NTHREADS 24U +#define VSIMPLEHT_DISABLE_REMOVE + +#include +#include +#include + +#define MIN_KEY 1U +#define MAX_KEY VSIMPLE_HT_CAPACITY +#define TID MAIN_TID + +vatomicsz_t g_cnt = VATOMIC_INIT(0); + + +void +run_operation(vsize_t tid, vuintptr_t key) +{ + vbool_t added = false; + data_t *data = imap_get(tid, key); + if (!data) { + added = imap_add(tid, key, key); + if (added) { + vatomicsz_inc_rlx(&g_cnt); + } + } +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + for (vuintptr_t key = MIN_KEY; key <= MAX_KEY; key++) { + run_operation(tid, key); + } + return NULL; +} + +int +main(void) +{ + imap_init(); + launch_threads(NTHREADS, run); + imap_destroy(); + vsize_t cnt = vatomicsz_read(&g_cnt); + ASSERT(cnt == VSIMPLE_HT_CAPACITY); + return 0; +} diff --git a/test/simpleht/ut_test.c b/test/simpleht/ut_test.c new file mode 100644 index 0000000..c02a6da --- /dev/null +++ b/test/simpleht/ut_test.c @@ -0,0 +1,95 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#define NTHREADS 1U +#include +#include + +#define MIN_KEY 1U +#define MAX_KEY VSIMPLE_HT_CAPACITY +#define TID MAIN_TID + +void +ut_insert_full(void) +{ + vuintptr_t key = 0; + vbool_t success = false; + vsize_t cnt = 0; + + for (key = MIN_KEY; key <= MAX_KEY; key++) { + success = imap_add(TID, key, key); + ASSERT(success); + cnt++; + } + ASSERT(cnt == VSIMPLE_HT_CAPACITY); + DBG_YELLOW("Inserted %zu entries", cnt); +} + +void +ut_get_full(void) +{ + vuintptr_t key = 0; + data_t *data = NULL; + + for (key = MIN_KEY; key <= MAX_KEY; key++) { + data = imap_get(TID, key); + ASSERT(data); + ASSERT(data->key == key); + ASSERT(data->val == key); + } + DBG_YELLOW("Found all inserted entries"); +} + +void +ut_rem_insert(void) +{ + data_t *data = NULL; + vbool_t success = false; + + success = imap_rem(TID, MAX_KEY); + ASSERT(success); + data = imap_get(TID, MAX_KEY); + ASSERT(data == NULL); + success = imap_add(TID, MAX_KEY, MAX_KEY); + ASSERT(success); + data = imap_get(TID, MAX_KEY); + ASSERT(data); + ASSERT(data->key == MAX_KEY); + ASSERT(data->val == MAX_KEY); +} + +void +ut_rem_full(void) +{ + vuintptr_t key = 0; + vbool_t success = false; + vsize_t cnt = 0; + + for (key = MIN_KEY; key <= MAX_KEY; key++) { + success = imap_rem(TID, key); + ASSERT(success); + cnt++; + } + ASSERT(cnt == VSIMPLE_HT_CAPACITY); + DBG_YELLOW("Removed %zu entries", cnt); +} + +int +main(void) +{ + imap_init(); + imap_reg(TID); + + ut_insert_full(); + imap_print(); + ut_get_full(); + ut_rem_insert(); + ut_rem_full(); + imap_print(); + + imap_dereg(TID); + imap_destroy(); + return 0; +} diff --git a/test/spinlock/CMakeLists.txt b/test/spinlock/CMakeLists.txt index 6f9fc66..1b00c7e 100644 --- a/test/spinlock/CMakeLists.txt +++ b/test/spinlock/CMakeLists.txt @@ -43,25 +43,28 @@ math(EXPR ARRAY_LOCK_BOUND "${ARRAY_LOCK_LEN} + 1") set(CFLAGS_arraylock -DARRAY_LOCK_LEN=${ARRAY_LOCK_LEN}) # Enable some tests with VMM and set extra options for Dartagnan set(DAT3M_BOUND_arraylock ${ARRAY_LOCK_BOUND}) -set(DAT3M_BOUND_caslock 4) -set(DAT3M_BOUND_clhlock 4) +set(DAT3M_BOUND_caslock 1) +set(DAT3M_BOUND_clhlock 1) if(${VSYNC_VERIFICATION_QUICK}) set(DAT3M_BOUND_cnalock 5) else() set(DAT3M_BOUND_cnalock 6) endif() set(DAT3M_BOUND_hclhlock_2t 4) -set(DAT3M_BOUND_hemlock 5) -set(DAT3M_BOUND_mcslock 4) -set(DAT3M_BOUND_rec_seqlock 5) -set(DAT3M_BOUND_rec_spinlock 4) -set(DAT3M_BOUND_rec_ticketlock 4) -set(DAT3M_BOUND_rwlock 5) -set(DAT3M_BOUND_semaphore 5) -set(DAT3M_BOUND_seqcount 4) -set(DAT3M_BOUND_seqlock 5) -set(DAT3M_BOUND_ticketlock 4) -set(DAT3M_BOUND_ttaslock 4) +set(DAT3M_BOUND_hemlock 1) +set(DAT3M_BOUND_mcslock 1) +set(DAT3M_BOUND_rec_mcslock 1) +set(DAT3M_BOUND_rec_seqlock 3) +set(DAT3M_BOUND_rec_spinlock 1) +set(DAT3M_BOUND_rec_ticketlock 1) +set(DAT3M_BOUND_rwlock 3) +set(DAT3M_BOUND_semaphore 3) +set(DAT3M_BOUND_seqcount 1) +set(DAT3M_BOUND_seqlock 3) +set(DAT3M_BOUND_ticketlock 1) +set(DAT3M_BOUND_ttaslock 3) +set(CFLAGS_twalock -DTWA_A=128) +set(DAT3M_BOUND_twalock 2) foreach(SRC ${SRCS}) get_filename_component(TEST ${SRC} NAME_WE) diff --git a/test/spinlock/arraylock.c b/test/spinlock/arraylock.c new file mode 100644 index 0000000..c5f1042 --- /dev/null +++ b/test/spinlock/arraylock.c @@ -0,0 +1,43 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#define WITH_INIT + +#define REACQUIRE 1 + +#include +#include + +#ifndef ARRAY_LOCK_LEN + #define ARRAY_LOCK_LEN 4U +#endif +arraylock_flag_t flags[ARRAY_LOCK_LEN]; +arraylock_t lock; +static __thread vuint32_t slot; + +V_STATIC_ASSERT(NTHREADS <= ARRAY_LOCK_LEN, + "ARRAY_LOCK_LEN must be a power of two greater than NTHREADS. " + "You can overwrite it with `-DARRAY_LOCK_LEN=N`"); + +// Static initialisation causes problems to Dartagnan for this benchmark +void +init(void) +{ + arraylock_init(&lock, flags, ARRAY_LOCK_LEN); +} + +void +acquire(vuint32_t tid) +{ + V_UNUSED(tid); + arraylock_acquire(&lock, &slot); +} + +void +release(vuint32_t tid) +{ + V_UNUSED(tid); + arraylock_release(&lock, slot); +} diff --git a/test/spinlock/clhlock.c b/test/spinlock/clhlock.c new file mode 100644 index 0000000..62258b5 --- /dev/null +++ b/test/spinlock/clhlock.c @@ -0,0 +1,38 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#define REACQUIRE 1 +#define WITH_INIT + +#define NULL ((void *)(0)) +#include +#include + +clhlock_t lock; +clh_qnode_t initial; +clh_node_t node[NTHREADS]; + +void +init(void) +{ + clhlock_init(&lock); + + verification_loop_bound(NTHREADS + 1); + for (int i = 0; i < NTHREADS; i++) { + clhlock_node_init(&node[i]); + } +} + +void +acquire(vuint32_t tid) +{ + clhlock_acquire(&lock, &node[tid]); +} + +void +release(vuint32_t tid) +{ + clhlock_release(&lock, &node[tid]); +} diff --git a/test/spinlock/cnalock.c b/test/spinlock/cnalock.c new file mode 100644 index 0000000..930e537 --- /dev/null +++ b/test/spinlock/cnalock.c @@ -0,0 +1,40 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifdef VSYNC_VERIFICATION_QUICK + #define NTHREADS 4 +#else + #define NTHREADS 5 +#endif + +#define WITH_POST + +#include +#include + +cnalock_t lock = CNALOCK_INIT(); +struct cna_node_s nodes[NTHREADS]; + +void +post(void) +{ +#ifdef VSYNC_VERIFICATION + vatomic32_write_rlx(&rand, 1); +#endif +} + +#define NUMA(tid) (tid < NTHREADS / 2) + +void +acquire(vuint32_t tid) +{ + cnalock_acquire(&lock, &nodes[tid], NUMA(tid)); +} + +void +release(vuint32_t tid) +{ + cnalock_release(&lock, &nodes[tid], NUMA(tid)); +} diff --git a/test/spinlock/hclhlock.c b/test/spinlock/hclhlock.c new file mode 100644 index 0000000..9691be4 --- /dev/null +++ b/test/spinlock/hclhlock.c @@ -0,0 +1,37 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#define HCLH_MAX_CLUSTERS 2 +#define REACQUIRE 1 +#define WITH_INIT + +#include +#include + +hclh_lock_t lock; +hclh_tnode_t tnode[NTHREADS]; +hclh_qnode_t qnode[NTHREADS]; + +void +init(void) +{ + hclhlock_init(&lock); + for (vsize_t i = 0; i < NTHREADS; i++) { + vuint32_t cluster = i % HCLH_MAX_CLUSTERS; + hclhlock_init_tnode(&tnode[i], &qnode[i], cluster); + } +} + +void +acquire(vuint32_t tid) +{ + hclhlock_acquire(&lock, &tnode[tid]); +} + +void +release(vuint32_t tid) +{ + hclhlock_release(&tnode[tid]); +} diff --git a/test/spinlock/hemlock.c b/test/spinlock/hemlock.c new file mode 100644 index 0000000..3d83f57 --- /dev/null +++ b/test/spinlock/hemlock.c @@ -0,0 +1,39 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifdef VSYNC_VERIFICATION_QUICK + #define REACQUIRE 1 + #define NTHREADS 3 +#else + #define REACQUIRE 1 + #define NTHREADS 4 +#endif + +#include +#include + +hemlock_t lock = HEMLOCK_INIT(); +struct hem_node_s nodes[NTHREADS]; + +void +acquire(vuint32_t tid) +{ + if (tid == NTHREADS - 1) { +#if defined(VSYNC_VERIFICATION_DAT3M) + vbool_t acquired = hemlock_tryacquire(&lock, &nodes[tid]); + verification_assume(acquired); +#else + await_while (!hemlock_tryacquire(&lock, &nodes[tid])) {} +#endif + } else { + hemlock_acquire(&lock, &nodes[tid]); + } +} + +void +release(vuint32_t tid) +{ + hemlock_release(&lock, &nodes[tid]); +} diff --git a/test/spinlock/hmcslock.c b/test/spinlock/hmcslock.c new file mode 100644 index 0000000..294e73e --- /dev/null +++ b/test/spinlock/hmcslock.c @@ -0,0 +1,70 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#define REACQUIRE 1 +#define WITH_INIT +#define MAX_THREADS 4 + +#if NTHREADS > MAX_THREADS + #error NTHREADS > MAX_THREADS +#endif + +#include +#include + +#define NUM_LEVELS 3 +#define LEVEL_1 1 +#define LEVEL_2 2 +#define LEVEL_3 2 + +#define LEVEL_1_THRESHOLD 1 +#define LEVEL_2_THRESHOLD 1 +#define LEVEL_3_THRESHOLD 1 + +#define NUM_LOCKS \ + ((LEVEL_3 * LEVEL_2 * LEVEL_1) + (LEVEL_1 * LEVEL_2) + (LEVEL_1)) + +typedef struct { + hmcslock_t hmcs_locks[NUM_LOCKS]; // array of hmcs locks + hmcslock_t *leaf_locks[MAX_THREADS]; +} lock_t; + +typedef struct { + hmcs_node_t qnode; +} ctx_t; + +lock_t lock; +ctx_t ctx[NTHREADS]; + +void +init(void) +{ + hmcslock_level_spec_t level_specs[NUM_LEVELS] = { + {LEVEL_1, LEVEL_1_THRESHOLD}, + {LEVEL_2, LEVEL_2_THRESHOLD}, + {LEVEL_3, LEVEL_3_THRESHOLD}, + }; + + hmcslock_init(lock.hmcs_locks, NUM_LOCKS, level_specs, NUM_LEVELS); + + for (vuint32_t i = 0; i < MAX_THREADS; i++) { + lock.leaf_locks[i] = hmcslock_which_lock( + lock.hmcs_locks, NUM_LOCKS, level_specs, NUM_LEVELS, LEVEL_3, i); + } +} + +void +acquire(vuint32_t tid) +{ + const vuint32_t coreid = tid; + hmcslock_acquire(lock.leaf_locks[coreid], &(ctx[tid].qnode), NUM_LEVELS); +} + +void +release(vuint32_t tid) +{ + const vuint32_t coreid = tid; + hmcslock_release(lock.leaf_locks[coreid], &(ctx[tid].qnode), NUM_LEVELS); +} diff --git a/test/spinlock/rec_seqlock.c b/test/spinlock/rec_seqlock.c new file mode 100644 index 0000000..d2cd243 --- /dev/null +++ b/test/spinlock/rec_seqlock.c @@ -0,0 +1,83 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +#ifdef VSYNC_VERIFICATION_QUICK + #define NREADERS 2 + #define NWRITERS 2 +#else + #define NREADERS 1 + #define NWRITERS 3 +#endif +#define REENTRY_COUNT 2 +#define EXPECTED_FINAL_VALUE NWRITERS +#define WITH_CS +#define WITH_FINI +#define WITH_INIT + +#include + +/* This lock is both a reader-writer lock and recursive (reentrant), it uses + * the reader_writer boilerplate, but also makes sure to acquire the lock + * multiple times */ +#include + +rec_seqlock_t lock; + +vuint32_t g_cs_x; +vuint32_t g_cs_y; + +void +writer_cs(vuint32_t tid) +{ + vsize_t i = 0; + + verification_loop_bound(REENTRY_COUNT + 1); + for (i = 0; i < REENTRY_COUNT; i++) { + rec_seqlock_acquire(&lock, tid); + } + g_cs_x++; + g_cs_y++; + + verification_loop_bound(REENTRY_COUNT + 1); + for (i = 0; i < REENTRY_COUNT; i++) { + rec_seqlock_release(&lock); + } +} + +void +reader_cs(vuint32_t tid) +{ + vuint32_t a = 0; + vuint32_t b = 0; + seqvalue_t s = 0; + + await_do { + s = rec_seqlock_rbegin(&lock); + a = g_cs_x; + b = g_cs_y; + } + while_await(!rec_seqlock_rend(&lock, s)); + + ASSERT(a == b); + ASSERT((s % 2) == 0); + V_UNUSED(tid, a, b); +} + +void +init(void) +{ + rec_seqlock_init(&lock); +} + +void +fini(void) +{ + vuint32_t x = g_cs_x; + vuint32_t y = g_cs_y; + ASSERT(x == y); + ASSERT(x == EXPECTED_FINAL_VALUE); + V_UNUSED(x, y); +} diff --git a/test/spinlock/twalock.c b/test/spinlock/twalock.c new file mode 100644 index 0000000..e526c56 --- /dev/null +++ b/test/spinlock/twalock.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include + +TWALOCK_ARRAY_DECL; +twalock_t lock = TWALOCK_INIT(); + +void +acquire(vuint32_t tid) +{ + if (tid == NTHREADS - 1) { + await_while (!twalock_tryacquire(&lock)) {} + } else { + twalock_acquire(&lock); + } +} + +void +release(vuint32_t tid) +{ + V_UNUSED(tid); + twalock_release(&lock); +} diff --git a/test/stack/CMakeLists.txt b/test/stack/CMakeLists.txt new file mode 100644 index 0000000..4fb943b --- /dev/null +++ b/test/stack/CMakeLists.txt @@ -0,0 +1,87 @@ +# detect test files +file(GLOB TEST_FILES test*.c) + +# find the number of processors +ProcessorCount(PCOUNT) + +if(${LIBVSYNC_CROSS_TESTS}) + # QEMU might be too slow, we cannot really stress test + set(NUM_THREADS 4) + set(ITERATIONS 10) +else() + math(EXPR NUM_THREADS "${PCOUNT}*2") + if(${VSYNCER_CHECK_FULL}) + set(ITERATIONS 10000) + else() + set(ITERATIONS 1000) + endif() +endif() + +# count the main thread in +math(EXPR SMR_NUM_THREADS "${NUM_THREADS} + 2") + +if(NOT DEFINED ALGOS) + set(ALGOS STACK_XBO_BACKOFF STACK_ELIMINATION_BACKOFF) +endif() + +set(TEST_DEFS + VSTACK_TESTING IT=${ITERATIONS} VSTACK_XCHG_MAX_DURATION_MS=100 + VSTACK_XCHGER_COUNT=5 VSTACK_MAX_BACKOFF_MS=1000 + SMR_MAX_NTHREADS=${SMR_NUM_THREADS}) + +if(DEFINED LIBVSYNC_DISTRO_TESTING) + set(ALGOS STACK_XBO_BACKOFF STACK_ELIMINATION_BACKOFF) +endif() + +# for all files that start with test +foreach(test_path IN ITEMS ${TEST_FILES}) + + # extract test_name with extension + get_filename_component(test_name ${test_path} NAME) + + # name without extension + get_filename_component(test_prefix ${test_path} NAME_WE) + + foreach(algo IN ITEMS ${ALGOS}) + set(TEST ${test_prefix}_${algo}) + + if(${test_prefix} MATCHES ".*specific.*" AND (NOT ${algo} MATCHES + "HELPER")) + set(SKIP on) + else() + set(SKIP off) + endif() + + if(NOT ${SKIP}) + + if(${algo} MATCHES "HELPER") + set(TC_DEFS ${TEST_DEFS} ${algo} NTHREADS=16) + else() + set(TC_DEFS ${TEST_DEFS} ${algo} NTHREADS=${NUM_THREADS}) + endif() + # make it lower-case + string(TOLOWER ${TEST} TEST) + + # add the executable + add_executable(${TEST} ${test_name}) + + # link libs + target_link_libraries(${TEST} vsync pthread) + + # activate target algo by adding the appropriate define + target_compile_definitions(${TEST} PUBLIC ${TC_DEFS}) + + # add it as a test + v_add_bin_test_advanced( + NAME + ${TEST} + COMMAND + ${TEST} + TIMEOUT + 3600 + PROCESSORS + 4) + endif() + endforeach() + +endforeach() diff --git a/test/stack/test_sanity.c b/test/stack/test_sanity.c new file mode 100644 index 0000000..7a11646 --- /dev/null +++ b/test/stack/test_sanity.c @@ -0,0 +1,87 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +#ifndef NTHREADS + #error "undefined number of threads" +#endif + +#define NUM_PUSH_PER_THREAD 3 + +vatomic64_t g_key = VATOMIC_INIT(1); +vsize_t ds_idx = 0; + +void +single_threaded(void) +{ + vuint64_t id = 0; + vuint64_t first_id = 1; + vuint64_t last_id = 10; + vuint64_t out_id = 0; + + init(); + + for (id = first_id; id <= last_id; id++) { + push(MAIN_TID, ds_idx, id); + } + + for (id = last_id; id >= first_id; id--) { + out_id = pop(MAIN_TID, ds_idx); + ASSERT(out_id == id); + } + + out_id = pop(MAIN_TID, ds_idx); + ASSERT(out_id == 0); + + verify(ds_idx); + + destroy(); +} + +void * +run(void *arg) +{ + vsize_t tid = ((vsize_t)arg); + vsize_t i = 0; + vuintptr_t key = 0; + + reg(tid); + + for (i = 0; i < NUM_PUSH_PER_THREAD; i++) { + key = vatomic64_get_inc(&g_key); + push(tid, ds_idx, key); + } + + vbool_t stop = false; + while (!stop) { + stack_enter(tid); + stop = pop(tid, ds_idx) == 0; + stack_exit(tid); + } + + dereg(tid); + + return NULL; +} + +void +multi_threaded(void) +{ + init(); + launch_threads(NTHREADS, run); + verify(ds_idx); + destroy(); +} + +int +main(void) +{ + single_threaded(); + multi_threaded(); + return 0; +} diff --git a/test/stack/test_stress.c b/test/stack/test_stress.c new file mode 100644 index 0000000..08841dd --- /dev/null +++ b/test/stack/test_stress.c @@ -0,0 +1,72 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +#ifndef NTHREADS + #error "undefined number of threads" +#endif + +#ifndef IT + #error "undefined number of iterations" +#endif + +vatomic64_t g_key = VATOMIC_INIT(1); + +vsize_t ds_idx = 0; + +void +pusher(vsize_t tid) +{ + vsize_t i = 0; + vuintptr_t key = 0; + + for (i = 0; i < IT; i++) { + key = vatomic64_get_inc_rlx(&g_key); + push(tid, ds_idx, key); + } +} + +void +popper(vsize_t tid) +{ + vsize_t i = 0; + + for (i = 0; i < IT; i++) { + stack_enter(tid); + pop(tid, ds_idx); + stack_exit(tid); + } +} + +void * +run(void *arg) +{ + vsize_t tid = ((vsize_t)arg); + + reg(tid); + + if (tid % 2 == 0) { + pusher(tid); + } else { + popper(tid); + } + + dereg(tid); + + return NULL; +} + +int +main(void) +{ + init(); + launch_threads(NTHREADS, run); + verify(ds_idx); + destroy(); + return 0; +} diff --git a/test/treeset/CMakeLists.txt b/test/treeset/CMakeLists.txt new file mode 100644 index 0000000..8460eed --- /dev/null +++ b/test/treeset/CMakeLists.txt @@ -0,0 +1,35 @@ +ProcessorCount(PCOUNT) + +file(GLOB TEST_FILES test_*.c) + +set(ALGOS BST_FINE RB_FINE BST_COARSE RB_COARSE) +set(TEST_DEFS TREESET_LOCK_TTAS) + +# For Code coverage mode use only 4 threads, otherwise PCOUNT. +set(NUM_THREADS $,4,${PCOUNT}>) + +list(APPEND TEST_DEFS NTHREADS=${NUM_THREADS}) + +foreach(test_path IN ITEMS ${TEST_FILES}) + get_filename_component(test_name ${test_path} NAME) + get_filename_component(test_prefix ${test_path} NAME_WE) + + foreach(algo IN ITEMS ${ALGOS}) + set(TEST ${test_prefix}_${algo}) + string(TOLOWER ${TEST} TEST) + add_executable(${TEST} ${test_name}) + target_link_libraries(${TEST} vsync pthread) + target_compile_definitions(${TEST} PUBLIC TREESET_${algo} ${TEST_DEFS}) + + v_add_heavy_stress_test( + NAME + ${TEST} + COMMAND + ${TEST} + STRESS_TIMEOUT + 4200 + TIMEOUT + 3600) + endforeach() + +endforeach() diff --git a/test/treeset/test_treeset_sanity.c b/test/treeset/test_treeset_sanity.c new file mode 100644 index 0000000..9682399 --- /dev/null +++ b/test/treeset/test_treeset_sanity.c @@ -0,0 +1,74 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +#define NKEYS 4000 + +void +test_add_rem_con(vsize_t num) +{ + vsize_t n = 0; + vbool_t *exist = malloc(num * sizeof(vbool_t)); + for (vsize_t i = 0; i < num; ++i) { + exist[i] = false; + } + + vsize_t oper = 0; + + while (n < num) { + int key = random() % num; + vbool_t suc_con = tr_con(key); + vbool_t suc_add = tr_add(key); + + ASSERT(suc_con == exist[key]); + ASSERT(suc_add == !exist[key]); + + tr_verify(); + + if (!exist[key]) { + exist[key] = true; + n++; + } + oper++; + } + + printf("Operations: %zu\n", oper); + + oper = 0; + n = 0; + while (n < num) { + int key = random() % num; + vbool_t suc_con = tr_con(key); + vbool_t suc_rem = tr_rem(key); + + ASSERT(suc_con == exist[key]); + ASSERT(suc_rem == exist[key]); + + tr_verify(); + + if (exist[key]) { + exist[key] = false; + n++; + } + oper++; + } + + printf("Operations: %zu\n", oper); + + free(exist); +} + +int +main(void) +{ + tr_init(); + + test_add_rem_con(NKEYS); + + tr_destroy(); +} diff --git a/test/treeset/test_treeset_sanity_interface.c b/test/treeset/test_treeset_sanity_interface.c new file mode 100644 index 0000000..af3b8ab --- /dev/null +++ b/test/treeset/test_treeset_sanity_interface.c @@ -0,0 +1,51 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include + +int +main(void) +{ + treeset_t tree; + int res; + + vmem_lib_t mem_lib = VMEM_LIB_DEFAULT(); + treeset_init(&tree, mem_lib); + + treeset_key_t key = 3; + int value = 3; + int *out_value = NULL; + void **ptr_out_value = (void **)&out_value; + + res = treeset_add(&tree, key, &value, ptr_out_value); + ASSERT(res); + ASSERT(out_value == NULL); + + res = treeset_add(&tree, key, &value, ptr_out_value); + ASSERT(!res); + ASSERT(out_value == &value); + + out_value = NULL; + res = treeset_contains(&tree, key, ptr_out_value); + ASSERT(res); + ASSERT(out_value == &value); + + out_value = NULL; + res = treeset_remove(&tree, key, ptr_out_value); + ASSERT(res); + ASSERT(out_value == &value); + + out_value = NULL; + res = treeset_remove(&tree, key, ptr_out_value); + ASSERT(!res); + ASSERT(out_value == NULL); + + res = treeset_contains(&tree, key, ptr_out_value); + ASSERT(!res); + ASSERT(out_value == NULL); + + treeset_destroy(&tree); +} diff --git a/test/treeset/test_treeset_sanity_visit.c b/test/treeset/test_treeset_sanity_visit.c new file mode 100644 index 0000000..d1ef317 --- /dev/null +++ b/test/treeset/test_treeset_sanity_visit.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include + +void +visitor(treeset_key_t key, void *value, void *arg) +{ + ASSERT(*(int *)arg == -1); + ASSERT((key == 3 && *(int *)value == 30) || + (key == 5 && *(int *)value == 50)); +} + +int +main(void) +{ + treeset_t tree; + + vmem_lib_t mem_lib = VMEM_LIB_DEFAULT(); + treeset_init(&tree, mem_lib); + + int value3 = 30; + treeset_add(&tree, 3, &value3, NULL); + + int value5 = 50; + treeset_add(&tree, 5, &value5, NULL); + + int arg = -1; + treeset_visit(&tree, visitor, &arg); + + treeset_destroy(&tree); +} diff --git a/test/treeset/test_treeset_stress.c b/test/treeset/test_treeset_stress.c new file mode 100644 index 0000000..457f1f8 --- /dev/null +++ b/test/treeset/test_treeset_stress.c @@ -0,0 +1,159 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +#include + +#ifndef NTHREADS + #error "Choose number of threads by setting NTHREADS" +#endif + +#define NKEYS 200000 +#define NITERS 400 + +// Each thread gets a pool of keys (consecutive block in g_keys array) +// and it will be adding/removing them in patterns, chosen randomly and +// independently from other threads + +#define BLOCK_LEN (NKEYS / NTHREADS) + +treeset_key_t *g_keys; +vbool_t *g_exist; + +enum { + ITER_ONLY_ADDING, + ITER_RANDOM_ADDING, + ITER_ONLY_REMOVING, + ITER_RANDOM_REMOVING, + ITER_ONLY_SEARCHING, + ITER_RANDOM_SEARCHING, + ITER_ONLY_FLIPPING, + ITER_RANDOM_FLIPPING, + NITER_TYPES +}; + +void +init(void) +{ + g_keys = malloc(BLOCK_LEN * NTHREADS * sizeof(treeset_key_t)); + g_exist = malloc(BLOCK_LEN * NTHREADS * sizeof(vbool_t)); + for (vsize_t i = 0; i < BLOCK_LEN * NTHREADS; ++i) { + g_keys[i] = i; + g_exist[i] = false; + } + + // Random shuffle using Fisher-Yates algorithm + for (vsize_t i = 1; i < BLOCK_LEN * NTHREADS; ++i) { + vsize_t index = rand() % i; + treeset_key_t temp = g_keys[i]; + g_keys[i] = g_keys[index]; + g_keys[index] = temp; + } +} + +void * +run(void *arg) +{ + vsize_t tid = (vsize_t)(vuintptr_t)arg; + unsigned int seed = tid; + + vsize_t begin = tid * BLOCK_LEN; + vsize_t end = (tid + 1) * BLOCK_LEN; + + for (vsize_t iter = 0; iter < NITERS; ++iter) { + int iter_type = rand_r(&seed) % NITER_TYPES; + vbool_t norand = false; + + switch (iter_type) { + case ITER_ONLY_ADDING: + case ITER_ONLY_REMOVING: + case ITER_ONLY_SEARCHING: + case ITER_ONLY_FLIPPING: + norand = true; + break; + } + + switch (iter_type) { + case ITER_ONLY_ADDING: + case ITER_RANDOM_ADDING: + for (vsize_t i = begin; i < end; ++i) { + if (norand || rand_r(&seed) % 2) { + vbool_t res = tr_add(g_keys[i]); + ASSERT(res != g_exist[i]); + g_exist[i] = true; + } + } + break; + case ITER_ONLY_REMOVING: + case ITER_RANDOM_REMOVING: + for (vsize_t i = begin; i < end; ++i) { + if (norand || rand_r(&seed) % 2) { + vbool_t res = tr_rem(g_keys[i]); + ASSERT(res == g_exist[i]); + g_exist[i] = false; + } + } + break; + case ITER_ONLY_SEARCHING: + case ITER_RANDOM_SEARCHING: + for (vsize_t i = begin; i < end; ++i) { + if (norand || rand_r(&seed) % 2) { + vbool_t res = tr_con(g_keys[i]); + ASSERT(res == g_exist[i]); + } + } + break; + case ITER_ONLY_FLIPPING: + case ITER_RANDOM_FLIPPING: + for (vsize_t i = begin; i < end; ++i) { + if (norand || rand_r(&seed) % 2) { + if (g_exist[i]) { + ASSERT(tr_rem(g_keys[i])); + g_exist[i] = false; + } else { + ASSERT(tr_add(g_keys[i])); + g_exist[i] = true; + } + } + } + break; + default: + ASSERT(0); + } + + int count = 0; + for (vsize_t i = begin; i < end; ++i) { + count += g_exist[i]; + } + printf("Thr %zu, iter %zu, type %d, count %d\n", tid, iter, iter_type, + count); + } + + return NULL; +} + +void +destroy(void) +{ + free(g_keys); + free(g_exist); +} + +int +main(void) +{ + tr_init(); + + init(); + launch_threads(NTHREADS, run); + destroy(); + + tr_verify(); + + tr_destroy(); +} diff --git a/test/treeset/test_treeset_traces.c b/test/treeset/test_treeset_traces.c new file mode 100644 index 0000000..aeb01d9 --- /dev/null +++ b/test/treeset/test_treeset_traces.c @@ -0,0 +1,48 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include + +#include +#include +#include + +#ifndef NTHREADS + #error "Choose number of threads by setting NTHREADS" +#endif + +#define NITERS 20000 + +void * +run(void *arg) +{ + vsize_t tid = (vsize_t)(vuintptr_t)arg; + + for (vsize_t i = 0; i < NITERS; ++i) { + int key = i; + if (tr_con_trace(tid, key)) { + tr_rem_trace(tid, key); + } else { + tr_add_trace(tid, key); + } + } + + return NULL; +} + +int +main(void) +{ + tr_init_trace(); + + launch_threads(NTHREADS, run); + + tr_verify_traces(); + + tr_destroy_trace(); + + tr_verify_allocs(); +} diff --git a/verify/bitmap/CMakeLists.txt b/verify/bitmap/CMakeLists.txt new file mode 100644 index 0000000..5e0bfc5 --- /dev/null +++ b/verify/bitmap/CMakeLists.txt @@ -0,0 +1,31 @@ +# detect test cases header files +file(GLOB TEST_CASES test_case*.h) + +# verification template file +set(VERIFY_FILE verify_bitmap.c) + +set(ALGO bitmap) + +foreach(TC IN ITEMS ${TEST_CASES}) + + get_filename_component(TC_NAME ${TC} NAME) + get_filename_component(TC_NAME_WE ${TC} NAME_WE) + set(TEST_NAME ${TC_NAME_WE}_${ALGO}) + + # add executable test + set(TC_DEF TEST_CASE="${TC_NAME}") + add_executable(${TEST_NAME} ${VERIFY_FILE}) + target_link_libraries(${TEST_NAME} vsync pthread) + target_compile_definitions(${TEST_NAME} PUBLIC ${TC_DEF}) + v_add_bin_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + + # add verification client + set(TC_CHECK_DEF -DTEST_CASE="'\"${TC_NAME}\"'") + add_vsyncer_check( + TARGET check_${TEST_NAME} + CFLAGS ${TC_CHECK_DEF} + SOURCE ${VERIFY_FILE} + TIMEOUT 120 + DEPENDENCIES vsync) + +endforeach() diff --git a/verify/bitmap/test_case_clr_interset.h b/verify/bitmap/test_case_clr_interset.h new file mode 100644 index 0000000..e716c8b --- /dev/null +++ b/verify/bitmap/test_case_clr_interset.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +void +pre(void) +{ + vbitmap_init(g_bitmap, BITMAP_LEN, true); +} +void +t0(vsize_t tid) +{ + vbitmap_clr_range(g_bitmap, 60, 62); + vbitmap_set_bit(g_bitmap, 61); + V_UNUSED(tid); +} +void +t1(vsize_t tid) +{ + vbitmap_clr_range(g_bitmap, 62, 64); + vbitmap_set_bit(g_bitmap, 63); + V_UNUSED(tid); +} +void +t2(vsize_t tid) +{ + vbitmap_clr_range(g_bitmap, 64, 66); + vbitmap_set_bit(g_bitmap, 65); + V_UNUSED(tid); +} +void +post(void) +{ + for (vsize_t i = 0; i < 59; i++) { + vbool_t set = vbitmap_get(g_bitmap, i); + ASSERT(set); + } + for (vsize_t i = 60; i < 66; i++) { + vbool_t set = vbitmap_get(g_bitmap, i); + + if (i == 61 || i == 63 || i == 65) { + ASSERT(set); + } else { + ASSERT(!set); + } + } +} diff --git a/verify/bitmap/test_case_intact.h b/verify/bitmap/test_case_intact.h new file mode 100644 index 0000000..85b5240 --- /dev/null +++ b/verify/bitmap/test_case_intact.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +void +pre(void) +{ + vbitmap_set_range(g_bitmap, NTHREADS, BITMAP_LEN - 1); +} +void +t(vsize_t tid) +{ + vbool_t set = false; + + vbitmap_set_bit(g_bitmap, tid); + set = vbitmap_get(g_bitmap, tid); + ASSERT(set); + + vbitmap_clr_bit(g_bitmap, tid); + set = vbitmap_get(g_bitmap, tid); + ASSERT(!set); +} +void +t0(vsize_t tid) +{ + t(tid); +} +void +t1(vsize_t tid) +{ + t(tid); +} +void +t2(vsize_t tid) +{ + t(tid); +} +void +post(void) +{ + for (vsize_t i = 0; i < BITMAP_LEN; i++) { + vbool_t set = vbitmap_get(g_bitmap, i); + if (i < NTHREADS) { + ASSERT(!set); + } else { + ASSERT(set); + } + } +} diff --git a/verify/bitmap/test_case_iter.h b/verify/bitmap/test_case_iter.h new file mode 100644 index 0000000..22e0143 --- /dev/null +++ b/verify/bitmap/test_case_iter.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#define RANGE_MIN 31U +#define RANGE_MAX (BITMAP_LEN - 1U) + +void +pre(void) +{ +} + +void +t0(vsize_t tid) +{ + vbitmap_set_range(g_bitmap, RANGE_MIN, VUINT64_WIDTH); + V_UNUSED(tid); +} +void +t1(vsize_t tid) +{ + vbitmap_set_range(g_bitmap, VUINT64_WIDTH, RANGE_MAX); + V_UNUSED(tid); +} +void +t2(vsize_t tid) +{ + vbitmap_iter_t iter; + vsize_t idx = 0; + vbitmap_iter_init(g_bitmap, &iter); + + while (vbitmap_iter_next(&iter, &idx)) { + ASSERT(idx >= RANGE_MIN && idx <= RANGE_MAX); + } + + V_UNUSED(tid); +} +void +post(void) +{ + vbitmap_iter_t iter; + vsize_t idx = 0; + vsize_t expected = RANGE_MIN; + vbitmap_iter_init(g_bitmap, &iter); + + while (vbitmap_iter_next(&iter, &idx)) { + ASSERT(idx >= RANGE_MIN && idx <= RANGE_MAX); + ASSERT(idx == expected++); + } + ASSERT(idx == RANGE_MAX); +} diff --git a/verify/bitmap/test_case_range.h b/verify/bitmap/test_case_range.h new file mode 100644 index 0000000..2d8570c --- /dev/null +++ b/verify/bitmap/test_case_range.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +void +pre(void) +{ +} +void +t0(vsize_t tid) +{ + V_UNUSED(tid); + vbitmap_set_range(g_bitmap, 1, 4); +} +void +t1(vsize_t tid) +{ + V_UNUSED(tid); + vbitmap_clr_range(g_bitmap, 2, 3); +} +void +t2(vsize_t tid) +{ + V_UNUSED(tid); + vbitmap_set_range(g_bitmap, 63, 66); +} +void +post(void) +{ + vsize_t set_indexes[6] = {1, 4, 63, 64, 65, 66}; + for (vsize_t i = 0; i < 6; i++) { + vbool_t set = vbitmap_get(g_bitmap, set_indexes[i]); + ASSERT(set); + } +} diff --git a/verify/bitmap/test_case_set_intersect.h b/verify/bitmap/test_case_set_intersect.h new file mode 100644 index 0000000..6a24448 --- /dev/null +++ b/verify/bitmap/test_case_set_intersect.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +void +pre(void) +{ +} +void +t0(vsize_t tid) +{ + vbitmap_set_range(g_bitmap, 60, 62); + vbitmap_clr_bit(g_bitmap, 61); + V_UNUSED(tid); +} +void +t1(vsize_t tid) +{ + vbitmap_set_range(g_bitmap, 62, 64); + vbitmap_clr_bit(g_bitmap, 63); + V_UNUSED(tid); +} +void +t2(vsize_t tid) +{ + vbitmap_set_range(g_bitmap, 64, 66); + vbitmap_clr_bit(g_bitmap, 65); + V_UNUSED(tid); +} +void +post(void) +{ + for (vsize_t i = 0; i < 59; i++) { + vbool_t set = vbitmap_get(g_bitmap, i); + ASSERT(!set); + } + for (vsize_t i = 60; i < 66; i++) { + vbool_t set = vbitmap_get(g_bitmap, i); + + if (i == 61 || i == 63 || i == 65) { + ASSERT(!set); + } else { + ASSERT(set); + } + } +} diff --git a/verify/bitmap/verify_bitmap.c b/verify/bitmap/verify_bitmap.c new file mode 100644 index 0000000..e24e4dc --- /dev/null +++ b/verify/bitmap/verify_bitmap.c @@ -0,0 +1,77 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +#define NTHREADS 3 +#include +#include + +#define BITMAP_LEN (VUINT64_WIDTH + 3U) +#define BITMAP_SLOTS 2U +#define BITMAP_SIZE (sizeof(vbitmap_t) + (sizeof(vatomic64_t) * BITMAP_SLOTS)) +char g_mem[BITMAP_SIZE]; // workaround +vbitmap_t *g_bitmap; + +void *run(void *args); +void t0(vsize_t); +void t1(vsize_t); +void t2(vsize_t); +void pre(void); +void post(void); +void init(void); +void destroy(void); + +#ifdef TEST_CASE + #include TEST_CASE +#else + #error "no test case was defined" +#endif + +int +main(void) +{ + init(); + pre(); + launch_threads(NTHREADS, run); + post(); + destroy(); +} + +void +init(void) +{ + vsize_t size = vbitmap_size(BITMAP_LEN); + ASSERT(BITMAP_SIZE == size); + g_bitmap = (vbitmap_t *)g_mem; + ASSERT(g_bitmap); + vbitmap_init(g_bitmap, BITMAP_LEN, false); +} + +void +destroy(void) +{ +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + ASSERT(tid < NTHREADS); + + switch (tid) { + case 0: + t0(tid); + break; + case 1: + t1(tid); + break; + case 2: + t2(tid); + break; + default: + break; + } + return NULL; +} diff --git a/verify/simpleht/CMakeLists.txt b/verify/simpleht/CMakeLists.txt new file mode 100644 index 0000000..f8f4b56 --- /dev/null +++ b/verify/simpleht/CMakeLists.txt @@ -0,0 +1,83 @@ +# detect test cases header files +file(GLOB TEST_CASES test_case*.h) + +set(VERIFY_FILE verify.c) + +set(VERIFY_WITH_GENMC10 # + check_test_case_add_get_rem_simpleht_imm # + check_test_case_add_get_rem_simpleht_rc11 # + check_test_case_same_bucket_simpleht_imm # + check_test_case_same_bucket_simpleht_rc11 # +) + +set(DISABLE_IPR_SR) + +set(MEMORY_MODELS "imm" "rc11") + +set(ALGOS SIMPLEHT) + +# for all files that start with test +foreach(test_path IN ITEMS ${TEST_CASES}) + + # extract test_name with extension + get_filename_component(test_name ${test_path} NAME) + + # name without extension + get_filename_component(test_prefix ${test_path} NAME_WE) + + # define TEST_CASE, we need to pass it like this to be recognized + set(tc TEST_CASE="${test_name}") + # we have to escape it like this to work for check + set(tc_check -DTEST_CASE="'\"${test_name}\"'") + + foreach(algo IN ITEMS ${ALGOS}) + + # construct the test name + set(TEST ${test_prefix}_${algo}) + + # make it lower-case + string(TOLOWER ${TEST} TEST) + + # add the executable + add_executable(${TEST} ${VERIFY_FILE}) + + # link libs + target_link_libraries(${TEST} vsync pthread) + + # activate target algo by adding the appropriate define + target_compile_definitions(${TEST} PUBLIC ${algo} ${TEST_DEFS} ${tc}) + + # add it as a test + v_add_bin_test(NAME ${TEST} COMMAND ${TEST}) + + # set timeout + set_tests_properties(${TEST} PROPERTIES TIMEOUT 10) + + foreach(WMM IN ITEMS ${MEMORY_MODELS}) + set(CHECK_NAME_PREFIX check_${TEST}) + set(CHECK_NAME ${CHECK_NAME_PREFIX}_${WMM}) + + if(${CHECK_NAME} IN_LIST VERIFY_WITH_GENMC10) + set(CHECK_OPTIONS USE_GENMC10) + if("${CHECK_NAME}" IN_LIST DISABLE_IPR_SR) + set(CHECK_EXTRA_OPTIONS "-disable-sr -disable-ipr") + else() + unset(CHECK_EXTRA_OPTIONS) + endif() + else() + unset(CHECK_OPTIONS) + endif() + + add_vsyncer_check( + TARGET ${CHECK_NAME_PREFIX} + SOURCE ${VERIFY_FILE} + CFLAGS -D${algo} ${tc_check} + TIMEOUT 1200 + MEMORY_MODELS + ${WMM} # + ${CHECK_OPTIONS} # + GENMC10_EXTRA_OPTIONS ${CHECK_EXTRA_OPTIONS} # + DEPENDENCIES vsync) + endforeach() + endforeach() +endforeach() diff --git a/verify/simpleht/test_case_add.h b/verify/simpleht/test_case_add.h new file mode 100644 index 0000000..8d8ed1a --- /dev/null +++ b/verify/simpleht/test_case_add.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +void +pre(void) +{ +} +void +t0(vsize_t tid) +{ + vbool_t success = imap_add(tid, 1, 1); + ASSERT(success); +} +void +t1(vsize_t tid) +{ + vbool_t success = imap_add(tid, 2, 2); + ASSERT(success); +} +void +t2(vsize_t tid) +{ + vbool_t success = imap_add(tid, 3, 3); + ASSERT(success); +} +void +post(void) +{ + for (vuintptr_t k = 1; k <= 3; k++) { + data_t *d = imap_get(MAIN_TID, k); + ASSERT(d); + ASSERT(d->val == k); + } +} diff --git a/verify/simpleht/test_case_add_get.h b/verify/simpleht/test_case_add_get.h new file mode 100644 index 0000000..ab341a7 --- /dev/null +++ b/verify/simpleht/test_case_add_get.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +#define PRE_KEYS_LEN 2U + +vuintptr_t g_pre_keys[PRE_KEYS_LEN] = {1, 3}; +vuintptr_t g_new_key = 2; +vbool_t g_added = false; +vbool_t g_got_nw_val = false; + +void +pre(void) +{ + for (vsize_t i = 0; i < PRE_KEYS_LEN; i++) { + vbool_t success = imap_add(MAIN_TID, g_pre_keys[i], g_pre_keys[i]); + ASSERT(success); + } +} +void +t0(vsize_t tid) +{ + ASSERT(tid < PRE_KEYS_LEN); + data_t *data = imap_get(tid, g_pre_keys[tid]); + ASSERT(data); +} +void +t1(vsize_t tid) +{ + ASSERT(tid < PRE_KEYS_LEN); + data_t *data = imap_get(tid, g_pre_keys[tid]); + ASSERT(data); +} +void +t2(vsize_t tid) +{ + vbool_t success = imap_add(tid, g_new_key, g_new_key); + ASSERT(success); + data_t *data = imap_get(tid, g_new_key); + ASSERT(data); +} +void +post(void) +{ + for (vsize_t i = 0; i < PRE_KEYS_LEN; i++) { + data_t *data = imap_get(MAIN_TID, g_pre_keys[i]); + ASSERT(data); + } + data_t *data = imap_get(MAIN_TID, g_new_key); + ASSERT(data); +} diff --git a/verify/simpleht/test_case_add_get_rem.h b/verify/simpleht/test_case_add_get_rem.h new file mode 100644 index 0000000..4077fb5 --- /dev/null +++ b/verify/simpleht/test_case_add_get_rem.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#define MIN_KEY 1U +#define MAX_KEY VSIMPLE_HT_CAPACITY + +#define CONFLICT_KEY (MIN_KEY + 1U) +#define NEW_VAL 999U + +vbool_t g_added = false; +vbool_t g_got_nw_val = false; + +void +pre(void) +{ + for (vuintptr_t k = MIN_KEY; k <= MAX_KEY; k++) { + vbool_t success = imap_add(MAIN_TID, k, k); + ASSERT(success); + } +} +void +t0(vsize_t tid) +{ + g_added = imap_add(tid, CONFLICT_KEY, NEW_VAL); +} +void +t1(vsize_t tid) +{ + vbool_t success = imap_rem(tid, CONFLICT_KEY); + ASSERT(success); +} +void +t2(vsize_t tid) +{ + data_t *data = imap_get(tid, CONFLICT_KEY); + if (data) { + if (data->val == NEW_VAL) { + g_got_nw_val = true; + } + } +} +void +post(void) +{ + for (vuintptr_t k = MIN_KEY; k <= MAX_KEY; k++) { + data_t *d = imap_get(MAIN_TID, k); + if (k == CONFLICT_KEY) { + if (g_added) { + ASSERT(d && d->val == NEW_VAL); + } else if (d) { + ASSERT(d->val == k); + } + } else { + ASSERT(d); + } + } + if (g_got_nw_val) { + ASSERT(g_added); + } +} diff --git a/verify/simpleht/test_case_same_bucket.h b/verify/simpleht/test_case_same_bucket.h new file mode 100644 index 0000000..a64a896 --- /dev/null +++ b/verify/simpleht/test_case_same_bucket.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +#define LEN (VSIMPLE_HT_CAPACITY - 1U) +vuintptr_t g_keys[LEN]; +void +pre(void) +{ + vsize_t cnt = 0; + vuintptr_t k = 0; + vuintptr_t b = LEN; + + while (cnt < LEN) { + k++; + if (k % VSIMPLE_HT_CAPACITY == b) { + vbool_t success = imap_add(MAIN_TID, k, k); + ASSERT(success); + g_keys[cnt] = k; + cnt++; + } + } +} +void +t0(vsize_t tid) +{ + ASSERT(tid < LEN); + vbool_t success = imap_add(tid, g_keys[tid], g_keys[tid]); + ASSERT(!success); +} +void +t1(vsize_t tid) +{ + ASSERT(tid < LEN); + data_t *data = imap_get(tid, g_keys[tid]); + ASSERT(data); +} +void +t2(vsize_t tid) +{ + ASSERT(tid < LEN); + vbool_t success = imap_rem(tid, g_keys[tid]); + ASSERT(success); +} +void +post(void) +{ +} diff --git a/verify/simpleht/verify.c b/verify/simpleht/verify.c new file mode 100644 index 0000000..3067dcd --- /dev/null +++ b/verify/simpleht/verify.c @@ -0,0 +1,54 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#define NTHREADS 3U +#include +#include + +void t0(vsize_t); +void t1(vsize_t); +void t2(vsize_t); +void pre(void); +void post(void); + +#ifdef TEST_CASE + #include TEST_CASE +#else + #error "no test case was defined" +#endif + + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + imap_reg(tid); + switch (tid) { + case 0: + t0(tid); + break; + case 1: + t1(tid); + break; + case 2: + t2(tid); + break; + default: + break; + } + imap_dereg(tid); + return NULL; +} + +int +main(void) +{ + imap_init(); + pre(); + launch_threads(NTHREADS, run); + post(); + imap_destroy(); + return 0; +} diff --git a/verify/stack/CMakeLists.txt b/verify/stack/CMakeLists.txt new file mode 100644 index 0000000..3679601 --- /dev/null +++ b/verify/stack/CMakeLists.txt @@ -0,0 +1,113 @@ +# detect test cases header files +file(GLOB TEST_CASES test_case*.h) + +# verification template file +set(VERIFY_FILE verify_stack.c) + +# verification limited by 4 threads max +set(NUM_THREADS 4) + +# add -DDBG_ALL to enable debug messages + +set(TEST_DEFS VSTACK_TESTING SMR_MAX_NTHREADS=${NUM_THREADS} + -DVSTACK_XCHGER_COUNT=1 -DVSTACK_XCHG_MAX_DURATION_MS=100) + +set(CHECK_FLAGS -DVSYNC_VERIFICATION -DVSTACK_TESTING -DSMR_THRESHOLD=1 + -DSMR_MAX_NTHREADS=${NUM_THREADS} -DVSTACK_XCHGER_COUNT=1) + +set(VERIFY_WITH_GENMC10 + check_test_case_2push_2pop_stack_elimination_backoff_imm + check_test_case_2push_2pop_stack_xbo_backoff_imm + check_test_case_2push_msg_passing_stack_elimination_backoff_imm + check_test_case_2push_msg_passing_stack_xbo_backoff_imm + check_test_case_lifo_stack_elimination_backoff_imm + check_test_case_lifo_stack_xbo_backoff_imm + check_test_case_only_pop_stack_elimination_backoff_imm + check_test_case_only_pop_stack_xbo_backoff_imm) + +set(VERIFIY_WITHOUT_SMR_ON_QUICK + check_test_case_lifo_stack_xbo_backoff_imm + check_test_case_lifo_stack_elimination_backoff_imm + check_test_case_2push_2pop_stack_xbo_backoff_imm + check_test_case_2push_2pop_stack_elimination_backoff_imm + check_test_case_only_pop_stack_xbo_backoff_imm + check_test_case_only_pop_stack_elimination_backoff_imm) + +set(MEMORY_MODELS "imm" "rc11") + +if(NOT DEFINED ALGOS) + set(ALGOS STACK_XBO_BACKOFF STACK_ELIMINATION_BACKOFF) +endif() + +# for all files that start with test +foreach(test_path IN ITEMS ${TEST_CASES}) + + # extract test_name with extension + get_filename_component(test_name ${test_path} NAME) + + # name without extension + get_filename_component(test_prefix ${test_path} NAME_WE) + + # The test includes whatever is given as TEST_CASE define TEST_CASE, we need + # to pass it like this to be recognized + set(tc TEST_CASE="${test_name}") + + # we have to escape it like this to work for check + set(tc_check -DTEST_CASE="'\"${test_name}\"'") + + foreach(algo IN ITEMS ${ALGOS}) + + if("${test_name}" MATCHES ".*specific.*") + if(NOT ${algo} MATCHES "HELPER") + continue() + endif() + endif() + # construct the test name + set(TEST ${test_prefix}_${algo}) + + # make it lower-case + string(TOLOWER ${TEST} TEST) + + set(TEST_NAME test_${TEST}) + + # add the executable + add_executable(${TEST_NAME} ${VERIFY_FILE}) + + # link libs + target_link_libraries(${TEST_NAME} vsync pthread) + + # activate target algo by adding the appropriate define + target_compile_definitions(${TEST_NAME} PUBLIC ${algo} ${TEST_DEFS} + ${tc}) + + # add it as a test + v_add_bin_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + + # set timeout + set_tests_properties(${TEST_NAME} PROPERTIES TIMEOUT 10) + + foreach(WMM IN ITEMS ${MEMORY_MODELS}) + set(CHECK_NAME_PREFIX check_${TEST}) + set(CHECK_NAME ${CHECK_NAME_PREFIX}_${WMM}) + if("${CHECK_NAME}" IN_LIST VERIFY_WITH_GENMC10) + set(CHECK_OPTIONS USE_GENMC10) + else() + unset(CHECK_OPTIONS) + endif() + if((NOT ${VSYNCER_CHECK_FULL}) AND ("${CHECK_NAME}" IN_LIST + VERIFIY_WITHOUT_SMR_ON_QUICK)) + set(SMR_FLAG -DSMR_NONE) + else() + unset(SMR_FLAG) + endif() + add_vsyncer_check( + TARGET ${CHECK_NAME_PREFIX} + SOURCE ${VERIFY_FILE} + CFLAGS -D${algo} ${tc_check} ${CHECK_FLAGS} ${SMR_FLAG} + MEMORY_MODELS ${WMM} + TIMEOUT 600 # + ${CHECK_OPTIONS} # + DEPENDENCIES vsync) + endforeach() + endforeach() +endforeach() diff --git a/verify/stack/test_case_2push_2pop.h b/verify/stack/test_case_2push_2pop.h new file mode 100644 index 0000000..909b95c --- /dev/null +++ b/verify/stack/test_case_2push_2pop.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +vsize_t ds_idx = 0; + +void +pre(void) +{ +} + +void +t0(vsize_t tid) +{ + push(tid, ds_idx, 1); + stack_enter(tid); + pop(tid, ds_idx); + stack_exit(tid); +} + +void +t1(vsize_t tid) +{ + stack_enter(tid); + pop(tid, ds_idx); + stack_exit(tid); + push(tid, ds_idx, 2); +} + +void +t2(vsize_t tid) +{ + stack_enter(tid); + pop(tid, ds_idx); + stack_exit(tid); +#if !defined(VSYNC_VERIFICATION_QUICK) + stack_clean(tid); +#endif +} + +void +post(void) +{ +} diff --git a/verify/stack/test_case_2push_msg_passing.h b/verify/stack/test_case_2push_msg_passing.h new file mode 100644 index 0000000..62f434a --- /dev/null +++ b/verify/stack/test_case_2push_msg_passing.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +vsize_t ds_idx = 0; + +int msg_pushed_1 = 0; +int msg_pushed_2 = 0; + +void +pre(void) +{ +} +void +t0(vsize_t tid) +{ + msg_pushed_1 = 1; + push(tid, ds_idx, 1); + + msg_pushed_2 = 1; + push(tid, ds_idx, 2); +} + +void +t1(vsize_t tid) +{ + stack_enter(tid); + vuintptr_t val = pop(tid, ds_idx); + if (val == 2) { + ASSERT(msg_pushed_1); + ASSERT(msg_pushed_2); + val = pop(tid, ds_idx); + ASSERT(val == 1); + + } else if (val == 1) { + ASSERT(msg_pushed_1); + } + stack_exit(tid); +} + +void +t2(vsize_t tid) +{ + stack_clean(tid); +} + +void +post(void) +{ +} diff --git a/verify/stack/test_case_lifo.h b/verify/stack/test_case_lifo.h new file mode 100644 index 0000000..989759c --- /dev/null +++ b/verify/stack/test_case_lifo.h @@ -0,0 +1,69 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +vsize_t ds_idx = 0; + +void +pre(void) +{ + push(NTHREADS, ds_idx, 1); +} + +void +t0(vsize_t tid) +{ + push(tid, ds_idx, 2); + push(tid, ds_idx, 3); +} + +#define NUM_OF_POPS 3 +#define NUM_POSSIBLE_OUTCOMES 10 +vuintptr_t allowed_outcomes[NUM_POSSIBLE_OUTCOMES][NUM_OF_POPS] = { + {1, 0, 0}, {1, 0, 3}, {1, 0, 2}, {1, 2, 0}, {1, 3, 2}, + {1, 2, 3}, {2, 1, 0}, {2, 1, 3}, {2, 3, 1}, {3, 2, 1}}; + +vuintptr_t outcome[NUM_OF_POPS]; + +void +t1(vsize_t tid) +{ + vbool_t full_match = false; + + + for (vsize_t j = 0; j < NUM_OF_POPS; j++) { + stack_enter(tid); + outcome[j] = pop(tid, ds_idx); + stack_exit(tid); + DBG("outcome[%zu] popped %" VUINTPTR_FORMAT " \n", j, outcome[j]); + } + + + for (vsize_t i = 0; i < NUM_POSSIBLE_OUTCOMES; i++) { + full_match = true; + + for (vsize_t j = 0; j < NUM_OF_POPS; j++) { + if (outcome[j] != allowed_outcomes[i][j]) { + full_match = false; + break; + } + } + + if (full_match) { + break; + } + } + ASSERT(full_match); +} + +void +t2(vsize_t tid) +{ + stack_clean(tid); +} + +void +post(void) +{ +} diff --git a/verify/stack/test_case_only_pop.h b/verify/stack/test_case_only_pop.h new file mode 100644 index 0000000..c66c57a --- /dev/null +++ b/verify/stack/test_case_only_pop.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +vsize_t ds_idx = 0; + +void +pre(void) +{ + for (vsize_t i = 1; i <= 4; i++) { + push(NTHREADS, ds_idx, i); + } +} + + +void +t0(vsize_t tid) +{ + stack_enter(tid); + vuintptr_t val_1 = pop(tid, ds_idx); + vuintptr_t val_2 = pop(tid, ds_idx); + stack_exit(tid); + ASSERT(val_1 > val_2); +} + +void +t1(vsize_t tid) +{ + stack_enter(tid); + vuintptr_t val_1 = pop(tid, ds_idx); + stack_exit(tid); + ASSERT(val_1); +} + +vbool_t t2_is_active = false; +void +t2(vsize_t tid) +{ +#if !defined(VSYNC_VERIFICATION_QUICK) + t2_is_active = true; + stack_enter(tid); + vuintptr_t val_1 = pop(tid, ds_idx); + stack_exit(tid); + stack_clean(tid); + ASSERT(val_1); +#endif + V_UNUSED(tid); +} + +void +post(void) +{ + vuintptr_t val = pop(NTHREADS, ds_idx); + if (t2_is_active) { + ASSERT(val == 0); + } else { + ASSERT(val == 1); + } +} diff --git a/verify/stack/test_case_only_push.h b/verify/stack/test_case_only_push.h new file mode 100644 index 0000000..612d349 --- /dev/null +++ b/verify/stack/test_case_only_push.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +vsize_t ds_idx = 0; +vuintptr_t popped[5] = {0}; + +void +pre(void) +{ +} + +void +t0(vsize_t tid) +{ + push(tid, ds_idx, 1); + push(tid, ds_idx, 2); +} + +void +t1(vsize_t tid) +{ + push(tid, ds_idx, 3); +} + +vbool_t t2_is_active = false; +void +t2(vsize_t tid) +{ +#if !defined(VSYNC_VERIFICATION_QUICK) + t2_is_active = true; + push(tid, ds_idx, 4); +#endif +} + +void +post(void) +{ + vsize_t len = t2_is_active ? 4 : 3; + + + for (vsize_t i = 0; i < len; i++) { + popped[i] = pop(NTHREADS, ds_idx); + ASSERT(popped[i]); + if (popped[i] == 1) { + vbool_t found = false; + /* if 1 is popped then then 2 have been popped ... */ + for (vsize_t j = 0; j < i; j++) { + if (2 == popped[j]) { + found = true; + } + } + ASSERT(found); + } + } + + ASSERT(pop(NTHREADS, ds_idx) == 0); +} diff --git a/verify/stack/test_case_pop_twice_push_pop.h b/verify/stack/test_case_pop_twice_push_pop.h new file mode 100644 index 0000000..cae99da --- /dev/null +++ b/verify/stack/test_case_pop_twice_push_pop.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +vsize_t ds_idx = 0; + +vuintptr_t v1 = 0; +vuintptr_t v2 = 0; +vuintptr_t v3 = 0; + +vbool_t g_push_msg = false; + +void +pre(void) +{ + push(0, ds_idx, 1); + push(0, ds_idx, 2); + push(0, ds_idx, 3); +} + +void +t0(vsize_t tid) +{ + stack_enter(tid); + v1 = pop(tid, ds_idx); + v2 = pop(tid, ds_idx); + stack_exit(tid); + + if (v2 == 4 || v1 == 4) + ASSERT(g_push_msg); + else + ASSERT(v1 > v2); +} + +void +t1(vsize_t tid) +{ + g_push_msg = true; + push(tid, ds_idx, 4); +} + +void +t2(vsize_t tid) +{ +#if defined(VSYNC_VERIFICATION_QUICK) + V_UNUSED(tid); +#else + stack_enter(tid); + v3 = pop(tid, ds_idx); + stack_exit(tid); + if (v3 == 4) + ASSERT(g_push_msg); + stack_clean(tid); +#endif +} + +void +post(void) +{ + printf("%lu %lu %lu\n", v1, v2, v3); + ASSERT(v1 != v3); + ASSERT(v2 != v3); +} diff --git a/verify/stack/test_case_popped_once.h b/verify/stack/test_case_popped_once.h new file mode 100644 index 0000000..b594ce5 --- /dev/null +++ b/verify/stack/test_case_popped_once.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +vsize_t ds_idx = 0; + +#define MAX_PUSHING 4 +int pop_counters[MAX_PUSHING] = {0}; + + +void +pre(void) +{ + for (vsize_t i = 1; i <= 3; i++) { + push(NTHREADS, ds_idx, i); + } +} + + +void +update_counters(vuintptr_t val) +{ + if (val == 0) + return; + for (vsize_t i = 0; i < MAX_PUSHING; i++) { + if (val - 1 == i) { + pop_counters[i]++; + } + } +} + +void +t0(vsize_t tid) +{ + stack_enter(tid); + vuintptr_t val1 = pop(tid, ds_idx); + + update_counters(val1); + + vuintptr_t val2 = pop(tid, ds_idx); + stack_exit(tid); + + update_counters(val2); + + if (val2 != 4) { + ASSERT(val1 > val2); + } +} + +void +t1(vsize_t tid) +{ + stack_enter(tid); + vuintptr_t val = pop(tid, ds_idx); + update_counters(val); + stack_exit(tid); + + push(tid, ds_idx, 4); +} + + +void +t2(vsize_t tid) +{ +#if defined(VSYNC_VERIFICATION_QUICK) + V_UNUSED(tid); +#else + stack_clean(tid); +#endif +} + +void +post(void) +{ + vsize_t count_not_popped = 0; + for (vsize_t i = 0; i < MAX_PUSHING; i++) { + ASSERT(pop_counters[i] <= 1); + if (pop_counters[i] == 0) { + count_not_popped++; + } + } + ASSERT(count_not_popped == 1); +} diff --git a/verify/stack/verify_stack.c b/verify/stack/verify_stack.c new file mode 100644 index 0000000..f832942 --- /dev/null +++ b/verify/stack/verify_stack.c @@ -0,0 +1,72 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + + +#define NTHREADS 3 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void *run(void *args); +void t0(vsize_t); +void t1(vsize_t); +void t2(vsize_t); +void pre(void); +void post(void); +void init(void); + +#ifdef TEST_CASE + #include TEST_CASE +#else + #error "no test case was defined" +#endif + +int +main(void) +{ + init(); + + pre(); + launch_threads(NTHREADS, run); + post(); + +// This has to be before destroy +#ifndef SKIP_VERIFY + verify(0); +#endif + destroy(); +} + +void * +run(void *args) +{ + vsize_t tid = (vsize_t)(vuintptr_t)args; + ASSERT(tid < NTHREADS); + + reg(tid); + switch (tid) { + case 0: + t0(tid); + break; + case 1: + t1(tid); + break; + case 2: + t2(tid); + break; + default: + break; + } + dereg(tid); + return NULL; +} diff --git a/verify/thread/CMakeLists.txt b/verify/thread/CMakeLists.txt new file mode 100644 index 0000000..fbb22f2 --- /dev/null +++ b/verify/thread/CMakeLists.txt @@ -0,0 +1,15 @@ +file(GLOB SRCS *.c) +foreach(SRC ${SRCS}) + get_filename_component(TEST ${SRC} NAME_WE) + + add_executable(${TEST} ${SRC}) + target_link_libraries(${TEST} vsync pthread) + target_include_directories(${TEST} PRIVATE include) + v_add_bin_test(NAME ${TEST} COMMAND ${TEST}) + + add_vsyncer_check( + TARGET ${TEST}-check + SOURCE ${SRC} + TIMEOUT 20 + DEPENDENCIES vsync) +endforeach() diff --git a/verify/thread/cnd_test1.c b/verify/thread/cnd_test1.c new file mode 100644 index 0000000..bdde32c --- /dev/null +++ b/verify/thread/cnd_test1.c @@ -0,0 +1,50 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +#include "mock_mutex.h" +#include +#include + +#ifdef VSYNC_VERIFICATION_QUICK + #define NTHREADS 2 +#else + #define NTHREADS 3 +#endif + +vcond_t g_cond; +vmutex_t g_mutex; +int g_cs_x, g_cs_y; + +void * +run(void *arg) +{ + vmutex_acquire(&g_mutex); + if (g_cs_x++ < NTHREADS - 1) { + while (g_cs_x != NTHREADS) { + vcond_wait(&g_cond, &g_mutex); + } + g_cs_y++; + } else { + g_cs_y -= NTHREADS; + } + vmutex_release(&g_mutex); + vcond_signal(&g_cond); + + V_UNUSED(arg); + return NULL; +} + +int +main(void) +{ + launch_threads(NTHREADS, run); + ASSERT(g_cs_x == NTHREADS); + ASSERT(g_cs_y == -1); + return 0; +} diff --git a/verify/thread/cnd_test2.c b/verify/thread/cnd_test2.c new file mode 100644 index 0000000..dd545fc --- /dev/null +++ b/verify/thread/cnd_test2.c @@ -0,0 +1,44 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +#include "mock_mutex.h" +#include +#include + +#ifdef VSYNC_VERIFICATION_QUICK + #define NTHREADS 2U +#else + #define NTHREADS 4U +#endif + +vcond_t g_cond; +vmutex_t g_mutex; +vuint32_t g_shared; + +void * +run(void *arg) +{ + vsize_t i = (vsize_t)(vuintptr_t)arg; + vmutex_acquire(&g_mutex); + g_shared++; + while (i == 0 && g_shared != NTHREADS) { + vcond_wait(&g_cond, &g_mutex); + } + vmutex_release(&g_mutex); + vcond_signal(&g_cond); + return NULL; +} + +int +main(void) +{ + launch_threads(NTHREADS, run); + ASSERT(g_shared == NTHREADS); + return 0; +} diff --git a/verify/thread/mock_mutex.h b/verify/thread/mock_mutex.h new file mode 100644 index 0000000..10cd1c4 --- /dev/null +++ b/verify/thread/mock_mutex.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef _MOCK_MUTEX_ +#define _MOCK_MUTEX_ + +#if 1 + #include + +typedef pthread_mutex_t vmutex_t; + +static inline void +vmutex_acquire(vmutex_t *l) +{ + pthread_mutex_lock(l); +} +static inline void +vmutex_release(vmutex_t *l) +{ + pthread_mutex_unlock(l); +} +#else + #include + +typedef caslock_t vmutex_t; + +static inline void +vmutex_acquire(vmutex_t *l) +{ + caslock_acquire(l); +} +static inline void +vmutex_release(vmutex_t *l) +{ + caslock_release(l); +} +#endif +#endif diff --git a/verify/thread/mutex_musl.c b/verify/thread/mutex_musl.c new file mode 100644 index 0000000..4664fc5 --- /dev/null +++ b/verify/thread/mutex_musl.c @@ -0,0 +1,29 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifdef VSYNC_VERIFICATION_QUICK + #define NTHREADS 2 +#else + #define NTHREADS 3 +#endif + +#include +#include + +vmutex_t lock; + +void +acquire(vuint32_t tid) +{ + V_UNUSED(tid); + vmutex_acquire(&lock); +} + +void +release(vuint32_t tid) +{ + V_UNUSED(tid); + vmutex_release(&lock); +} diff --git a/verify/thread/mutex_slim.c b/verify/thread/mutex_slim.c new file mode 100644 index 0000000..d4dd1d4 --- /dev/null +++ b/verify/thread/mutex_slim.c @@ -0,0 +1,29 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifdef VSYNC_VERIFICATION_QUICK + #define NTHREADS 2 +#else + #define NTHREADS 3 +#endif + +#include +#include + +vmutex_t lock; + +void +acquire(vuint32_t tid) +{ + V_UNUSED(tid); + vmutex_acquire(&lock); +} + +void +release(vuint32_t tid) +{ + V_UNUSED(tid); + vmutex_release(&lock); +} diff --git a/verify/thread/mutex_tristate.c b/verify/thread/mutex_tristate.c new file mode 100644 index 0000000..1f75b77 --- /dev/null +++ b/verify/thread/mutex_tristate.c @@ -0,0 +1,23 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include + +vmutex_t lock; + +void +acquire(vuint32_t tid) +{ + V_UNUSED(tid); + vmutex_acquire(&lock); +} + +void +release(vuint32_t tid) +{ + V_UNUSED(tid); + vmutex_release(&lock); +} diff --git a/verify/thread/once.c b/verify/thread/once.c new file mode 100644 index 0000000..549c788 --- /dev/null +++ b/verify/thread/once.c @@ -0,0 +1,50 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include + +#define NTHREADS 4U + +vonce_t g_once = VONCE_INIT(); +vsize_t g_winner = 0; + +void * +__once_verification_cb(void *arg) +{ + g_winner = (vsize_t)(vuintptr_t)arg; + ASSERT(g_winner != 0); + return arg; +} + +void * +run(void *arg) +{ + vsize_t tid = ((vsize_t)(vuintptr_t)arg) + 1U; + ASSERT(tid != 0); + + void *r = + vonce_call(&g_once, __once_verification_cb, (void *)(vuintptr_t)tid); + + if (r != NULL) { + /* if once returns something, then I was the winner */ + ASSERT(tid == (vsize_t)(vuintptr_t)r); + ASSERT(g_winner == tid); + } else { + /* if once returned nothing, then somebody else is the winner */ + ASSERT(g_winner != tid); + } + return NULL; +} + +int +main(void) +{ + launch_threads(NTHREADS, run); + return 0; +} diff --git a/verify/treeset/CMakeLists.txt b/verify/treeset/CMakeLists.txt new file mode 100644 index 0000000..daa29f3 --- /dev/null +++ b/verify/treeset/CMakeLists.txt @@ -0,0 +1,36 @@ +file(GLOB TEST_CASES testcase_*.h) + +set(VERIFY_FILE verify_treeset.c) + +set(NUM_THREADS 4) + +set(ALGOS BST_FINE RB_FINE BST_COARSE RB_COARSE) +set(TEST_DEFS TREESET_LOCK_TTAS) +set(CHECK_FLAGS -DVSYNC_VERIFICATION -DVMEM_LIB_ALLOC_TRACKING_OFF) + +foreach(test_path IN ITEMS ${TEST_CASES}) + get_filename_component(test_name ${test_path} NAME) + get_filename_component(test_prefix ${test_path} NAME_WE) + + set(tc TEST_CASE="${test_name}") + set(tc_check -DTEST_CASE="'\"${test_name}\"'") + + foreach(algo IN ITEMS ${ALGOS}) + set(TEST treeset_${test_prefix}_${algo}) + string(TOLOWER ${TEST} TEST) + add_executable(${TEST} ${VERIFY_FILE}) + target_link_libraries(${TEST} vsync pthread) + target_compile_definitions(${TEST} PUBLIC TREESET_${algo} ${TEST_DEFS} + ${tc}) + add_test(NAME ${TEST} COMMAND ${TEST}) + set_tests_properties(${TEST} PROPERTIES TIMEOUT 10) + + add_vsyncer_check( + TARGET ${TEST}_check + SOURCE ${VERIFY_FILE} + CFLAGS -DTREESET_${algo} ${tc_check} ${CHECK_FLAGS} + TIMEOUT 240 + DEPENDENCIES vsync) + endforeach() + +endforeach() diff --git a/verify/treeset/testcase_add_1.h b/verify/treeset/testcase_add_1.h new file mode 100644 index 0000000..a96dd09 --- /dev/null +++ b/verify/treeset/testcase_add_1.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Concurrent adds with different keys + +void +tinit(void) +{ +} + +void +t0(void) +{ + ASSERT(!tr_con(0)); + ASSERT(tr_add(0)); + ASSERT(tr_con(0)); +} + +void +t1(void) +{ + ASSERT(!tr_con(1)); + ASSERT(tr_add(1)); + ASSERT(tr_con(1)); +} + +void +t2(void) +{ + ASSERT(!tr_con(2)); + ASSERT(tr_add(2)); + ASSERT(tr_con(2)); +} + +void +t3(void) +{ +} + +void +tfini(void) +{ +} diff --git a/verify/treeset/testcase_add_2.h b/verify/treeset/testcase_add_2.h new file mode 100644 index 0000000..8715763 --- /dev/null +++ b/verify/treeset/testcase_add_2.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Concurrent adds with the same key + +int r0, r1, r2; + +void +tinit(void) +{ +} + +void +t0(void) +{ + r0 = tr_add(0); +} + +void +t1(void) +{ + r1 = tr_add(0); +} + +void +t2(void) +{ + r2 = tr_add(0); +} + +void +t3(void) +{ + tr_add(-1); + tr_add(1); + tr_add(-2); + tr_add(2); +} + +void +tfini(void) +{ + ASSERT(r0 + r1 + r2 == 1); +} diff --git a/verify/treeset/testcase_add_3.h b/verify/treeset/testcase_add_3.h new file mode 100644 index 0000000..827a294 --- /dev/null +++ b/verify/treeset/testcase_add_3.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Linearizability of adds from the same thread + +void +tinit(void) +{ +} + +void +t0(void) +{ + ASSERT(tr_add(0)); + ASSERT(tr_add(1)); + ASSERT(tr_add(2)); +} + +void +t1(void) +{ + if (tr_con(1)) { + ASSERT(tr_con(0)); + } + if (tr_con(2)) { + ASSERT(tr_con(0)); + ASSERT(tr_con(1)); + } +} + +void +t2(void) +{ +} + +void +t3(void) +{ +} + +void +tfini(void) +{ +} diff --git a/verify/treeset/testcase_add_4.h b/verify/treeset/testcase_add_4.h new file mode 100644 index 0000000..fb9960a --- /dev/null +++ b/verify/treeset/testcase_add_4.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Forcing execution of complicated cases when adding + +void +tinit(void) +{ +} + +void +t0(void) +{ + ASSERT(tr_add(0)); + ASSERT(tr_add(3)); + ASSERT(tr_add(6)); +} + +void +t1(void) +{ + ASSERT(tr_add(1)); + ASSERT(tr_add(4)); + ASSERT(tr_add(7)); +} + +void +t2(void) +{ + ASSERT(tr_add(2)); + ASSERT(tr_add(5)); + ASSERT(tr_add(8)); +} + +void +t3(void) +{ +} + +void +tfini(void) +{ +} diff --git a/verify/treeset/testcase_add_5.h b/verify/treeset/testcase_add_5.h new file mode 100644 index 0000000..f376ea8 --- /dev/null +++ b/verify/treeset/testcase_add_5.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Linearizability of adds from different threads + +int r2, r3; + +void +tinit(void) +{ +} + +void +t0(void) +{ + ASSERT(tr_add(0)); +} + +void +t1(void) +{ + ASSERT(tr_add(1)); +} + +void +t2(void) +{ + r2 = tr_con(0) && !tr_con(1); +} + +void +t3(void) +{ + r3 = tr_con(1) && !tr_con(0); +} + +void +tfini(void) +{ + ASSERT(!(r2 && r3)); +} diff --git a/verify/treeset/testcase_add_rem_1.h b/verify/treeset/testcase_add_rem_1.h new file mode 100644 index 0000000..29e70a4 --- /dev/null +++ b/verify/treeset/testcase_add_rem_1.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Concurrent adds/removes with same/different keys + +int r00, r01, r10, r11, r20, r21, r30, r31; + +void +tinit(void) +{ +} + +void +t0(void) +{ + r00 = tr_add(0); + r01 = tr_add(1); +} + +void +t1(void) +{ + r10 = tr_add(0); + r11 = tr_add(1); +} + +void +t2(void) +{ + r20 = tr_rem(0); + r21 = tr_rem(1); +} + +void +t3(void) +{ + r30 = tr_rem(0); + r31 = tr_rem(1); +} + +void +tfini(void) +{ + int con0 = r00 + r10 - r20 - r30; + int con1 = r01 + r11 - r21 - r31; + int real_con0 = tr_con(0); + int real_con1 = tr_con(1); + ASSERT(con0 == real_con0); + ASSERT(con1 == real_con1); +} diff --git a/verify/treeset/testcase_add_rem_2.h b/verify/treeset/testcase_add_rem_2.h new file mode 100644 index 0000000..a001597 --- /dev/null +++ b/verify/treeset/testcase_add_rem_2.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Linearizability of adds/removes from the same thread + +void +tinit(void) +{ + tr_add(1); + tr_add(3); +} + +void +t0(void) +{ + ASSERT(tr_add(0)); + ASSERT(tr_add(2)); +} + +void +t1(void) +{ + if (tr_rem(2)) { + ASSERT(tr_rem(0)); + } else { + verification_ignore(); + } +} + +void +t2(void) +{ + ASSERT(tr_rem(1)); + ASSERT(tr_rem(3)); +} + +void +t3(void) +{ + if (tr_add(3)) { + ASSERT(tr_add(1)); + } else { + verification_ignore(); + } +} + +void +tfini(void) +{ +} diff --git a/verify/treeset/testcase_add_rem_3.h b/verify/treeset/testcase_add_rem_3.h new file mode 100644 index 0000000..bfe78ab --- /dev/null +++ b/verify/treeset/testcase_add_rem_3.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Linearizability of adds/removes from the same thread + +void +tinit(void) +{ + tr_add(0); + tr_add(3); +} + +void +t0(void) +{ + ASSERT(tr_rem(0)); + ASSERT(tr_add(2)); +} + +void +t1(void) +{ + if (tr_rem(2)) { + ASSERT(tr_add(0)); + } else { + verification_ignore(); + } +} + +void +t2(void) +{ + ASSERT(tr_add(1)); + ASSERT(tr_rem(3)); +} + +void +t3(void) +{ + if (tr_add(3)) { + ASSERT(tr_rem(1)); + } else { + verification_ignore(); + } +} + +void +tfini(void) +{ +} diff --git a/verify/treeset/testcase_add_rem_4.h b/verify/treeset/testcase_add_rem_4.h new file mode 100644 index 0000000..e6e5bd6 --- /dev/null +++ b/verify/treeset/testcase_add_rem_4.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Concurrent adds/removes with same/different keys + +void +tinit(void) +{ + tr_add(2); + tr_add(3); +} + +void +t0(void) +{ + ASSERT(tr_add(0)); + tr_add(2); +} + +void +t1(void) +{ + tr_add(3); + ASSERT(tr_add(1)); +} + +void +t2(void) +{ + tr_rem(0); + ASSERT(tr_rem(2)); +} + +void +t3(void) +{ + ASSERT(tr_rem(3)); + tr_rem(1); +} + +void +tfini(void) +{ +} diff --git a/verify/treeset/testcase_add_rem_5.h b/verify/treeset/testcase_add_rem_5.h new file mode 100644 index 0000000..ff15da6 --- /dev/null +++ b/verify/treeset/testcase_add_rem_5.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Linearizability of add/remove from different threads + +int r2, r3; + +void +tinit(void) +{ + tr_add(1); +} + +void +t0(void) +{ + ASSERT(tr_add(0)); +} + +void +t1(void) +{ + ASSERT(tr_rem(1)); +} + +void +t2(void) +{ + r2 = tr_con(0) && tr_con(1); +} + +void +t3(void) +{ + r3 = !tr_con(1) && !tr_con(0); +} + +void +tfini(void) +{ + ASSERT(!(r2 && r3)); +} diff --git a/verify/treeset/testcase_add_rem_6.h b/verify/treeset/testcase_add_rem_6.h new file mode 100644 index 0000000..e8ad56f --- /dev/null +++ b/verify/treeset/testcase_add_rem_6.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Forcing execution of complicated cases when adding/removing + +void +tinit(void) +{ + ASSERT(tr_add(0)); + ASSERT(tr_add(4)); + ASSERT(tr_add(8)); + ASSERT(tr_add(9)); + ASSERT(tr_add(-1)); +} + +void +t0(void) +{ + ASSERT(tr_add(3)); + ASSERT(tr_add(6)); + ASSERT(tr_rem(0)); +} + +void +t1(void) +{ + ASSERT(tr_add(1)); + ASSERT(tr_add(7)); + ASSERT(tr_rem(4)); +} + +void +t2(void) +{ + ASSERT(tr_add(2)); + ASSERT(tr_add(5)); + ASSERT(tr_rem(8)); +} + +void +t3(void) +{ +} + +void +tfini(void) +{ +} diff --git a/verify/treeset/testcase_rem_1.h b/verify/treeset/testcase_rem_1.h new file mode 100644 index 0000000..db2e1f9 --- /dev/null +++ b/verify/treeset/testcase_rem_1.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Concurrent removes with different keys + +void +tinit(void) +{ + tr_add(0); + tr_add(1); + tr_add(2); +} + +void +t0(void) +{ + ASSERT(tr_con(0)); + ASSERT(tr_rem(0)); + ASSERT(!tr_con(0)); +} + +void +t1(void) +{ + ASSERT(tr_con(1)); + ASSERT(tr_rem(1)); + ASSERT(!tr_con(1)); +} + +void +t2(void) +{ + ASSERT(tr_con(2)); + ASSERT(tr_rem(2)); + ASSERT(!tr_con(2)); +} + +void +t3(void) +{ +} + +void +tfini(void) +{ +} diff --git a/verify/treeset/testcase_rem_2.h b/verify/treeset/testcase_rem_2.h new file mode 100644 index 0000000..c6f5711 --- /dev/null +++ b/verify/treeset/testcase_rem_2.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Concurrent removes with the same key + +int r0, r1, r2; + +void +tinit(void) +{ + tr_add(0); +} + +void +t0(void) +{ + r0 = tr_rem(0); +} + +void +t1(void) +{ + r1 = tr_rem(0); +} + +void +t2(void) +{ + r2 = tr_rem(0); +} + +void +t3(void) +{ + tr_add(-1); + tr_add(1); + tr_add(-2); + tr_add(2); +} + +void +tfini(void) +{ + ASSERT(r0 + r1 + r2 == 1); +} diff --git a/verify/treeset/testcase_rem_3.h b/verify/treeset/testcase_rem_3.h new file mode 100644 index 0000000..8924ee1 --- /dev/null +++ b/verify/treeset/testcase_rem_3.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Linearizability of removes from the same thread + +void +tinit(void) +{ + tr_add(0); + tr_add(1); + tr_add(2); +} + +void +t0(void) +{ + ASSERT(tr_rem(0)); + ASSERT(tr_rem(1)); + ASSERT(tr_rem(2)); +} + +void +t1(void) +{ + if (!tr_con(1)) { + ASSERT(!tr_con(0)); + } + if (!tr_con(2)) { + ASSERT(!tr_con(0)); + ASSERT(!tr_con(1)); + } +} + +void +t2(void) +{ +} + +void +t3(void) +{ +} + +void +tfini(void) +{ +} diff --git a/verify/treeset/testcase_rem_4.h b/verify/treeset/testcase_rem_4.h new file mode 100644 index 0000000..87a7c20 --- /dev/null +++ b/verify/treeset/testcase_rem_4.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Forcing execution of complicated cases when removing + +void +tinit(void) +{ + for (int i = 0; i < 9; ++i) { + tr_add(i); + } +} + +void +t0(void) +{ + ASSERT(tr_rem(0)); + ASSERT(tr_rem(3)); + ASSERT(tr_rem(6)); +} + +void +t1(void) +{ + ASSERT(tr_rem(1)); + ASSERT(tr_rem(4)); + ASSERT(tr_rem(7)); +} + +void +t2(void) +{ + ASSERT(tr_rem(2)); + ASSERT(tr_rem(5)); + ASSERT(tr_rem(8)); +} + +void +t3(void) +{ +} + +void +tfini(void) +{ +} diff --git a/verify/treeset/testcase_rem_5.h b/verify/treeset/testcase_rem_5.h new file mode 100644 index 0000000..9912281 --- /dev/null +++ b/verify/treeset/testcase_rem_5.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +// Linearizability of removes from different threads + +int r2, r3; + +void +tinit(void) +{ + tr_add(0); + tr_add(1); +} + +void +t0(void) +{ + ASSERT(tr_rem(0)); +} + +void +t1(void) +{ + ASSERT(tr_rem(1)); +} + +void +t2(void) +{ + r2 = !tr_con(0) && tr_con(1); +} + +void +t3(void) +{ + r3 = !tr_con(1) && tr_con(0); +} + +void +tfini(void) +{ + ASSERT(!(r2 && r3)); +} diff --git a/verify/treeset/verify_treeset.c b/verify/treeset/verify_treeset.c new file mode 100644 index 0000000..9a1bf94 --- /dev/null +++ b/verify/treeset/verify_treeset.c @@ -0,0 +1,56 @@ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#include +#include + +#include +#include + +#define NTHREADS 4 + +#ifdef TEST_CASE + #include TEST_CASE +#else + #error "Choose testcase by setting TEST_CASE" +#endif + +void * +run(void *arg) +{ + vsize_t tid = (vsize_t)(vuintptr_t)arg; + ASSERT(tid < NTHREADS); + + switch (tid) { + case 0: + t0(); + break; + case 1: + t1(); + break; + case 2: + t2(); + break; + case 3: + t3(); + break; + } + + return NULL; +} + +int +main(void) +{ + tr_init(); + + tinit(); + + launch_threads(NTHREADS, run); + + tfini(); + + tr_destroy(); +} diff --git a/verify/unbounded_queue/CMakeLists.txt b/verify/unbounded_queue/CMakeLists.txt index 428b0ab..c0e5ef9 100644 --- a/verify/unbounded_queue/CMakeLists.txt +++ b/verify/unbounded_queue/CMakeLists.txt @@ -19,6 +19,9 @@ endif() # verification template file set(VERIFY_FILE unbounded_queue_verify.c) +# Enable some tests with VMM and set extra options for Dartagnan +set(DAT3M_BOUND_VQUEUE_UB_TOTAL 4) + foreach(test_path IN ITEMS ${TEST_CASES}) # extract test_name with extension get_filename_component(test_name ${test_path} NAME) @@ -60,6 +63,18 @@ foreach(test_path IN ITEMS ${TEST_CASES}) TIMEOUT 1200 # ${CHECK_OPTIONS}) + if(${DAT3M_BOUND_${algo}}) + add_vsyncer_check( + TARGET check_${TEST} + SOURCE ${VERIFY_FILE} + CFLAGS -D${algo} ${TEST_DEFS} ${tc_check} USE_DAT3M + MEMORY_MODELS vmm DARTAGNAN_OPTIONS + --bound=${DAT3M_BOUND_${algo}} + DEPENDENCIES vsync + TIMEOUT 1200 # + ${CHECK_OPTIONS}) + endif() + endforeach() endforeach() diff --git a/vmm/vmm.cat b/vmm/vmm.cat index 8732edc..4e35202 100644 --- a/vmm/vmm.cat +++ b/vmm/vmm.cat @@ -2,7 +2,7 @@ * Copyright (C) Huawei Technologies Co., Ltd. 2024. All rights reserved. * SPDX-License-Identifier: MIT * - * Version 0.9.1 + * Version 0.9.2 */ (* Notation @@ -70,8 +70,8 @@ acyclic co | rf | fr | po-loc let Marked = RLX | ACQ | REL | SC let Plain = ~Marked -let Acq = ACQ | (SC & R) -let Rel = REL | (SC & W) +let Acq = ACQ | (SC & R) | (SC & F) +let Rel = REL | (SC & W) | (SC & F) (** Ordering **) (* In our model, dependencies only order stores: