From b831d06f22c41643694be58a2a0d60c869cf6c87 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sat, 26 Aug 2023 18:43:28 +0800 Subject: [PATCH] Add a concurrent map --- .clang-format | 3 + README.md | 3 +- cmap/Makefile | 7 ++ cmap/cmap.c | 262 +++++++++++++++++++++++++++++++++++++++++++++++ cmap/cmap.h | 83 +++++++++++++++ cmap/hash.h | 76 ++++++++++++++ cmap/list.h | 90 ++++++++++++++++ cmap/locks.h | 165 +++++++++++++++++++++++++++++ cmap/perf.h | 29 ++++++ cmap/random.c | 45 ++++++++ cmap/random.h | 10 ++ cmap/rcu.c | 84 +++++++++++++++ cmap/rcu.h | 49 +++++++++ cmap/test-cmap.c | 209 +++++++++++++++++++++++++++++++++++++ cmap/util.c | 19 ++++ cmap/util.h | 149 +++++++++++++++++++++++++++ 16 files changed, 1282 insertions(+), 1 deletion(-) create mode 100644 cmap/Makefile create mode 100644 cmap/cmap.c create mode 100644 cmap/cmap.h create mode 100644 cmap/hash.h create mode 100644 cmap/list.h create mode 100644 cmap/locks.h create mode 100644 cmap/perf.h create mode 100644 cmap/random.c create mode 100644 cmap/random.h create mode 100644 cmap/rcu.c create mode 100644 cmap/rcu.h create mode 100644 cmap/test-cmap.c create mode 100644 cmap/util.c create mode 100644 cmap/util.h diff --git a/.clang-format b/.clang-format index 8f3dd33..d553e1b 100644 --- a/.clang-format +++ b/.clang-format @@ -26,3 +26,6 @@ ForEachMacros: - rb_list_foreach_safe - EV_FOREACH - LIST_FOREACH + - LIST_FOREACH_POP + - MAP_FOREACH + - MAP_FOREACH_WITH_HASH diff --git a/README.md b/README.md index 7b3af07..a8ec43f 100644 --- a/README.md +++ b/README.md @@ -31,12 +31,13 @@ purpose of these programs is to be illustrative and educational. * [Synchronization](https://en.wikipedia.org/wiki/Synchronization_(computer_science)) - [mcslock](mcslock/): An MCS lock implementation. - [seqlock](seqlock/): A seqlock (sequence lock) implementation. - - [hp\_list](hp_list)/: A concurrent linked list utilizing Hazard Pointers. + - [hp\_list](hp_list/)/: A concurrent linked list utilizing Hazard Pointers. - [rcu-list](rcu-list/): A concurrent linked list utilizing the simplified RCU algorithm. - [qsbr](qsbr/): An implementation of Quiescent state based reclamation (QSBR). - [list-move](list-move/): Evaluation of two concurrent linked lists: QSBR and lock-based. - [rcu\_queue](rcu_queue/): An efficient concurrent queue based on QSBR. - [thread-rcu](thread-rcu/): A Linux Kernel style thread-based simple RCU. + - [cmap](cmap/): A concurrent map implementation based on RCU. * Applications - [httpd](httpd/): A multi-threaded web server. - [map-reduce](map-reduce/): word counting using MapReduce. diff --git a/cmap/Makefile b/cmap/Makefile new file mode 100644 index 0000000..5826f46 --- /dev/null +++ b/cmap/Makefile @@ -0,0 +1,7 @@ +all: + gcc -Wall -O2 -o test-cmap \ + cmap.c random.c rcu.c test-cmap.c \ + -lpthread + +clean: + rm -f test-cmap diff --git a/cmap/cmap.c b/cmap/cmap.c new file mode 100644 index 0000000..130362c --- /dev/null +++ b/cmap/cmap.c @@ -0,0 +1,262 @@ +#include "cmap.h" +#include +#include +#include +#include "locks.h" +#include "util.h" + +#define MAP_INITIAL_SIZE 512 + +struct cmap_entry { + struct cmap_node *first; +}; + +struct cmap_impl { + struct cmap_entry *arr; /* Map entreis */ + size_t count; /* Number of elements in this */ + size_t max; /* Capacity of this */ + size_t utilization; /* Number of utialized entries */ + struct cond fence; /* Prevent new reads while old still exist */ +}; + +struct cmap_impl_pair { + struct cmap_impl *old, *new; +}; + +static void cmap_expand(struct cmap *cmap); +static void cmap_expand_callback(void *args); +static void cmap_destroy_callback(void *args); +static size_t cmap_count__(const struct cmap *cmap); +static void cmap_insert__(struct cmap_impl *, struct cmap_node *); + +/* Only a single concurrent writer to cmap is allowed */ +static void cmap_insert__(struct cmap_impl *impl, struct cmap_node *node) +{ + size_t i = node->hash & impl->max; + node->next = impl->arr[i].first; + if (!impl->arr[i].first) + impl->utilization++; + impl->arr[i].first = node; +} + +static void cmap_destroy_callback(void *args) +{ + struct cmap_impl *impl = (struct cmap_impl *) args; + cond_destroy(&impl->fence); + free(impl); +} + +static struct cmap_impl *cmap_impl_init(size_t entry_num) +{ + size_t size = + sizeof(struct cmap_impl) + sizeof(struct cmap_entry) * entry_num; + struct cmap_impl *impl = xmalloc(size); + impl->max = entry_num - 1; + impl->count = 0; + impl->utilization = 0; + impl->arr = OBJECT_END(struct cmap_entry *, impl); + cond_init(&impl->fence); + + for (int i = 0; i < entry_num; ++i) + impl->arr[i].first = NULL; + return impl; +} + +static void cmap_expand_callback(void *args) +{ + struct cmap_impl_pair *pair = (struct cmap_impl_pair *) args; + struct cmap_node *c, *n; + + /* Rehash */ + for (int i = 0; i <= pair->old->max; i++) { + for (c = pair->old->arr[i].first; c; c = n) { + n = c->next; + cmap_insert__(pair->new, c); + } + } + + /* Remove fence */ + cond_unlock(&pair->new->fence); + free(pair->old); + free(pair); +} + +/* Only a single concurrent writer to cmap is allowed */ +static void cmap_expand(struct cmap *cmap) +{ + struct rcu *impl_rcu = rcu_acquire(cmap->impl->p); + struct cmap_impl_pair *pair = xmalloc(sizeof(*pair)); + pair->old = rcu_get(impl_rcu, struct cmap_impl *); + + /* Do not allow two expansions in parallel */ + /* Prevent new reads while old still exist */ + while (cond_is_locked(&pair->old->fence)) { + rcu_release(impl_rcu); + cond_wait(&pair->old->fence); + impl_rcu = rcu_acquire(cmap->impl->p); + pair->old = rcu_get(impl_rcu, struct cmap_impl *); + } + + /* Initiate new rehash array */ + pair->new = cmap_impl_init((pair->old->max + 1) * 2); + pair->new->count = pair->old->count; + + /* Prevent new reads/updates while old reads still exist */ + cond_lock(&pair->new->fence); + + rcu_postpone(impl_rcu, cmap_expand_callback, pair); + rcu_release(impl_rcu); + rcu_set(cmap->impl->p, pair->new); +} + +void cmap_init(struct cmap *cmap) +{ + struct cmap_impl *impl = cmap_impl_init(MAP_INITIAL_SIZE); + cmap->impl = xmalloc(sizeof(*cmap->impl)); + rcu_init(cmap->impl->p, impl); +} + +void cmap_destroy(struct cmap *cmap) +{ + if (!cmap) + return; + + struct rcu *impl_rcu = rcu_acquire(cmap->impl->p); + struct cmap_impl *impl = rcu_get(impl_rcu, struct cmap_impl *); + rcu_postpone(impl_rcu, cmap_destroy_callback, impl); + rcu_release(impl_rcu); + rcu_destroy(impl_rcu); + free(cmap->impl); +} + +static size_t cmap_count__(const struct cmap *cmap) +{ + struct rcu *impl_rcu = rcu_acquire(cmap->impl->p); + struct cmap_impl *impl = rcu_get(impl_rcu, struct cmap_impl *); + size_t count = impl->count; + rcu_release(impl_rcu); + return count; +} + +double cmap_utilization(const struct cmap *cmap) +{ + struct rcu *impl_rcu = rcu_acquire(cmap->impl->p); + struct cmap_impl *impl = rcu_get(impl_rcu, struct cmap_impl *); + double res = (double) impl->utilization / (impl->max + 1); + rcu_release(impl_rcu); + return res; +} + +size_t cmap_size(const struct cmap *cmap) +{ + return cmap_count__(cmap); +} + +/* Only one concurrent writer */ +size_t cmap_insert(struct cmap *cmap, struct cmap_node *node, uint32_t hash) +{ + node->hash = hash; + + struct rcu *impl_rcu = rcu_acquire(cmap->impl->p); + struct cmap_impl *impl = rcu_get(impl_rcu, struct cmap_impl *); + cmap_insert__(impl, node); + impl->count++; + size_t count = impl->count; + bool expand = impl->count > impl->max * 2; + rcu_release(impl_rcu); + + if (expand) + cmap_expand(cmap); + return count; +} + +/* Only one concurrent writer */ +size_t cmap_remove(struct cmap *cmap, struct cmap_node *node) +{ + struct rcu *impl_rcu = rcu_acquire(cmap->impl->p); + struct cmap_impl *impl = rcu_get(impl_rcu, struct cmap_impl *); + size_t pos = node->hash & impl->max; + struct cmap_entry *cmap_entry = &impl->arr[pos]; + size_t count = impl->count; + + struct cmap_node **node_p = &cmap_entry->first; + while (*node_p) { + if (*node_p == node) { + *node_p = node->next; + count--; + break; + } + node_p = &(*node_p)->next; + } + impl->count = count; + rcu_release(impl_rcu); + return count; +} + +struct cmap_state cmap_state_acquire(struct cmap *cmap) +{ + struct cmap_state state = {.p = rcu_acquire(cmap->impl->p)}; + return state; +} + +void cmap_state_release(struct cmap_state state) +{ + rcu_release(state.p); +} + +struct cmap_cursor cmap_find__(struct cmap_state state, uint32_t hash) +{ + struct cmap_impl *impl = rcu_get(state.p, struct cmap_impl *); + + /* Prevent new reads while old still exist */ + while (cond_is_locked(&impl->fence)) + cond_wait(&impl->fence); + + struct cmap_cursor cursor = { + .entry_idx = hash & impl->max, + .node = impl->arr[hash & impl->max].first, + .next = NULL, + .accross_entries = false, + }; + if (cursor.node) + cursor.next = cursor.node->next; + return cursor; +} + +struct cmap_cursor cmap_start__(struct cmap_state state) +{ + struct cmap_cursor cursor = cmap_find__(state, 0); + cursor.accross_entries = true; + /* Don't start with an empty node */ + if (!cursor.node) + cmap_next__(state, &cursor); + return cursor; +} + +void cmap_next__(struct cmap_state state, struct cmap_cursor *cursor) +{ + struct cmap_impl *impl = rcu_get(state.p, struct cmap_impl *); + + cursor->node = cursor->next; + if (cursor->node) { + cursor->next = cursor->node->next; + return; + } + + /* We got to the end of the current entry. Try to find + * a valid node in next entries + */ + while (cursor->accross_entries) { + cursor->entry_idx++; + if (cursor->entry_idx > impl->max) + break; + cursor->node = impl->arr[cursor->entry_idx].first; + if (cursor->node) { + cursor->next = cursor->node->next; + return; + } + } + + cursor->node = NULL; + cursor->next = NULL; +} diff --git a/cmap/cmap.h b/cmap/cmap.h new file mode 100644 index 0000000..bdbd4bf --- /dev/null +++ b/cmap/cmap.h @@ -0,0 +1,83 @@ +#pragma once + +#include +#include +#include + +#include "rcu.h" +#include "util.h" + +/* Concurrent cmap. + * It supports multiple concurrent readers and a single concurrent writer. + * To iterate, the user needs to acquire a "cmap state" (snapshot). + */ +struct cmap_node { + struct cmap_node *next; /* Next node with same hash. */ + uint32_t hash; +}; + +/* Used for going over all cmap nodes */ +struct cmap_cursor { + struct cmap_node *node; /* Pointer to cmap_node */ + struct cmap_node *next; /* Pointer to cmap_node */ + size_t entry_idx; /* Current entry */ + bool accross_entries; /* Hold cursor accross cmap entries */ +}; + +/* Map state (snapshot), must be acquired before cmap iteration, and released + * afterwards. + */ +struct cmap_state { + struct rcu *p; +}; + +/* Concurrent hash cmap. */ +struct cmap { + struct cmap_state *impl; +}; + +/* Initialization. */ +void cmap_init(struct cmap *); +void cmap_destroy(struct cmap *); + +/* Counters. */ +size_t cmap_size(const struct cmap *); +double cmap_utilization(const struct cmap *cmap); + +/* Insertion and deletion. Return the current count after the operation. */ +size_t cmap_insert(struct cmap *, struct cmap_node *, uint32_t hash); +size_t cmap_remove(struct cmap *, struct cmap_node *); + +/* Acquire/release cmap concurrent state. Use with iteration macros. + * Each acquired state must be released. */ +struct cmap_state cmap_state_acquire(struct cmap *cmap); +void cmap_state_release(struct cmap_state state); + +/* Iteration macros. Usage example: + * + * struct { + * struct cmap_node node; + * int value; + * } *data; + * struct cmap_state *cmap_state = cmap_state_acquire(&cmap); + * MAP_FOREACH(data, node, cmap_state) { + * ... + * } + * cmap_state_release(cmap_state); + */ +#define MAP_FOREACH(NODE, MEMBER, STATE) \ + MAP_FOREACH__(NODE, MEMBER, MAP, cmap_start__(STATE), STATE) + +#define MAP_FOREACH_WITH_HASH(NODE, MEMBER, HASH, STATE) \ + MAP_FOREACH__(NODE, MEMBER, MAP, cmap_find__(STATE, HASH), STATE) + +/* Ieration, private methods. Use iteration macros instead */ +struct cmap_cursor cmap_start__(struct cmap_state state); +struct cmap_cursor cmap_find__(struct cmap_state state, uint32_t hash); +void cmap_next__(struct cmap_state state, struct cmap_cursor *cursor); + +#define MAP_FOREACH__(NODE, MEMBER, MAP, START, STATE) \ + for (struct cmap_cursor cursor_ = START; \ + (cursor_.node ? (INIT_CONTAINER(NODE, cursor_.node, MEMBER), true) \ + : false); \ + cmap_next__(STATE, &cursor_)) diff --git a/cmap/hash.h b/cmap/hash.h new file mode 100644 index 0000000..4973aaf --- /dev/null +++ b/cmap/hash.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include +#include +#include + +#include "util.h" + +/* A Universally Unique IDentifier (UUID) compliant with RFC 4122. + * + * Each of the parts is stored in host byte order, but the parts themselves are + * ordered from left to right. That is, (parts[0] >> 24) is the first 8 bits + * of the UUID when output in the standard form, and (parts[3] & 0xff) is the + * final 8 bits. + */ +struct uuid { + uint32_t parts[4]; +}; + +static inline uint32_t hash_rot(uint32_t x, int k) +{ + return (x << k) | (x >> (32 - k)); +} + +/* Murmurhash by Austin Appleby, + * from https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp + */ +static inline uint32_t mhash_add__(uint32_t hash, uint32_t data) +{ + /* zero-valued 'data' will not change the 'hash' value */ + if (!data) + return hash; + + data *= 0xcc9e2d51; + data = hash_rot(data, 15); + data *= 0x1b873593; + return hash ^ data; +} + +static inline uint32_t mhash_add(uint32_t hash, uint32_t data) +{ + hash = mhash_add__(hash, data); + hash = hash_rot(hash, 13); + return hash * 5 + 0xe6546b64; +} + +static inline uint32_t mhash_finish(uint32_t hash) +{ + hash ^= hash >> 16; + hash *= 0x85ebca6b; + hash ^= hash >> 13; + hash *= 0xc2b2ae35; + hash ^= hash >> 16; + return hash; +} + +static inline uint32_t hash_add(uint32_t hash, uint32_t data) +{ + return mhash_add(hash, data); +} + +static inline uint32_t hash_finish(uint32_t hash, uint32_t final) +{ + return mhash_finish(hash ^ final); +} + +static inline uint32_t hash_2words(uint32_t x, uint32_t y) +{ + return hash_finish(hash_add(hash_add(x, 0), y), 8); +} + +static inline uint32_t hash_int(uint32_t x, uint32_t basis) +{ + return hash_2words(x, basis); +} diff --git a/cmap/list.h b/cmap/list.h new file mode 100644 index 0000000..3371b70 --- /dev/null +++ b/cmap/list.h @@ -0,0 +1,90 @@ +#pragma once + +#include +#include +#include + +#include "util.h" + +/* Doubly-linked list head or element. */ +struct list { + struct list *prev, *next; /* previous/next list element */ +}; + +#define LIST_INITIALIZER(LIST) \ + { \ + LIST, LIST \ + } + +/* Static initilizer */ +static inline void list_init(struct list *); + +/* list insertion */ +static inline void list_insert(struct list *, struct list *); +static inline void list_push_back(struct list *, struct list *); + +/* list removal */ +static inline struct list *list_remove(struct list *); +static inline struct list *list_pop_front(struct list *); + +static inline bool list_is_empty(const struct list *); + +/* Iterate through the list */ +#define LIST_FOREACH(ITER, MEMBER, LIST) \ + for (INIT_CONTAINER(ITER, (LIST)->next, MEMBER); \ + &(ITER)->MEMBER != (LIST); \ + ASSIGN_CONTAINER(ITER, (ITER)->MEMBER.next, MEMBER)) + +/* Iterate and pop */ +#define LIST_FOREACH_POP(ITER, MEMBER, LIST) \ + while (!list_is_empty(LIST) && \ + (INIT_CONTAINER(ITER, list_pop_front(LIST), MEMBER), 1)) + +/* Initializes 'list' as an empty list. */ +static inline void list_init(struct list *list) +{ + list->next = list->prev = list; +} + +/* Inserts 'elem' just before 'before'. */ +static inline void list_insert(struct list *before, struct list *elem) +{ + elem->prev = before->prev; + elem->next = before; + before->prev->next = elem; + before->prev = elem; +} + +/* Inserts 'elem' at the end of 'list', so that it becomes the back in + * 'list'. + */ +static inline void list_push_back(struct list *list, struct list *elem) +{ + list_insert(list, elem); +} + +/* Removes 'elem' from its list and returns the element that followed it. + * Undefined behavior if 'elem' is not in a list. + */ +static inline struct list *list_remove(struct list *elem) +{ + elem->prev->next = elem->next; + elem->next->prev = elem->prev; + return elem->next; +} + +/* Removes the front element from 'list' and returns it. Undefined behavior if + * 'list' is empty before removal. + */ +static inline struct list *list_pop_front(struct list *list) +{ + struct list *front = list->next; + list_remove(front); + return front; +} + +/* Returns true if 'list' is empty, false otherwise. */ +static inline bool list_is_empty(const struct list *list) +{ + return list->next == list; +} diff --git a/cmap/locks.h b/cmap/locks.h new file mode 100644 index 0000000..6fad1a3 --- /dev/null +++ b/cmap/locks.h @@ -0,0 +1,165 @@ +#pragma once + +#include +#include +#include + +#include "util.h" + +struct spinlock { + const char *where; + atomic_uint value; +}; + +static inline void spinlock_init(struct spinlock *spin); +static inline void spinlock_destroy(struct spinlock *spin); + +/* Locks "spin" and returns status code */ +#define spinlock_lock(spin) spinlock_lock_at(spin, SOURCE_LOCATOR) + +static inline void spinlock_lock_at(struct spinlock *spin, const char *where); +static inline void spinlock_unlock(struct spinlock *spin); + +struct mutex { + const char *where; + pthread_mutex_t lock; +}; + +static inline void mutex_init(struct mutex *mutex); +static inline void mutex_destroy(struct mutex *mutex); + +#define mutex_lock(mutex) mutex_lock_at(mutex, SOURCE_LOCATOR); + +static inline void mutex_lock_at(struct mutex *mutex_, const char *where); +static inline void mutex_unlock(struct mutex *mutex_); + +struct cond { + pthread_cond_t cond; + struct mutex mutex; + atomic_uint value; +}; + +#define cond_wait(cond) cond_wait_at(cond, SOURCE_LOCATOR); +#define cond_lock(cond) cond_lock_at(cond, SOURCE_LOCATOR); + +static inline void cond_init(struct cond *cond); +static inline void cond_destroy(struct cond *cond); +static inline void cond_wait_at(struct cond *cond, const char *where); +static inline void cond_lock_at(struct cond *cond, const char *where); +static inline void cond_unlock(struct cond *cond); +static inline bool cond_is_locked(struct cond *cond); + +static inline void spinlock_init(struct spinlock *spin) +{ + if (!spin) + return; + atomic_init(&spin->value, 0); + spin->where = NULL; +} + +static inline void spinlock_destroy(struct spinlock *spin) +{ + if (!spin) + return; + ASSERT(atomic_load(&spin->value) == 0); + spin->where = NULL; +} + +static inline void spinlock_lock_at(struct spinlock *spin, const char *where) +{ + if (!spin) + return; + uint32_t zero = 0; + while (!atomic_compare_exchange_strong(&spin->value, &zero, 1)) + zero = 0; + spin->where = where; +} + +static inline void spinlock_unlock(struct spinlock *spin) +{ + if (!spin) + return; + ASSERT(atomic_load(&spin->value) == 1); + atomic_store(&spin->value, 0); + spin->where = NULL; +} + +static inline void mutex_init(struct mutex *mutex) +{ + if (pthread_mutex_init(&mutex->lock, NULL)) + abort_msg("pthread_mutex_init fail"); +} + +static inline void mutex_destroy(struct mutex *mutex) +{ + if (pthread_mutex_destroy(&mutex->lock)) + abort_msg("pthread_mutex_destroy fail"); +} + +static inline void mutex_lock_at(struct mutex *mutex, const char *where) +{ + if (!mutex) + return; + if (pthread_mutex_lock(&mutex->lock)) + abort_msg("pthread_mutex_lock fail"); + mutex->where = where; +} + +static inline void mutex_unlock(struct mutex *mutex) +{ + if (!mutex) + return; + if (pthread_mutex_unlock(&mutex->lock)) + abort_msg("pthread_mutex_unlock fail"); + mutex->where = NULL; +} + +static inline void cond_init(struct cond *cond) +{ + if (pthread_cond_init(&cond->cond, NULL)) + abort_msg("pthread_cond_init fail"); + mutex_init(&cond->mutex); + atomic_init(&cond->value, 0); +} + +static inline void cond_destroy(struct cond *cond) +{ + uint32_t value = atomic_load(&cond->value); + ASSERT(!value); + if (pthread_cond_destroy(&cond->cond)) + abort_msg("pthread_cond_destroy fail"); + mutex_destroy(&cond->mutex); +} + +static inline void cond_wait_at(struct cond *cond, const char *where) +{ + mutex_lock_at(&cond->mutex, where); + while (1) { + uint32_t value = atomic_load(&cond->value); + if (!value) + break; + if (pthread_cond_wait(&cond->cond, &cond->mutex.lock)) + abort_msg("pthread_cond_wait fail"); + } + mutex_unlock(&cond->mutex); +} + +static inline void cond_lock_at(struct cond *cond, const char *where) +{ + atomic_store(&cond->value, 1); +} + +static inline bool cond_is_locked(struct cond *cond) +{ + uint32_t value = atomic_load(&cond->value); + return value == 1; +} + +static inline void cond_unlock(struct cond *cond) +{ + mutex_lock(&cond->mutex); + atomic_store(&cond->value, 0); + if (pthread_cond_broadcast(&cond->cond)) + abort_msg("pthread_cond_broadcast fail"); + mutex_unlock(&cond->mutex); +} diff --git a/cmap/perf.h b/cmap/perf.h new file mode 100644 index 0000000..dce450d --- /dev/null +++ b/cmap/perf.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +#define TIMESPAN_GET_NS(DEST, START, END) \ + DEST = -(START.tv_sec * 1e9 + START.tv_nsec) + \ + (END.tv_sec * 1e9 + END.tv_nsec); + +#define TIMESPAN_MEASURE(NAME) clock_gettime(CLOCK_MONOTONIC, &NAME); + +/* Start measuring time (nanosec) to variable "name" */ +#define PERF_START(name) \ + struct timespec name##_start, name##_end; \ + double name; \ + TIMESPAN_MEASURE(name##_start); + +/* Stop measuring time, store results to variable double "name" */ +#define PERF_END(name) \ + TIMESPAN_MEASURE(name##_end); \ + TIMESPAN_GET_NS(name, name##_start, name##_end) + +/* Returns the current time in ms */ +static inline uint64_t get_time_ns() +{ + struct timespec clk; + TIMESPAN_MEASURE(clk); + return clk.tv_sec * 1e9 + clk.tv_nsec; +} diff --git a/cmap/random.c b/cmap/random.c new file mode 100644 index 0000000..b9a2d5c --- /dev/null +++ b/cmap/random.c @@ -0,0 +1,45 @@ +#include +#include +#include + +#include "hash.h" +#include "random.h" +#include "util.h" + +static uint32_t seed = 0; + +static uint32_t random_next(void); + +void random_init(void) +{ + while (!seed) { + uint32_t t = (uint32_t) time(NULL); + seed = t; + srand48(seed); + } +} + +void random_set_seed(uint32_t seed_) +{ + while (!seed_) + seed_ = (uint32_t) time(NULL); + seed = seed_; + srand48(seed_); +} + +uint32_t random_uint32(void) +{ + random_init(); + return random_next(); +} + +static uint32_t random_next(void) +{ + uint32_t *seedp = &seed; + + *seedp ^= *seedp << 13; + *seedp ^= *seedp >> 17; + *seedp ^= *seedp << 5; + + return *seedp; +} diff --git a/cmap/random.h b/cmap/random.h new file mode 100644 index 0000000..54aaed6 --- /dev/null +++ b/cmap/random.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include +#include + +void random_init(void); +void random_set_seed(uint32_t); + +uint32_t random_uint32(void); diff --git a/cmap/rcu.c b/cmap/rcu.c new file mode 100644 index 0000000..4a15ec2 --- /dev/null +++ b/cmap/rcu.c @@ -0,0 +1,84 @@ +#include +#include + +#include "list.h" +#include "locks.h" +#include "rcu.h" +#include "util.h" + +struct rcu_cb { + struct list node; /* Inside "struct rcu" */ + rcu_callback_t cb; + void *args; +}; + +static inline struct rcu *rcu_allocate_new(void *val) +{ + struct rcu *new_rcu = xmalloc(sizeof(*new_rcu)); + list_init(&new_rcu->cb_list); + spinlock_init(&new_rcu->lock); + atomic_init(&new_rcu->counter, 1); + new_rcu->ptr = val; + return new_rcu; +} + +static void rcu_free(struct rcu *rcu) +{ + struct rcu_cb *rcu_cb; + LIST_FOREACH_POP (rcu_cb, node, &rcu->cb_list) { + rcu_cb->cb(rcu_cb->args); + free(rcu_cb); + } + spinlock_destroy(&rcu->lock); + free(rcu); +} + +void rcu_init__(struct rcu **rcu_p, void *val) +{ + struct rcu *new_rcu = rcu_allocate_new(val); + atomic_init(rcu_p, new_rcu); +} + +void rcu_destroy__(struct rcu *rcu) +{ + uint32_t counter = atomic_fetch_sub(&rcu->counter, 1); + ASSERT(counter == 1); + rcu_free(rcu); +} + +struct rcu *rcu_acquire__(struct rcu **rcu_p) +{ + struct rcu *rcu = atomic_load(rcu_p); + atomic_fetch_add(&rcu->counter, 1); + return rcu; +} + +void rcu_release__(struct rcu *rcu) +{ + uint32_t counter = atomic_fetch_sub(&rcu->counter, 1); + if (counter == 1) + rcu_free(rcu); +} + +void rcu_set__(struct rcu **rcu_p, void *val) +{ + struct rcu *old_rcu = atomic_load(rcu_p); + struct rcu *new_rcu = rcu_allocate_new(val); + atomic_store(rcu_p, new_rcu); + uint32_t counter = atomic_fetch_sub(&old_rcu->counter, 1); + if (counter == 1) + rcu_free(old_rcu); +} + +void rcu_postpone__(struct rcu *rcu, + rcu_callback_t cb, + void *args, + const char *where) +{ + struct rcu_cb *rcu_cb = xmalloc(sizeof(*rcu_cb)); + rcu_cb->cb = cb; + rcu_cb->args = args; + spinlock_lock_at(&rcu->lock, where); + list_push_back(&rcu->cb_list, &rcu_cb->node); + spinlock_unlock(&rcu->lock); +} diff --git a/cmap/rcu.h b/cmap/rcu.h new file mode 100644 index 0000000..45fc259 --- /dev/null +++ b/cmap/rcu.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include "list.h" +#include "locks.h" +#include "util.h" + +/* Callback method for RCU type */ +typedef void (*rcu_callback_t)(void *); + +struct rcu { + struct list cb_list; /* Holds "struct rcu_cb" */ + struct spinlock lock; /* Locks on "cb_list" */ + void *ptr; /* Pointer to data */ + atomic_uint counter; /* Number of active pointers to this */ +}; + +/* Initiate VAR to VAL */ +#define rcu_init(VAR, VAL) rcu_init__(CONST_CAST(struct rcu **, &VAR), VAL) +#define rcu_destroy(VAR) rcu_destroy__(VAR) + +/* Acquire & release an RCU pointer + * Usage: + * struct rcu* var = rcu_acquire(&rcu); + * ... + * rcu_release(var); + */ +#define rcu_acquire(VAR) rcu_acquire__(CONST_CAST(struct rcu **, &VAR)) +#define rcu_release(VAR) rcu_release__(VAR) + +/* Getter, setter. */ +#define rcu_get(VAR, TYPE) ((TYPE) (VAR->ptr)) +#define rcu_set(VAR, VAL) rcu_set__(CONST_CAST(struct rcu **, &VAR), VAL) + +/* Postpone FUNCTION(ARG) when the current value of VAR is not longer used */ +#define rcu_postpone(VAR, FUNCTION, ARG) \ + rcu_postpone__(VAR, FUNCTION, ARG, SOURCE_LOCATOR) + +void rcu_init__(struct rcu **, void *val); +void rcu_destroy__(struct rcu *); +struct rcu *rcu_acquire__(struct rcu **); +void rcu_release__(struct rcu *); +void rcu_set__(struct rcu **, void *val); +void rcu_set_and_wait__(struct rcu **, void *val); +void rcu_postpone__(struct rcu *, + rcu_callback_t, + void *args, + const char *where); diff --git a/cmap/test-cmap.c b/cmap/test-cmap.c new file mode 100644 index 0000000..a74082c --- /dev/null +++ b/cmap/test-cmap.c @@ -0,0 +1,209 @@ +#include +#include +#include +#include +#include +#include + +#include "cmap.h" +#include "hash.h" +#include "perf.h" +#include "random.h" +#include "util.h" + +#define DEFAULT_SECONDS 3 +#define DEFAULT_READERS 3 + +struct elem { + struct cmap_node node; + uint32_t value; +}; + +static size_t num_values; +static uint32_t max_value; +static uint32_t *values; +static uint32_t hash_base; +static struct cmap cmap_values; +static volatile bool running; +static volatile bool error; + +static atomic_size_t checks; +static volatile uint32_t inserts; +static volatile uint32_t removes; + +/* Insert new value to cmap */ +static void insert_value(uint32_t value) +{ + struct elem *elem = xmalloc(sizeof(*elem)); + elem->value = value; + cmap_insert(&cmap_values, &elem->node, hash_int(value, hash_base)); +} + +/* Initiate all static variables */ +static void initiate_values(uint32_t seed) +{ + running = true; + error = false; + inserts = 0; + removes = 0; + random_set_seed(seed); + num_values = (random_uint32() & 0xFF) + 16; + max_value = (random_uint32() & 4096) + 2048; + hash_base = random_uint32(); + values = xmalloc(sizeof(*values) * num_values); + cmap_init(&cmap_values); + atomic_init(&checks, 0); + + for (int i = 0; i < num_values; i++) { + values[i] = random_uint32() & max_value; + insert_value(values[i]); + } +} + +/* Destroy all static variables */ +static void destroy_values() +{ + struct cmap_state cmap_state; + struct elem *elem; + free(values); + + cmap_state = cmap_state_acquire(&cmap_values); + MAP_FOREACH (elem, node, cmap_state) { + cmap_remove(&cmap_values, &elem->node); + free(elem); + } + cmap_state_release(cmap_state); + + cmap_destroy(&cmap_values); +} + +/* Returns true iff "value" can be composed from two integers in "cmap" */ +static bool can_compose_value(uint32_t value) +{ + struct elem *elem; + uint32_t hash; + struct cmap_state cmap_state = cmap_state_acquire(&cmap_values); + + hash = hash_int(value, hash_base); + MAP_FOREACH_WITH_HASH (elem, node, hash, cmap_state) { + if (elem->value == value) { + cmap_state_release(cmap_state); + return true; + } + } + + cmap_state_release(cmap_state); + return false; +} + +static inline void wait() +{ + usleep(1); +} + +/* Constantly writes and removes values from cmap */ +static void *update_cmap(void *args) +{ + struct elem *elem; + struct cmap_state cmap_state; + + while (running) { + /* Insert */ + uint32_t value = random_uint32() + max_value + 1; + insert_value(value); + inserts++; + wait(); + + /* Remove */ + uint32_t hash = hash_int(random_uint32(), hash_base); + cmap_state = cmap_state_acquire(&cmap_values); + MAP_FOREACH_WITH_HASH (elem, node, hash, cmap_state) { + if (elem->value > max_value) { + cmap_remove(&cmap_values, &elem->node); + free(elem); + removes++; + break; + } + } + cmap_state_release(cmap_state); + wait(); + } + return NULL; +} + +/* Constantly check whever values in cmap can be composed */ +static void *read_cmap(void *args) +{ + while (running) { + uint32_t index = random_uint32() % num_values; + if (!can_compose_value(values[index])) { + running = false; + error = true; + break; + } + atomic_fetch_add(&checks, 1); + wait(); + if (can_compose_value(values[index] + max_value + 1)) { + running = false; + error = true; + break; + } + atomic_fetch_add(&checks, 1); + wait(); + } + return NULL; +} + +int main(int argc, char **argv) +{ + /* Parse arguments */ + for (int i = 0; i < argc; i++) { + if (!strcmp("--help", argv[i]) || !strcmp("-h", argv[i])) { + printf( + "Tests performance and correctness of cmap.\n" + "Usage: %s [SECONDS] [READERS]\n" + "Defaults: %d seconds, %d reader threads.\n", + argv[0], DEFAULT_SECONDS, DEFAULT_READERS); + exit(1); + } + } + + int seconds = argc >= 2 ? atoi(argv[1]) : DEFAULT_SECONDS; + int readers = argc >= 3 ? atoi(argv[2]) : DEFAULT_READERS; + + /* Initiate */ + initiate_values(1); + pthread_t *threads = xmalloc(sizeof(*threads) * (readers + 1)); + + /* Start threads */ + for (int i = 0; i < readers; ++i) + pthread_create(&threads[i], NULL, read_cmap, NULL); + pthread_create(&threads[readers], NULL, update_cmap, NULL); + + /* Print stats to user */ + size_t dst = get_time_ns() + 1e9 * seconds; + while (get_time_ns() < dst) { + size_t current_checks = atomic_load(&checks); + printf( + "#checks: %u, #inserts: %u, #removes: %u, " + "cmap elements: %u, utilization: %.2lf \n", + (uint32_t) current_checks, inserts, removes, + (uint32_t) cmap_size(&cmap_values), cmap_utilization(&cmap_values)); + usleep(250e3); + } + + /* Stop threads */ + running = false; + for (int i = 0; i <= readers; ++i) + pthread_join(threads[i], NULL); + + /* Delete memory */ + free(threads); + destroy_values(); + + /* Check for correctness errors */ + if (error) + printf("Error: correctness issue\n"); + + return error; +} diff --git a/cmap/util.c b/cmap/util.c new file mode 100644 index 0000000..0109a78 --- /dev/null +++ b/cmap/util.c @@ -0,0 +1,19 @@ +#include "util.h" +#include +#include +#include +#include + +void abort_msg(const char *msg) +{ + fprintf(stderr, "%s\n", msg); + abort(); +} + +void *xmalloc(size_t size) +{ + void *p = malloc(size ? size : 1); + if (!p) + abort_msg("Out of memory"); + return p; +} diff --git a/cmap/util.h b/cmap/util.h new file mode 100644 index 0000000..1ffc64b --- /dev/null +++ b/cmap/util.h @@ -0,0 +1,149 @@ +#pragma once + +#include +#include +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +#define LIKELY(CONDITION) __builtin_expect(!!(CONDITION), 1) +#else +#define LIKELY(CONDITION) (!!(CONDITION)) +#endif + +/* Expands to a string that looks like ":", e.g. "tmp.c:10". + * + * See http://c-faq.com/ansi/stringize.html for an explanation of STRINGIZE + * and STRINGIZE2. + */ +#define SOURCE_LOCATOR __FILE__ ":" STRINGIZE(__LINE__) +#define STRINGIZE(ARG) STRINGIZE2(ARG) +#define STRINGIZE2(ARG) #ARG + +/* Bit masks */ +#define BITMASK_8 0xff +#define BITMASK_16 0xffff +#define BITMASK_32 0xffffffff + +/* Cacheline marking is typically done using zero-sized array. */ +typedef uint8_t CACHE_LINE_MARKER[1]; + +/* This is a void expression that issues a compiler error if POINTER cannot be + * compared for equality with the given pointer TYPE. This generally means + * that POINTER is a qualified or unqualified TYPE. However, + * BUILD_ASSERT_TYPE(POINTER, void *) will accept any pointer to object type, + * because any pointer to object can be compared for equality with "void *". + * + * POINTER can be any expression. The use of "sizeof" ensures that the + * expression is not actually evaluated, so that any side effects of the + * expression do not occur. + * + * The cast to int is present only to suppress an "expression using sizeof + * bool" warning from "sparse" (see + * http://permalink.gmane.org/gmane.comp.parsers.sparse/2967). + */ +#define BUILD_ASSERT_TYPE(POINTER, TYPE) \ + ((void) sizeof((int) ((POINTER) == (TYPE) (POINTER)))) + +/* Casts 'pointer' to 'type' and issues a compiler warning if the cast changes + * anything other than an outermost "const" or "volatile" qualifier. */ +#define CONST_CAST(TYPE, POINTER) \ + (BUILD_ASSERT_TYPE(POINTER, TYPE), (TYPE) (POINTER)) + +/* Given OBJECT of type pointer-to-structure, expands to the offset of MEMBER + * within an instance of the structure. + * + * The GCC-specific version avoids the technicality of undefined behavior if + * OBJECT is null, invalid, or not yet initialized. This makes some static + * checkers (like Coverity) happier. But the non-GCC version does not actually + * dereference any pointer, so it would be surprising for it to cause any + * problems in practice. + */ +#if defined(__GNUC__) || defined(__clang__) +#define OBJECT_OFFSETOF(OBJECT, MEMBER) offsetof(typeof(*(OBJECT)), MEMBER) +#else +#define OBJECT_OFFSETOF(OBJECT, MEMBER) \ + ((char *) &(OBJECT)->MEMBER - (char *) (OBJECT)) +#endif + +/* Given a pointer-typed lvalue OBJECT, expands to a pointer type that may be + * assigned to OBJECT. */ +#ifdef __GNUC__ +#define TYPEOF(OBJECT) typeof(OBJECT) +#else +#define TYPEOF(OBJECT) void * +#endif + +/* Given POINTER, the address of the given MEMBER within an object of the type + * that that OBJECT points to, returns OBJECT as an assignment-compatible + * pointer type (either the correct pointer type or "void *"). OBJECT must be + * an lvalue. + * + * This is the same as CONTAINER_OF except that it infers the structure type + * from the type of '*OBJECT'. + */ +#define OBJECT_CONTAINING(POINTER, OBJECT, MEMBER) \ + ((TYPEOF(OBJECT)) (void *) ((char *) (POINTER) -OBJECT_OFFSETOF(OBJECT, \ + MEMBER))) + +/* Given POINTER, the address of the given MEMBER within an object of the type + * that that OBJECT points to, assigns the address of the outer object to + * OBJECT, which must be an lvalue. + * + * Evaluates to (void) 0 as the result is not to be used. + */ +#define ASSIGN_CONTAINER(OBJECT, POINTER, MEMBER) \ + ((OBJECT) = OBJECT_CONTAINING(POINTER, OBJECT, MEMBER), (void) 0) + +/* As explained in the comment above OBJECT_OFFSETOF(), non-GNUC compilers + * like MSVC will complain about un-initialized variables if OBJECT + * hasn't already been initialized. To prevent such warnings, INIT_CONTAINER() + * can be used as a wrapper around ASSIGN_CONTAINER. + */ +#define INIT_CONTAINER(OBJECT, POINTER, MEMBER) \ + ((OBJECT) = NULL, ASSIGN_CONTAINER(OBJECT, POINTER, MEMBER)) + +/* Yields the size of MEMBER within STRUCT. */ +#define MEMBER_SIZEOF(STRUCT, MEMBER) (sizeof(((STRUCT *) NULL)->MEMBER)) + +/* Yields the offset of the end of MEMBER within STRUCT. */ +#define OFFSETOFEND(STRUCT, MEMBER) \ + (offsetof(STRUCT, MEMBER) + MEMBER_SIZEOF(STRUCT, MEMBER)) + +/* Given POINTER, the address of the given MEMBER in a STRUCT object, returns + * the STRUCT object. + */ +#define CONTAINER_OF(POINTER, STRUCT, MEMBER) \ + ((STRUCT *) (void *) ((char *) (POINTER) -offsetof(STRUCT, MEMBER))) + +/* Returns the address after the object pointed by POINTER casted to TYPE */ +#define OBJECT_END(TYPE, POINTER) (TYPE)((char *) POINTER + sizeof(*POINTER)) + +/* Returns the size in bytes of array with NUM elements of type TYPE */ +#define ARRAY_SIZE(TYPE, NUM) (sizeof(TYPE) * NUM) + +/* Like the standard assert macro, except always evaluates the condition, + * even with NDEBUG. + */ +#ifndef NDEBUG +#define ASSERT(CONDITION) (LIKELY(CONDITION) ? (void) 0 : assert(0)) +#else +#define ASSERT(CONDITION) ((void) (CONDITION)) +#endif + +#include +#include + +static inline void abort_msg(const char *msg) +{ + fprintf(stderr, "%s\n", msg); + abort(); +} + +static inline void *xmalloc(size_t size) +{ + void *p = malloc(size ? size : 1); + if (!p) + abort_msg("Out of memory"); + return p; +}