diff --git a/include/tscore/List.h b/include/tscore/List.h index 25bfac5a498..919255240a6 100644 --- a/include/tscore/List.h +++ b/include/tscore/List.h @@ -721,12 +721,12 @@ template struct AtomicSLL { C * head() { - return (C *)TO_PTR(FREELIST_POINTER(al.head)); + return (C *)(al.head.load()); } C * next(C *c) { - return (C *)TO_PTR(c); + return (C *)ink_atomiclist_next(&al, c); } InkAtomicList al; diff --git a/include/tscore/ink_queue.h b/include/tscore/ink_queue.h index 24f860594fb..2e8860145c5 100644 --- a/include/tscore/ink_queue.h +++ b/include/tscore/ink_queue.h @@ -33,141 +33,20 @@ ***********************************************************************/ -#include "tscore/ink_platform.h" -#include "tscore/ink_defs.h" -#include "tscore/ink_apidefs.h" - -/* - For information on the structure of the x86_64 memory map: - - http://en.wikipedia.org/wiki/X86-64#Linux - - Essentially, in the current 48-bit implementations, the - top bit as well as the lower 47 bits are used, leaving - the upper-but one 16 bits free to be used for the version. - We will use the top-but-one 15 and sign extend when generating - the pointer was required by the standard. -*/ +#include +#include -/* -#if defined(POSIX_THREAD) -#include -#include -#endif -*/ +#include "tscore/ink_apidefs.h" #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ -void ink_queue_load_64(void *dst, void *src); - -#ifdef __x86_64__ -#define INK_QUEUE_LD64(dst, src) *((uint64_t *)&(dst)) = *((uint64_t *)&(src)) -#else -#define INK_QUEUE_LD64(dst, src) (ink_queue_load_64((void *)&(dst), (void *)&(src))) -#endif - -#if TS_HAS_128BIT_CAS -#define INK_QUEUE_LD(dst, src) \ - do { \ - *(__int128_t *)&(dst) = __sync_val_compare_and_swap((__int128_t *)&(src), 0, 0); \ - } while (0) -#else -#define INK_QUEUE_LD(dst, src) INK_QUEUE_LD64(dst, src) -#endif - -/* - * Generic Free List Manager - */ -// Warning: head_p is read and written in multiple threads without a -// lock, use INK_QUEUE_LD to read safely. -union head_p { - head_p() : data(){}; - -#if (defined(__i386__) || defined(__arm__) || defined(__mips__)) && (SIZEOF_VOIDP == 4) - typedef int32_t version_type; - typedef int64_t data_type; -#elif TS_HAS_128BIT_CAS - typedef int64_t version_type; - typedef __int128_t data_type; -#else - typedef int64_t version_type; - typedef int64_t data_type; -#endif - - struct { - void *pointer; - version_type version; - } s; - - data_type data; -}; - -/* - * Why is version required? One scenario is described below - * Think of a list like this -> A -> C -> D - * and you are popping from the list - * Between the time you take the ptr(A) and swap the head pointer - * the list could start looking like this - * -> A -> B -> C -> D - * If the version check is not there, the list will look like - * -> C -> D after the pop, which will result in the loss of "B" - */ -#define ZERO_HEAD_P(_x) - -#ifdef DEBUG -#define FROM_PTR(_x) (void *)(((uintptr_t)_x) + 1) -#define TO_PTR(_x) (void *)(((uintptr_t)_x) - 1) -#else -#define FROM_PTR(_x) ((void *)(_x)) -#define TO_PTR(_x) ((void *)(_x)) -#endif - -#if (defined(__i386__) || defined(__arm__) || defined(__mips__)) && (SIZEOF_VOIDP == 4) -#define FREELIST_POINTER(_x) (_x).s.pointer -#define FREELIST_VERSION(_x) (_x).s.version -#define SET_FREELIST_POINTER_VERSION(_x, _p, _v) \ - (_x).s.pointer = _p; \ - (_x).s.version = _v -#elif TS_HAS_128BIT_CAS -#define FREELIST_POINTER(_x) (_x).s.pointer -#define FREELIST_VERSION(_x) (_x).s.version -#define SET_FREELIST_POINTER_VERSION(_x, _p, _v) \ - (_x).s.pointer = _p; \ - (_x).s.version = _v -#elif defined(__x86_64__) || defined(__ia64__) || defined(__powerpc64__) || defined(__aarch64__) || defined(__mips64) -/* Layout of FREELIST_POINTER - * - * 0 ~ 47 bits : 48 bits, Virtual Address (47 bits for AMD64 and 48 bits for AArch64) - * 48 ~ 62 bits : 15 bits, Freelist Version - * 63 bits : 1 bits, The type of Virtual Address (0 = user space, 1 = kernel space) - */ -/* Detect which shift is implemented by the simple expression ((~0 >> 1) < 0): - * - * If the shift is 'logical' the highest order bit of the left side of the comparison is 0 so the result is positive. - * If the shift is 'arithmetic' the highest order bit of the left side is 1 so the result is negative. - */ -#if ((~0 >> 1) < 0) -/* the shift is `arithmetic' */ -#define FREELIST_POINTER(_x) \ - ((void *)((((intptr_t)(_x).data) & 0x0000FFFFFFFFFFFFLL) | ((((intptr_t)(_x).data) >> 63) << 48))) // sign extend -#else -/* the shift is `logical' */ -#define FREELIST_POINTER(_x) \ - ((void *)((((intptr_t)(_x).data) & 0x0000FFFFFFFFFFFFLL) | (((~((((intptr_t)(_x).data) >> 63) - 1)) >> 48) << 48))) -#endif - -#define FREELIST_VERSION(_x) ((((intptr_t)(_x).data) & 0x7FFF000000000000LL) >> 48) -#define SET_FREELIST_POINTER_VERSION(_x, _p, _v) (_x).data = ((((intptr_t)(_p)) & 0x8000FFFFFFFFFFFFLL) | (((_v)&0x7FFFLL) << 48)) -#else -#error "unsupported processor" -#endif - struct _InkFreeList { - head_p head; + std::atomic head{nullptr}; const char *name; - uint32_t type_size, chunk_size, used, allocated, alignment; + std::atomic used, allocated; + uint32_t type_size, chunk_size, alignment; uint32_t allocated_base, used_base; int advice; }; @@ -196,22 +75,18 @@ void ink_freelists_snap_baseline(); struct InkAtomicList { InkAtomicList() {} - head_p head{}; + std::atomic head{nullptr}; const char *name = nullptr; uint32_t offset = 0; }; -#if !defined(INK_QUEUE_NT) -#define INK_ATOMICLIST_EMPTY(_x) (!(TO_PTR(FREELIST_POINTER((_x.head))))) -#else -/* ink_queue_nt.c doesn't do the FROM/TO pointer swizzling */ -#define INK_ATOMICLIST_EMPTY(_x) (!((FREELIST_POINTER((_x.head))))) -#endif +#define INK_ATOMICLIST_EMPTY(_x) (_x.head == nullptr) inkcoreapi void ink_atomiclist_init(InkAtomicList *l, const char *name, uint32_t offset_to_next); inkcoreapi void *ink_atomiclist_push(InkAtomicList *l, void *item); void *ink_atomiclist_pop(InkAtomicList *l); inkcoreapi void *ink_atomiclist_popall(InkAtomicList *l); +inkcoreapi void *ink_atomiclist_next(InkAtomicList *l, void *item); /* * WARNING WARNING WARNING WARNING WARNING WARNING WARNING * only if only one thread is doing pops it is possible to have a "remove" diff --git a/include/tscore/ver_ptr_kruft.h b/include/tscore/ver_ptr_kruft.h new file mode 100644 index 00000000000..2fc62cf24aa --- /dev/null +++ b/include/tscore/ver_ptr_kruft.h @@ -0,0 +1,126 @@ +/** @file + + Really really scary code to store a "version" number in unused bits in a pointer. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +/* + For information on the structure of the x86_64 memory map: + + http://en.wikipedia.org/wiki/X86-64#Linux + + Essentially, in the current 48-bit implementations, the + top bit as well as the lower 47 bits are used, leaving + the upper-but one 16 bits free to be used for the version. + We will use the top-but-one 15 and sign extend when generating + the pointer was required by the standard. +*/ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +void ink_queue_load_64(void *dst, void *src); + +#ifdef __x86_64__ +#define INK_QUEUE_LD64(dst, src) *((uint64_t *)&(dst)) = *((uint64_t *)&(src)) +#else +#define INK_QUEUE_LD64(dst, src) (ink_queue_load_64((void *)&(dst), (void *)&(src))) +#endif + +#if TS_HAS_128BIT_CAS +#define INK_QUEUE_LD(dst, src) \ + do { \ + *(__int128_t *)&(dst) = __sync_val_compare_and_swap((__int128_t *)&(src), 0, 0); \ + } while (0) +#else +#define INK_QUEUE_LD(dst, src) INK_QUEUE_LD64(dst, src) +#endif + +// Warning: head_p is read and written in multiple threads without a +// lock, use INK_QUEUE_LD to read safely. This type was formerly used as a linked list +// head pointer. +union head_p { + head_p() : data(){}; + +#if (defined(__i386__) || defined(__arm__) || defined(__mips__)) && (SIZEOF_VOIDP == 4) + typedef int32_t version_type; + typedef int64_t data_type; +#elif TS_HAS_128BIT_CAS + typedef int64_t version_type; + typedef __int128_t data_type; +#else + typedef int64_t version_type; + typedef int64_t data_type; +#endif + + struct { + void *pointer; + version_type version; + } s; + + data_type data; +}; + +#if (defined(__i386__) || defined(__arm__) || defined(__mips__)) && (SIZEOF_VOIDP == 4) +#define FREELIST_POINTER(_x) (_x).s.pointer +#define FREELIST_VERSION(_x) (_x).s.version +#define SET_FREELIST_POINTER_VERSION(_x, _p, _v) \ + (_x).s.pointer = _p; \ + (_x).s.version = _v +#elif TS_HAS_128BIT_CAS +#define FREELIST_POINTER(_x) (_x).s.pointer +#define FREELIST_VERSION(_x) (_x).s.version +#define SET_FREELIST_POINTER_VERSION(_x, _p, _v) \ + (_x).s.pointer = _p; \ + (_x).s.version = _v +#elif defined(__x86_64__) || defined(__ia64__) || defined(__powerpc64__) || defined(__aarch64__) || defined(__mips64) +/* Layout of FREELIST_POINTER + * + * 0 ~ 47 bits : 48 bits, Virtual Address (47 bits for AMD64 and 48 bits for AArch64) + * 48 ~ 62 bits : 15 bits, Freelist Version + * 63 bits : 1 bits, The type of Virtual Address (0 = user space, 1 = kernel space) + */ +/* Detect which shift is implemented by the simple expression ((~0 >> 1) < 0): + * + * If the shift is 'logical' the highest order bit of the left side of the comparison is 0 so the result is positive. + * If the shift is 'arithmetic' the highest order bit of the left side is 1 so the result is negative. + */ +#if ((~0 >> 1) < 0) +/* the shift is `arithmetic' */ +#define FREELIST_POINTER(_x) \ + ((void *)((((intptr_t)(_x).data) & 0x0000FFFFFFFFFFFFLL) | ((((intptr_t)(_x).data) >> 63) << 48))) // sign extend +#else +/* the shift is `logical' */ +#define FREELIST_POINTER(_x) \ + ((void *)((((intptr_t)(_x).data) & 0x0000FFFFFFFFFFFFLL) | (((~((((intptr_t)(_x).data) >> 63) - 1)) >> 48) << 48))) +#endif + +#define FREELIST_VERSION(_x) ((((intptr_t)(_x).data) & 0x7FFF000000000000LL) >> 48) +#define SET_FREELIST_POINTER_VERSION(_x, _p, _v) (_x).data = ((((intptr_t)(_p)) & 0x8000FFFFFFFFFFFFLL) | (((_v)&0x7FFFLL) << 48)) +#else +#error "unsupported processor" +#endif + +#ifdef __cplusplus +} +#endif /* __cplusplus */ diff --git a/proxy/logging/LogObject.h b/proxy/logging/LogObject.h index 7a6b81782b6..2a0a30a4874 100644 --- a/proxy/logging/LogObject.h +++ b/proxy/logging/LogObject.h @@ -24,6 +24,7 @@ #pragma once #include "tscore/ink_platform.h" +#include "tscore/ver_ptr_kruft.h" #include "Log.h" #include "LogFile.h" #include "LogFormat.h" diff --git a/src/tscore/Makefile.am b/src/tscore/Makefile.am index ed6788ebc0c..e6b38b573fc 100644 --- a/src/tscore/Makefile.am +++ b/src/tscore/Makefile.am @@ -84,7 +84,6 @@ libtscore_la_SOURCES = \ ink_memory.cc \ ink_mutex.cc \ ink_queue.cc \ - ink_queue_utils.cc \ ink_rand.cc \ ink_res_init.cc \ ink_res_mkquery.cc \ @@ -119,6 +118,7 @@ libtscore_la_SOURCES = \ TextBuffer.cc \ Tokenizer.cc \ ts_file.cc \ + ver_ptr_kruft.cc \ Version.cc \ X509HostnameValidator.cc diff --git a/src/tscore/ink_queue.cc b/src/tscore/ink_queue.cc index f4a2841c06f..315ecd2f526 100644 --- a/src/tscore/ink_queue.cc +++ b/src/tscore/ink_queue.cc @@ -37,9 +37,9 @@ ****************************************************************************/ #include "tscore/ink_config.h" -#include #include #include +#include #include #include #include @@ -66,6 +66,151 @@ #define DEADBEEF #endif +namespace +{ +// Atomic stack, implemented as a linked list. The link is stored at the offset (given by the base class +// member function link_offset() ) within each stack element. The stack is empty when the head pointer is null. +// +template class AtomicStack_ : private Base +{ +public: + template + AtomicStack_(std::atomic &head, Args &&... args) : Base(std::forward(args)...), _head(head) + { + } + + // Return reference to "next" pointer within a stack element. + // + void *& + link(void *elem) + { + ink_assert(elem != nullptr); + + return *reinterpret_cast(static_cast(elem) + Base::link_offset()); + } + + // Splice this stack to the tail of another stack (whose head and tail are given by the parameters). + // Returns previous head. + // + void * + splice_to_top(void *other_head, void *other_tail) + { + void *curr_head = _head; + + do { + link(other_tail) = curr_head; + + } while (!_head.compare_exchange_weak(curr_head, other_head)); + + return curr_head; + } + + // Returns previous head. + // + void * + push(void *elem) + { + return splice_to_top(elem, elem); + } + + template + void * + pop() + { + void *curr_head = _head, *new_head; + + do { + if (!curr_head) { + // Stack is empty. + // + return nullptr; + } + + new_head = All_elements ? nullptr : link(curr_head); + + } while (!_head.compare_exchange_weak(curr_head, new_head)); + + link(curr_head) = nullptr; + + return curr_head; + } + + // WARNING: This is not safe, unless no other thread is popping or removing. + // + // If "item" is in list, it is removed from the list, and the function returns true. The function returns + // false if "item" is not in the list. If "item" was in the list, it's link pointer is cleared. + // + bool + remove(void *item) + { + void *p = item; + void *p2 = link(item); + + if (_head.compare_exchange_strong(p, p2)) { + // Item was at the top of the stack. + + link(item) = nullptr; + + return true; + } + if (p) { + p2 = link(p); + + while (p2 != item) { + if (nullptr == p2) { + // Item not in stack. + // + return false; + } + p = p2; + p2 = link(p2); + } + link(p) = link(p2); + + link(item) = nullptr; + + return true; + } + // Stack is empty. + // + return false; + } + +private: + std::atomic &_head; +}; + +class AtomicStackBaseNoOffset +{ +protected: + static constexpr size_t + link_offset() + { + return 0; + } +}; + +using AtomicStackNoOffset = AtomicStack_; + +class AtomicStackBaseWithOffset +{ +protected: + AtomicStackBaseWithOffset(size_t offset) : _link_offset(offset) {} + + size_t + link_offset() const + { + return _link_offset; + } + +private: + size_t const _link_offset; +}; + +using AtomicStackWithOffset = AtomicStack_; + +} // end anonymous namespace + static auto jna = jearena::globalJemallocNodumpAllocator(); struct ink_freelist_ops { @@ -151,7 +296,7 @@ ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32 f->chunk_size = INK_ALIGN(chunk_size * f->type_size, ats_pagesize()) / f->type_size; } Debug(DEBUG_TAG "_init", "<%s> Chunk Size request/actual (%" PRIu32 "/%" PRIu32 ")", name, chunk_size, f->chunk_size); - SET_FREELIST_POINTER_VERSION(f->head, FROM_PTR(0), 0); + f->head = nullptr; *fl = f; } @@ -173,8 +318,6 @@ ink_freelist_create(const char *name, uint32_t type_size, uint32_t chunk_size, u return f; } -#define ADDRESS_OF_NEXT(x, offset) ((void **)((char *)x + offset)) - #ifdef SANITY int fake_global_for_ink_queue = 0; #endif @@ -185,7 +328,7 @@ ink_freelist_new(InkFreeList *f) void *ptr; if (likely(ptr = freelist_global_ops->fl_new(f))) { - ink_atomic_increment(reinterpret_cast(&f->used), 1); + ++(f->used); } return ptr; @@ -194,13 +337,10 @@ ink_freelist_new(InkFreeList *f) static void * freelist_new(InkFreeList *f) { - head_p item; - head_p next; - int result = 0; + void *item; - do { - INK_QUEUE_LD(item, f->head); - if (TO_PTR(FREELIST_POINTER(item)) == nullptr) { + for (;;) { + if (f->head == nullptr) { uint32_t i; void *newp = nullptr; size_t alloc_size = f->chunk_size * f->type_size; @@ -219,13 +359,12 @@ freelist_new(InkFreeList *f) if (f->advice) { ats_madvise(static_cast(newp), INK_ALIGN(alloc_size, alignment), f->advice); } - SET_FREELIST_POINTER_VERSION(item, newp, 0); - ink_atomic_increment(reinterpret_cast(&f->allocated), f->chunk_size); + f->allocated += f->chunk_size; /* free each of the new elements */ for (i = 0; i < f->chunk_size; i++) { - char *a = (static_cast(FREELIST_POINTER(item))) + i * f->type_size; + char *a = static_cast(newp) + i * f->type_size; #ifdef DEADBEEF const char str[4] = {static_cast(0xde), static_cast(0xad), static_cast(0xbe), static_cast(0xef)}; for (int j = 0; j < static_cast(f->type_size); j++) { @@ -236,27 +375,28 @@ freelist_new(InkFreeList *f) } } else { - SET_FREELIST_POINTER_VERSION(next, *ADDRESS_OF_NEXT(TO_PTR(FREELIST_POINTER(item)), 0), FREELIST_VERSION(item) + 1); - result = ink_atomic_cas(&f->head.data, item.data, next.data); + item = AtomicStackNoOffset(f->head).pop<>(); + if (item) { #ifdef SANITY - if (result) { - if (FREELIST_POINTER(item) == TO_PTR(FREELIST_POINTER(next))) { + void *head = f->head; + if (head == item) { ink_abort("ink_freelist_new: loop detected"); } - if (((uintptr_t)(TO_PTR(FREELIST_POINTER(next)))) & 3) { + if (reinterpret_cast(head) & 3) { ink_abort("ink_freelist_new: bad list"); } - if (TO_PTR(FREELIST_POINTER(next))) { - fake_global_for_ink_queue = *static_cast(TO_PTR(FREELIST_POINTER(next))); + if (head) { + fake_global_for_ink_queue = *static_cast(head); } - } #endif /* SANITY */ + break; + } } - } while (result == 0); - ink_assert(!((uintptr_t)TO_PTR(FREELIST_POINTER(item)) & (((uintptr_t)f->alignment) - 1))); + } + ink_assert(!(reinterpret_cast(item) & (static_cast(f->alignment) - 1))); - return TO_PTR(FREELIST_POINTER(item)); + return item; } static void * @@ -279,20 +419,13 @@ ink_freelist_free(InkFreeList *f, void *item) if (likely(item != nullptr)) { ink_assert(f->used != 0); freelist_global_ops->fl_free(f, item); - ink_atomic_decrement(reinterpret_cast(&f->used), 1); + --(f->used); } } static void freelist_free(InkFreeList *f, void *item) { - void **adr_of_next = ADDRESS_OF_NEXT(item, 0); - head_p h; - head_p item_pair; - int result = 0; - - // ink_assert(!((long)item&(f->alignment-1))); XXX - why is this no longer working? -bcall - #ifdef DEADBEEF { static const char str[4] = {static_cast(0xde), static_cast(0xad), static_cast(0xbe), static_cast(0xef)}; @@ -304,24 +437,20 @@ freelist_free(InkFreeList *f, void *item) } #endif /* DEADBEEF */ - while (!result) { - INK_QUEUE_LD(h, f->head); #ifdef SANITY - if (TO_PTR(FREELIST_POINTER(h)) == item) { - ink_abort("ink_freelist_free: trying to free item twice"); - } - if (((uintptr_t)(TO_PTR(FREELIST_POINTER(h)))) & 3) { - ink_abort("ink_freelist_free: bad list"); - } - if (TO_PTR(FREELIST_POINTER(h))) { - fake_global_for_ink_queue = *static_cast(TO_PTR(FREELIST_POINTER(h))); - } -#endif /* SANITY */ - *adr_of_next = FREELIST_POINTER(h); - SET_FREELIST_POINTER_VERSION(item_pair, FROM_PTR(item), FREELIST_VERSION(h)); - INK_MEMORY_BARRIER; - result = ink_atomic_cas(&f->head.data, h.data, item_pair.data); + void *head = f->head; + if (head == item) { + ink_abort("ink_freelist_free: trying to free item twice"); } + if (reinterpret_cast(head) & 3) { + ink_abort("ink_freelist_free: bad list"); + } + if (head) { + fake_global_for_ink_queue = *static_cast(head); + } +#endif /* SANITY */ + + AtomicStackNoOffset(f->head).push(item); } static void @@ -340,53 +469,49 @@ ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item) ink_assert(f->used >= num_item); freelist_global_ops->fl_bulkfree(f, head, tail, num_item); - ink_atomic_decrement(reinterpret_cast(&f->used), num_item); + + f->used -= num_item; } static void -freelist_bulkfree(InkFreeList *f, void *head, void *tail, size_t num_item) +freelist_bulkfree(InkFreeList *f, void *head, void *tail, + size_t +#ifdef DEADBEEF + num_item +#endif +) { - void **adr_of_next = ADDRESS_OF_NEXT(tail, 0); - head_p h; - head_p item_pair; - int result = 0; - - // ink_assert(!((long)item&(f->alignment-1))); XXX - why is this no longer working? -bcall - #ifdef DEADBEEF { static const char str[4] = {static_cast(0xde), static_cast(0xad), static_cast(0xbe), static_cast(0xef)}; // set the entire item to DEADBEEF; void *temp = head; - for (size_t i = 0; i < num_item; i++) { + for (size_t i = 0; i < num_item;) { for (int j = sizeof(void *); j < static_cast(f->type_size); j++) { (static_cast(temp))[j] = str[j % 4]; } - *ADDRESS_OF_NEXT(temp, 0) = FROM_PTR(*ADDRESS_OF_NEXT(temp, 0)); - temp = TO_PTR(*ADDRESS_OF_NEXT(temp, 0)); + ++i; + ink_assert((i < num_item) || (temp == tail)); + temp = *static_cast(temp); } } #endif /* DEADBEEF */ - while (!result) { - INK_QUEUE_LD(h, f->head); #ifdef SANITY - if (TO_PTR(FREELIST_POINTER(h)) == head) { - ink_abort("ink_freelist_free: trying to free item twice"); - } - if (((uintptr_t)(TO_PTR(FREELIST_POINTER(h)))) & 3) { - ink_abort("ink_freelist_free: bad list"); - } - if (TO_PTR(FREELIST_POINTER(h))) { - fake_global_for_ink_queue = *static_cast(TO_PTR(FREELIST_POINTER(h))); - } -#endif /* SANITY */ - *adr_of_next = FREELIST_POINTER(h); - SET_FREELIST_POINTER_VERSION(item_pair, FROM_PTR(head), FREELIST_VERSION(h)); - INK_MEMORY_BARRIER; - result = ink_atomic_cas(&f->head.data, h.data, item_pair.data); + void *old_head = f->head; + if (old_head == head) { + ink_abort("ink_freelist_free: trying to free item twice"); + } + if (reinterpret_cast(old_head) & 3) { + ink_abort("ink_freelist_free: bad list"); + } + if (old_head) { + fake_global_for_ink_queue = *static_cast(old_head); } +#endif /* SANITY */ + + AtomicStackNoOffset(f->head).splice_to_top(head, tail); } static void @@ -481,117 +606,35 @@ ink_atomiclist_init(InkAtomicList *l, const char *name, uint32_t offset_to_next) { l->name = name; l->offset = offset_to_next; - SET_FREELIST_POINTER_VERSION(l->head, FROM_PTR(0), 0); + l->head = nullptr; } void * ink_atomiclist_pop(InkAtomicList *l) { - head_p item; - head_p next; - int result = 0; - do { - INK_QUEUE_LD(item, l->head); - if (TO_PTR(FREELIST_POINTER(item)) == nullptr) { - return nullptr; - } - SET_FREELIST_POINTER_VERSION(next, *ADDRESS_OF_NEXT(TO_PTR(FREELIST_POINTER(item)), l->offset), FREELIST_VERSION(item) + 1); - result = ink_atomic_cas(&l->head.data, item.data, next.data); - } while (result == 0); - { - void *ret = TO_PTR(FREELIST_POINTER(item)); - *ADDRESS_OF_NEXT(ret, l->offset) = nullptr; - return ret; - } + return AtomicStackWithOffset(l->head, l->offset).pop<>(); } void * ink_atomiclist_popall(InkAtomicList *l) { - head_p item; - head_p next; - int result = 0; - do { - INK_QUEUE_LD(item, l->head); - if (TO_PTR(FREELIST_POINTER(item)) == nullptr) { - return nullptr; - } - SET_FREELIST_POINTER_VERSION(next, FROM_PTR(nullptr), FREELIST_VERSION(item) + 1); - result = ink_atomic_cas(&l->head.data, item.data, next.data); - } while (result == 0); - { - void *ret = TO_PTR(FREELIST_POINTER(item)); - void *e = ret; - /* fixup forward pointers */ - while (e) { - void *n = TO_PTR(*ADDRESS_OF_NEXT(e, l->offset)); - *ADDRESS_OF_NEXT(e, l->offset) = n; - e = n; - } - return ret; - } + return AtomicStackWithOffset(l->head, l->offset).pop(); } void * ink_atomiclist_push(InkAtomicList *l, void *item) { - void **adr_of_next = ADDRESS_OF_NEXT(item, l->offset); - head_p head; - head_p item_pair; - int result = 0; - void *h = nullptr; - do { - INK_QUEUE_LD(head, l->head); - h = FREELIST_POINTER(head); - *adr_of_next = h; - ink_assert(item != TO_PTR(h)); - SET_FREELIST_POINTER_VERSION(item_pair, FROM_PTR(item), FREELIST_VERSION(head)); - INK_MEMORY_BARRIER; - result = ink_atomic_cas(&l->head.data, head.data, item_pair.data); - } while (result == 0); - - return TO_PTR(h); + return AtomicStackWithOffset(l->head, l->offset).push(item); } void * -ink_atomiclist_remove(InkAtomicList *l, void *item) +ink_atomiclist_next(InkAtomicList *l, void *item) { - head_p head; - void *prev = nullptr; - void **addr_next = ADDRESS_OF_NEXT(item, l->offset); - void *item_next = *addr_next; - int result = 0; - - /* - * first, try to pop it if it is first - */ - INK_QUEUE_LD(head, l->head); - while (TO_PTR(FREELIST_POINTER(head)) == item) { - head_p next; - SET_FREELIST_POINTER_VERSION(next, item_next, FREELIST_VERSION(head) + 1); - result = ink_atomic_cas(&l->head.data, head.data, next.data); - - if (result) { - *addr_next = nullptr; - return item; - } - INK_QUEUE_LD(head, l->head); - } + return item ? AtomicStackWithOffset(l->head, l->offset).link(item) : nullptr; +} - /* - * then, go down the list, trying to remove it - */ - prev = TO_PTR(FREELIST_POINTER(head)); - while (prev) { - void **prev_adr_of_next = ADDRESS_OF_NEXT(prev, l->offset); - void *prev_prev = prev; - prev = TO_PTR(*prev_adr_of_next); - if (prev == item) { - ink_assert(prev_prev != item_next); - *prev_adr_of_next = item_next; - *addr_next = nullptr; - return item; - } - } - return nullptr; +void * +ink_atomiclist_remove(InkAtomicList *l, void *item) +{ + return AtomicStackWithOffset(l->head, l->offset).remove(item) ? item : nullptr; } diff --git a/src/tscore/ink_queue_utils.cc b/src/tscore/ver_ptr_kruft.cc similarity index 96% rename from src/tscore/ink_queue_utils.cc rename to src/tscore/ver_ptr_kruft.cc index 66de0b48565..0ceb3e60035 100644 --- a/src/tscore/ink_queue_utils.cc +++ b/src/tscore/ver_ptr_kruft.cc @@ -21,11 +21,8 @@ limitations under the License. */ -#include "tscore/ink_config.h" -#include - -#include "tscore/ink_atomic.h" -#include "tscore/ink_queue.h" +#include +#include "tscore/ver_ptr_kruft.h" /* * This file was added during the debugging of Bug 50475.