From 17106c9e9dfdce60ae96079d73d1fa5057f96f29 Mon Sep 17 00:00:00 2001 From: Stefan Karlsson Date: Fri, 24 Jan 2020 09:15:08 +0100 Subject: [PATCH] 8236778: Add Atomic::fetch_and_add Reviewed-by: kbarrett, dholmes --- src/hotspot/os_cpu/aix_ppc/atomic_aix_ppc.hpp | 9 +- src/hotspot/os_cpu/bsd_x86/atomic_bsd_x86.hpp | 9 +- .../os_cpu/bsd_zero/atomic_bsd_zero.hpp | 9 +- .../linux_aarch64/atomic_linux_aarch64.hpp | 9 +- .../os_cpu/linux_arm/atomic_linux_arm.hpp | 9 +- .../os_cpu/linux_ppc/atomic_linux_ppc.hpp | 9 +- .../os_cpu/linux_s390/atomic_linux_s390.hpp | 9 +- .../os_cpu/linux_sparc/atomic_linux_sparc.hpp | 9 +- .../os_cpu/linux_x86/atomic_linux_x86.hpp | 9 +- .../os_cpu/linux_zero/atomic_linux_zero.hpp | 9 +- .../solaris_sparc/atomic_solaris_sparc.hpp | 7 +- .../os_cpu/solaris_x86/atomic_solaris_x86.hpp | 9 +- .../os_cpu/windows_x86/atomic_windows_x86.hpp | 9 +- src/hotspot/share/gc/g1/g1ConcurrentMark.cpp | 6 +- src/hotspot/share/gc/g1/g1HotCardCache.cpp | 2 +- .../share/gc/g1/g1PageBasedVirtualSpace.cpp | 2 +- src/hotspot/share/gc/g1/g1RemSet.cpp | 6 +- .../share/gc/parallel/psParallelCompact.cpp | 2 +- .../shared/stringdedup/stringDedupQueue.cpp | 2 +- .../shared/stringdedup/stringDedupTable.cpp | 2 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 2 +- .../share/gc/shenandoah/shenandoahNMethod.cpp | 2 +- .../shenandoah/shenandoahNMethod.inline.hpp | 2 +- .../gc/shenandoah/shenandoahVerifier.cpp | 2 +- src/hotspot/share/gc/z/zArray.inline.hpp | 2 +- .../share/gc/z/zMarkStackAllocator.cpp | 2 +- .../share/gc/z/zNMethodTableIteration.cpp | 2 +- src/hotspot/share/gc/z/zPageAllocator.cpp | 2 +- .../share/gc/z/zRelocationSet.inline.hpp | 2 +- src/hotspot/share/gc/z/zRootsIterator.cpp | 2 +- src/hotspot/share/runtime/atomic.hpp | 137 +++++++----------- .../concurrentHashTableTasks.inline.hpp | 2 +- 32 files changed, 151 insertions(+), 145 deletions(-) diff --git a/src/hotspot/os_cpu/aix_ppc/atomic_aix_ppc.hpp b/src/hotspot/os_cpu/aix_ppc/atomic_aix_ppc.hpp index 7eb9b830a61..dbcd356eada 100644 --- a/src/hotspot/os_cpu/aix_ppc/atomic_aix_ppc.hpp +++ b/src/hotspot/os_cpu/aix_ppc/atomic_aix_ppc.hpp @@ -93,11 +93,14 @@ inline void post_membar(atomic_memory_order order) { template -struct Atomic::PlatformAdd - : Atomic::AddAndFetch > -{ +struct Atomic::PlatformAdd { template D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const; + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; template<> diff --git a/src/hotspot/os_cpu/bsd_x86/atomic_bsd_x86.hpp b/src/hotspot/os_cpu/bsd_x86/atomic_bsd_x86.hpp index e727d56075e..292fc788b94 100644 --- a/src/hotspot/os_cpu/bsd_x86/atomic_bsd_x86.hpp +++ b/src/hotspot/os_cpu/bsd_x86/atomic_bsd_x86.hpp @@ -28,11 +28,14 @@ // Implementation of class atomic template -struct Atomic::PlatformAdd - : Atomic::FetchAndAdd > -{ +struct Atomic::PlatformAdd { template D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order /* order */) const; + + template + D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const { + return fetch_and_add(dest, add_value, order) + add_value; + } }; template<> diff --git a/src/hotspot/os_cpu/bsd_zero/atomic_bsd_zero.hpp b/src/hotspot/os_cpu/bsd_zero/atomic_bsd_zero.hpp index a042c204fb2..c702cec5b68 100644 --- a/src/hotspot/os_cpu/bsd_zero/atomic_bsd_zero.hpp +++ b/src/hotspot/os_cpu/bsd_zero/atomic_bsd_zero.hpp @@ -160,11 +160,14 @@ static inline int arm_lock_test_and_set(int newval, volatile int *ptr) { #endif // ARM template -struct Atomic::PlatformAdd - : Atomic::AddAndFetch > -{ +struct Atomic::PlatformAdd { template D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const; + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; template<> diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp index 3e5e6b5b139..8e275a4173e 100644 --- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp +++ b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp @@ -33,15 +33,18 @@ // See https://patchwork.kernel.org/patch/3575821/ template -struct Atomic::PlatformAdd - : Atomic::AddAndFetch > -{ +struct Atomic::PlatformAdd { template D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const { D res = __atomic_add_fetch(dest, add_value, __ATOMIC_RELEASE); FULL_MEM_BARRIER; return res; } + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; template diff --git a/src/hotspot/os_cpu/linux_arm/atomic_linux_arm.hpp b/src/hotspot/os_cpu/linux_arm/atomic_linux_arm.hpp index 8029d4e93ae..c6d77168718 100644 --- a/src/hotspot/os_cpu/linux_arm/atomic_linux_arm.hpp +++ b/src/hotspot/os_cpu/linux_arm/atomic_linux_arm.hpp @@ -67,11 +67,14 @@ inline void Atomic::PlatformStore<8>::operator()(T volatile* dest, // For ARMv7 we add explicit barriers in the stubs. template -struct Atomic::PlatformAdd - : Atomic::AddAndFetch > -{ +struct Atomic::PlatformAdd { template D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const; + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; template<> diff --git a/src/hotspot/os_cpu/linux_ppc/atomic_linux_ppc.hpp b/src/hotspot/os_cpu/linux_ppc/atomic_linux_ppc.hpp index 6be6eb04bf3..f471b05a9ff 100644 --- a/src/hotspot/os_cpu/linux_ppc/atomic_linux_ppc.hpp +++ b/src/hotspot/os_cpu/linux_ppc/atomic_linux_ppc.hpp @@ -93,11 +93,14 @@ inline void post_membar(atomic_memory_order order) { template -struct Atomic::PlatformAdd - : Atomic::AddAndFetch > -{ +struct Atomic::PlatformAdd { template D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const; + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; template<> diff --git a/src/hotspot/os_cpu/linux_s390/atomic_linux_s390.hpp b/src/hotspot/os_cpu/linux_s390/atomic_linux_s390.hpp index 3183a040a9b..a29c41eb713 100644 --- a/src/hotspot/os_cpu/linux_s390/atomic_linux_s390.hpp +++ b/src/hotspot/os_cpu/linux_s390/atomic_linux_s390.hpp @@ -75,11 +75,14 @@ inline void z196_fast_sync() { } template -struct Atomic::PlatformAdd - : Atomic::AddAndFetch > -{ +struct Atomic::PlatformAdd { template D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const; + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; template<> diff --git a/src/hotspot/os_cpu/linux_sparc/atomic_linux_sparc.hpp b/src/hotspot/os_cpu/linux_sparc/atomic_linux_sparc.hpp index 2ee8ad5d9c4..9f5f98982fb 100644 --- a/src/hotspot/os_cpu/linux_sparc/atomic_linux_sparc.hpp +++ b/src/hotspot/os_cpu/linux_sparc/atomic_linux_sparc.hpp @@ -28,11 +28,14 @@ // Implementation of class atomic template -struct Atomic::PlatformAdd - : Atomic::AddAndFetch > -{ +struct Atomic::PlatformAdd { template D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const; + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; template<> diff --git a/src/hotspot/os_cpu/linux_x86/atomic_linux_x86.hpp b/src/hotspot/os_cpu/linux_x86/atomic_linux_x86.hpp index 3ac96fc2977..e4bba0cfa58 100644 --- a/src/hotspot/os_cpu/linux_x86/atomic_linux_x86.hpp +++ b/src/hotspot/os_cpu/linux_x86/atomic_linux_x86.hpp @@ -28,11 +28,14 @@ // Implementation of class atomic template -struct Atomic::PlatformAdd - : Atomic::FetchAndAdd > -{ +struct Atomic::PlatformAdd { template D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const; + + template + D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const { + return fetch_and_add(dest, add_value, order) + add_value; + } }; template<> diff --git a/src/hotspot/os_cpu/linux_zero/atomic_linux_zero.hpp b/src/hotspot/os_cpu/linux_zero/atomic_linux_zero.hpp index 7b6b8060c29..dbd2fb0c587 100644 --- a/src/hotspot/os_cpu/linux_zero/atomic_linux_zero.hpp +++ b/src/hotspot/os_cpu/linux_zero/atomic_linux_zero.hpp @@ -31,11 +31,14 @@ // Implementation of class atomic template -struct Atomic::PlatformAdd - : Atomic::AddAndFetch > -{ +struct Atomic::PlatformAdd { template D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const; + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; template<> diff --git a/src/hotspot/os_cpu/solaris_sparc/atomic_solaris_sparc.hpp b/src/hotspot/os_cpu/solaris_sparc/atomic_solaris_sparc.hpp index 3b4c2cc2741..de82951663d 100644 --- a/src/hotspot/os_cpu/solaris_sparc/atomic_solaris_sparc.hpp +++ b/src/hotspot/os_cpu/solaris_sparc/atomic_solaris_sparc.hpp @@ -31,7 +31,7 @@ template struct Atomic::PlatformAdd { template - inline D operator()(D volatile* dest, I add_value, atomic_memory_order order) const { + inline D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const { D old_value = *dest; while (true) { D new_value = old_value + add_value; @@ -41,6 +41,11 @@ struct Atomic::PlatformAdd { } return old_value + add_value; } + + template + inline D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; template<> diff --git a/src/hotspot/os_cpu/solaris_x86/atomic_solaris_x86.hpp b/src/hotspot/os_cpu/solaris_x86/atomic_solaris_x86.hpp index 390f57dc3e1..4873e22716d 100644 --- a/src/hotspot/os_cpu/solaris_x86/atomic_solaris_x86.hpp +++ b/src/hotspot/os_cpu/solaris_x86/atomic_solaris_x86.hpp @@ -41,11 +41,14 @@ extern "C" { } template -struct Atomic::PlatformAdd - : Atomic::AddAndFetch > -{ +struct Atomic::PlatformAdd { template D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const; + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; // Not using add_using_helper; see comment for cmpxchg. diff --git a/src/hotspot/os_cpu/windows_x86/atomic_windows_x86.hpp b/src/hotspot/os_cpu/windows_x86/atomic_windows_x86.hpp index 25915e61910..d428724de41 100644 --- a/src/hotspot/os_cpu/windows_x86/atomic_windows_x86.hpp +++ b/src/hotspot/os_cpu/windows_x86/atomic_windows_x86.hpp @@ -54,11 +54,14 @@ template<> inline void ScopedFence::postfix() { OrderAccess::fe #pragma warning(disable: 4035) // Disables warnings reporting missing return statement template -struct Atomic::PlatformAdd - : Atomic::AddAndFetch > -{ +struct Atomic::PlatformAdd { template D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const; + + template + D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) const { + return add_and_fetch(dest, add_value, order) - add_value; + } }; #ifdef AMD64 diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp index 4fb863ae296..2e19f942dc5 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp @@ -209,7 +209,7 @@ G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::allocate_new_chunk() { return NULL; } - size_t cur_idx = Atomic::add(&_hwm, 1u) - 1; + size_t cur_idx = Atomic::fetch_and_add(&_hwm, 1u); if (cur_idx >= _chunk_capacity) { return NULL; } @@ -282,7 +282,7 @@ void G1CMRootMemRegions::reset() { void G1CMRootMemRegions::add(HeapWord* start, HeapWord* end) { assert_at_safepoint(); - size_t idx = Atomic::add(&_num_root_regions, (size_t)1) - 1; + size_t idx = Atomic::fetch_and_add(&_num_root_regions, 1u); assert(idx < _max_regions, "Trying to add more root MemRegions than there is space " SIZE_FORMAT, _max_regions); assert(start != NULL && end != NULL && start <= end, "Start (" PTR_FORMAT ") should be less or equal to " "end (" PTR_FORMAT ")", p2i(start), p2i(end)); @@ -310,7 +310,7 @@ const MemRegion* G1CMRootMemRegions::claim_next() { return NULL; } - size_t claimed_index = Atomic::add(&_claimed_root_regions, (size_t)1) - 1; + size_t claimed_index = Atomic::fetch_and_add(&_claimed_root_regions, 1u); if (claimed_index < _num_root_regions) { return &_root_regions[claimed_index]; } diff --git a/src/hotspot/share/gc/g1/g1HotCardCache.cpp b/src/hotspot/share/gc/g1/g1HotCardCache.cpp index 067a0670199..4a7a3a24129 100644 --- a/src/hotspot/share/gc/g1/g1HotCardCache.cpp +++ b/src/hotspot/share/gc/g1/g1HotCardCache.cpp @@ -70,7 +70,7 @@ CardTable::CardValue* G1HotCardCache::insert(CardValue* card_ptr) { return card_ptr; } // Otherwise, the card is hot. - size_t index = Atomic::add(&_hot_cache_idx, 1u) - 1; + size_t index = Atomic::fetch_and_add(&_hot_cache_idx, 1u); if (index == _hot_cache_size) { // Can use relaxed store because all racing threads are writing the same // value and there aren't any concurrent readers. diff --git a/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp b/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp index 84b45ae510d..1b2480bd5fd 100644 --- a/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp +++ b/src/hotspot/share/gc/g1/g1PageBasedVirtualSpace.cpp @@ -261,7 +261,7 @@ class G1PretouchTask : public AbstractGangTask { virtual void work(uint worker_id) { size_t const actual_chunk_size = MAX2(chunk_size(), _page_size); while (true) { - char* touch_addr = Atomic::add(&_cur_addr, actual_chunk_size) - actual_chunk_size; + char* touch_addr = Atomic::fetch_and_add(&_cur_addr, actual_chunk_size); if (touch_addr < _start_addr || touch_addr >= _end_addr) { break; } diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp index 99025621654..b2a21ba10dc 100644 --- a/src/hotspot/share/gc/g1/g1RemSet.cpp +++ b/src/hotspot/share/gc/g1/g1RemSet.cpp @@ -180,7 +180,7 @@ class G1RemSetScanState : public CHeapObj { bool marked_as_dirty = Atomic::cmpxchg(&_contains[region], false, true) == false; if (marked_as_dirty) { - uint allocated = Atomic::add(&_cur_idx, 1u) - 1; + uint allocated = Atomic::fetch_and_add(&_cur_idx, 1u); _buffer[allocated] = region; } } @@ -232,7 +232,7 @@ class G1RemSetScanState : public CHeapObj { void work(uint worker_id) { while (_cur_dirty_regions < _regions->size()) { - uint next = Atomic::add(&_cur_dirty_regions, _chunk_length) - _chunk_length; + uint next = Atomic::fetch_and_add(&_cur_dirty_regions, _chunk_length); uint max = MIN2(next + _chunk_length, _regions->size()); for (uint i = next; i < max; i++) { @@ -429,7 +429,7 @@ class G1RemSetScanState : public CHeapObj { uint claim_cards_to_scan(uint region, uint increment) { assert(region < _max_regions, "Tried to access invalid region %u", region); - return Atomic::add(&_card_table_scan_state[region], increment) - increment; + return Atomic::fetch_and_add(&_card_table_scan_state[region], increment); } void add_dirty_region(uint const region) { diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp index b132cc481fb..ec26ce70802 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp @@ -2452,7 +2452,7 @@ class TaskQueue : StackObj { } bool try_claim(PSParallelCompact::UpdateDensePrefixTask& reference) { - uint claimed = Atomic::add(&_counter, 1u) - 1; // -1 is so that we start with zero + uint claimed = Atomic::fetch_and_add(&_counter, 1u); if (claimed < _insert_index) { reference = _backing_array[claimed]; return true; diff --git a/src/hotspot/share/gc/shared/stringdedup/stringDedupQueue.cpp b/src/hotspot/share/gc/shared/stringdedup/stringDedupQueue.cpp index d338ea5e8dd..e4cf7d2f175 100644 --- a/src/hotspot/share/gc/shared/stringdedup/stringDedupQueue.cpp +++ b/src/hotspot/share/gc/shared/stringdedup/stringDedupQueue.cpp @@ -32,7 +32,7 @@ StringDedupQueue* StringDedupQueue::_queue = NULL; volatile size_t StringDedupQueue::_claimed_index = 0; size_t StringDedupQueue::claim() { - return Atomic::add(&_claimed_index, size_t(1)) - 1; + return Atomic::fetch_and_add(&_claimed_index, 1u); } void StringDedupQueue::unlink_or_oops_do(StringDedupUnlinkOrOopsDoClosure* cl) { diff --git a/src/hotspot/share/gc/shared/stringdedup/stringDedupTable.cpp b/src/hotspot/share/gc/shared/stringdedup/stringDedupTable.cpp index 39d60b6d71f..ee7e8efc9a4 100644 --- a/src/hotspot/share/gc/shared/stringdedup/stringDedupTable.cpp +++ b/src/hotspot/share/gc/shared/stringdedup/stringDedupTable.cpp @@ -592,7 +592,7 @@ void StringDedupTable::finish_rehash(StringDedupTable* rehashed_table) { } size_t StringDedupTable::claim_table_partition(size_t partition_size) { - return Atomic::add(&_claimed_index, partition_size) - partition_size; + return Atomic::fetch_and_add(&_claimed_index, partition_size); } void StringDedupTable::verify() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index bfae410cc28..19aa942ab6b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1362,7 +1362,7 @@ class ShenandoahParallelHeapRegionTask : public AbstractGangTask { size_t max = _heap->num_regions(); while (_index < max) { - size_t cur = Atomic::add(&_index, stride) - stride; + size_t cur = Atomic::fetch_and_add(&_index, stride); size_t start = cur; size_t end = MIN2(cur + stride, max); if (start >= max) break; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp index 3ee22b73ece..11a5f6c525d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp @@ -484,7 +484,7 @@ void ShenandoahNMethodTableSnapshot::concurrent_nmethods_do(NMethodClosure* cl) ShenandoahNMethod** list = _array; size_t max = (size_t)_length; while (_claimed < max) { - size_t cur = Atomic::add(&_claimed, stride) - stride; + size_t cur = Atomic::fetch_and_add(&_claimed, stride); size_t start = cur; size_t end = MIN2(cur + stride, max); if (start >= max) break; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp index 7ee0c3e35d9..36fafabbd3d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp @@ -90,7 +90,7 @@ void ShenandoahNMethodTableSnapshot::parallel_blobs_do(CodeBlobClosure *f) { size_t max = (size_t)_length; while (_claimed < max) { - size_t cur = Atomic::add(&_claimed, stride) - stride; + size_t cur = Atomic::fetch_and_add(&_claimed, stride); size_t start = cur; size_t end = MIN2(cur + stride, max); if (start >= max) break; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index 6d21e690ea4..4c21a1a29c6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -522,7 +522,7 @@ class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { _options); while (true) { - size_t v = Atomic::add(&_claimed, 1u) - 1; + size_t v = Atomic::fetch_and_add(&_claimed, 1u); if (v < _heap->num_regions()) { ShenandoahHeapRegion* r = _heap->get_region(v); if (!r->is_humongous() && !r->is_trash()) { diff --git a/src/hotspot/share/gc/z/zArray.inline.hpp b/src/hotspot/share/gc/z/zArray.inline.hpp index f141be02f5b..870dd2ec421 100644 --- a/src/hotspot/share/gc/z/zArray.inline.hpp +++ b/src/hotspot/share/gc/z/zArray.inline.hpp @@ -101,7 +101,7 @@ inline ZArrayIteratorImpl::ZArrayIteratorImpl(ZArray* array) : template inline bool ZArrayIteratorImpl::next(T* elem) { if (parallel) { - const size_t next = Atomic::add(&_next, 1u) - 1u; + const size_t next = Atomic::fetch_and_add(&_next, 1u); if (next < _array->size()) { *elem = _array->at(next); return true; diff --git a/src/hotspot/share/gc/z/zMarkStackAllocator.cpp b/src/hotspot/share/gc/z/zMarkStackAllocator.cpp index 8e17024f260..b965d68010a 100644 --- a/src/hotspot/share/gc/z/zMarkStackAllocator.cpp +++ b/src/hotspot/share/gc/z/zMarkStackAllocator.cpp @@ -110,7 +110,7 @@ uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) { // Increment top before end to make sure another // thread can't steal out newly expanded space. - addr = Atomic::add(&_top, size) - size; + addr = Atomic::fetch_and_add(&_top, size); Atomic::add(&_end, expand_size); return addr; diff --git a/src/hotspot/share/gc/z/zNMethodTableIteration.cpp b/src/hotspot/share/gc/z/zNMethodTableIteration.cpp index c74b1c56c0d..b3f9471bad7 100644 --- a/src/hotspot/share/gc/z/zNMethodTableIteration.cpp +++ b/src/hotspot/share/gc/z/zNMethodTableIteration.cpp @@ -58,7 +58,7 @@ void ZNMethodTableIteration::nmethods_do(NMethodClosure* cl) { // Claim table partition. Each partition is currently sized to span // two cache lines. This number is just a guess, but seems to work well. const size_t partition_size = (ZCacheLineSize * 2) / sizeof(ZNMethodTableEntry); - const size_t partition_start = MIN2(Atomic::add(&_claimed, partition_size) - partition_size, _size); + const size_t partition_start = MIN2(Atomic::fetch_and_add(&_claimed, partition_size), _size); const size_t partition_end = MIN2(partition_start + partition_size, _size); if (partition_start == partition_end) { // End of table diff --git a/src/hotspot/share/gc/z/zPageAllocator.cpp b/src/hotspot/share/gc/z/zPageAllocator.cpp index 3849f5f02d7..773e595dc49 100644 --- a/src/hotspot/share/gc/z/zPageAllocator.cpp +++ b/src/hotspot/share/gc/z/zPageAllocator.cpp @@ -176,7 +176,7 @@ class ZPreTouchTask : public ZTask { for (;;) { // Get granule offset const size_t size = ZGranuleSize; - const uintptr_t offset = Atomic::add(&_start, size) - size; + const uintptr_t offset = Atomic::fetch_and_add(&_start, size); if (offset >= _end) { // Done break; diff --git a/src/hotspot/share/gc/z/zRelocationSet.inline.hpp b/src/hotspot/share/gc/z/zRelocationSet.inline.hpp index a7e8c85c748..52f3a281707 100644 --- a/src/hotspot/share/gc/z/zRelocationSet.inline.hpp +++ b/src/hotspot/share/gc/z/zRelocationSet.inline.hpp @@ -38,7 +38,7 @@ inline bool ZRelocationSetIteratorImpl::next(ZForwarding** forwarding) if (parallel) { if (_next < nforwardings) { - const size_t next = Atomic::add(&_next, 1u) - 1u; + const size_t next = Atomic::fetch_and_add(&_next, 1u); if (next < nforwardings) { *forwarding = _relocation_set->_forwardings[next]; return true; diff --git a/src/hotspot/share/gc/z/zRootsIterator.cpp b/src/hotspot/share/gc/z/zRootsIterator.cpp index 885a9e39938..b1ad519f452 100644 --- a/src/hotspot/share/gc/z/zRootsIterator.cpp +++ b/src/hotspot/share/gc/z/zRootsIterator.cpp @@ -185,7 +185,7 @@ ZJavaThreadsIterator::ZJavaThreadsIterator() : _claimed(0) {} uint ZJavaThreadsIterator::claim() { - return Atomic::add(&_claimed, 1u) - 1u; + return Atomic::fetch_and_add(&_claimed, 1u); } void ZJavaThreadsIterator::threads_do(ThreadClosure* cl) { diff --git a/src/hotspot/share/runtime/atomic.hpp b/src/hotspot/share/runtime/atomic.hpp index 7e3bae33bc0..e66772e3025 100644 --- a/src/hotspot/share/runtime/atomic.hpp +++ b/src/hotspot/share/runtime/atomic.hpp @@ -98,13 +98,19 @@ class Atomic : AllStatic { template inline static T load_acquire(const volatile T* dest); - // Atomically add to a location. Returns updated value. add*() provide: + // Atomically add to a location. *add*() provide: // add-value-to-dest + // Returns updated value. template inline static D add(D volatile* dest, I add_value, atomic_memory_order order = memory_order_conservative); + // Returns previous value. + template + inline static D fetch_and_add(D volatile* dest, I add_value, + atomic_memory_order order = memory_order_conservative); + template inline static D sub(D volatile* dest, I sub_value, atomic_memory_order order = memory_order_conservative); @@ -230,53 +236,33 @@ WINDOWS_ONLY(public:) // VS2017 warns (C2027) use of undefined type if IsPointer // Platform-specific implementation of add. Support for sizes of 4 // bytes and (if different) pointer size bytes are required. The - // class is a function object that must be default constructable, - // with these requirements: + // class must be default constructable, with these requirements: // // - dest is of type D*, an integral or pointer type. // - add_value is of type I, an integral type. // - sizeof(I) == sizeof(D). // - if D is an integral type, I == D. + // - order is of type atomic_memory_order. // - platform_add is an object of type PlatformAdd. // - // Then - // platform_add(dest, add_value) - // must be a valid expression, returning a result convertible to D. - // - // No definition is provided; all platforms must explicitly define - // this class and any needed specializations. - template struct PlatformAdd; - - // Helper base classes for defining PlatformAdd. To use, define - // PlatformAdd or a specialization that derives from one of these, - // and include in the PlatformAdd definition the support function - // (described below) required by the base class. + // Then both + // platform_add.add_and_fetch(dest, add_value, order) + // platform_add.fetch_and_add(dest, add_value, order) + // must be valid expressions returning a result convertible to D. // - // These classes implement the required function object protocol for - // PlatformAdd, using a support function template provided by the - // derived class. Let add_value (of type I) and dest (of type D) be - // the arguments the object is called with. If D is a pointer type - // P*, then let addend (of type I) be add_value * sizeof(P); - // otherwise, addend is add_value. + // add_and_fetch atomically adds add_value to the value of dest, + // returning the new value. // - // FetchAndAdd requires the derived class to provide - // fetch_and_add(dest, addend) - // atomically adding addend to the value of dest, and returning the - // old value. + // fetch_and_add atomically adds add_value to the value of dest, + // returning the old value. // - // AddAndFetch requires the derived class to provide - // add_and_fetch(dest, addend) - // atomically adding addend to the value of dest, and returning the - // new value. + // When D is a pointer type P*, both add_and_fetch and fetch_and_add + // treat it as if it were an uintptr_t; they do not perform any + // scaling of add_value, as that has already been done by the caller. // - // When D is a pointer type P*, both fetch_and_add and add_and_fetch - // treat it as if it were a uintptr_t; they do not perform any - // scaling of the addend, as that has already been done by the - // caller. -public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11. - template struct FetchAndAdd; - template struct AddAndFetch; -private: + // No definition is provided; all platforms must explicitly define + // this class and any needed specializations. + template struct PlatformAdd; // Support for platforms that implement some variants of add using a // (typically out of line) non-template helper function. The @@ -512,22 +498,6 @@ struct Atomic::PlatformStore { } }; -// Define FetchAndAdd and AddAndFetch helper classes before including -// platform file, which may use these as base classes, requiring they -// be complete. - -template -struct Atomic::FetchAndAdd { - template - D operator()(D volatile* dest, I add_value, atomic_memory_order order) const; -}; - -template -struct Atomic::AddAndFetch { - template - D operator()(D volatile* dest, I add_value, atomic_memory_order order) const; -}; - template inline void Atomic::inc(D volatile* dest, atomic_memory_order order) { STATIC_ASSERT(IsPointer::value || IsIntegral::value); @@ -684,7 +654,13 @@ inline void Atomic::release_store_fence(volatile D* p, T v) { template inline D Atomic::add(D volatile* dest, I add_value, atomic_memory_order order) { - return AddImpl()(dest, add_value, order); + return AddImpl::add_and_fetch(dest, add_value, order); +} + +template +inline D Atomic::fetch_and_add(D volatile* dest, I add_value, + atomic_memory_order order) { + return AddImpl::fetch_and_add(dest, add_value, order); } template @@ -695,9 +671,13 @@ struct Atomic::AddImpl< (sizeof(I) <= sizeof(D)) && (IsSigned::value == IsSigned::value)>::type> { - D operator()(D volatile* dest, I add_value, atomic_memory_order order) const { + static D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) { + D addend = add_value; + return PlatformAdd().add_and_fetch(dest, addend, order); + } + static D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) { D addend = add_value; - return PlatformAdd()(dest, addend, order); + return PlatformAdd().fetch_and_add(dest, addend, order); } }; @@ -706,40 +686,25 @@ struct Atomic::AddImpl< P*, I, typename EnableIf::value && (sizeof(I) <= sizeof(P*))>::type> { - P* operator()(P* volatile* dest, I add_value, atomic_memory_order order) const { - STATIC_ASSERT(sizeof(intptr_t) == sizeof(P*)); - STATIC_ASSERT(sizeof(uintptr_t) == sizeof(P*)); - typedef typename Conditional::value, - intptr_t, - uintptr_t>::type CI; - CI addend = add_value; - return PlatformAdd()(dest, addend, order); + STATIC_ASSERT(sizeof(intptr_t) == sizeof(P*)); + STATIC_ASSERT(sizeof(uintptr_t) == sizeof(P*)); + typedef typename Conditional::value, + intptr_t, + uintptr_t>::type CI; + + static CI scale_addend(CI add_value) { + return add_value * sizeof(P); } -}; -template -template -inline D Atomic::FetchAndAdd::operator()(D volatile* dest, I add_value, - atomic_memory_order order) const { - I addend = add_value; - // If D is a pointer type P*, scale by sizeof(P). - if (IsPointer::value) { - addend *= sizeof(typename RemovePointer::type); + static P* add_and_fetch(P* volatile* dest, I add_value, atomic_memory_order order) { + CI addend = add_value; + return PlatformAdd().add_and_fetch(dest, scale_addend(addend), order); } - D old = static_cast(this)->fetch_and_add(dest, addend, order); - return old + add_value; -} - -template -template -inline D Atomic::AddAndFetch::operator()(D volatile* dest, I add_value, - atomic_memory_order order) const { - // If D is a pointer type P*, scale by sizeof(P). - if (IsPointer::value) { - add_value *= sizeof(typename RemovePointer::type); + static P* fetch_and_add(P* volatile* dest, I add_value, atomic_memory_order order) { + CI addend = add_value; + return PlatformAdd().fetch_and_add(dest, scale_addend(addend), order); } - return static_cast(this)->add_and_fetch(dest, add_value, order); -} +}; template inline D Atomic::add_using_helper(Fn fn, D volatile* dest, I add_value) { diff --git a/src/hotspot/share/utilities/concurrentHashTableTasks.inline.hpp b/src/hotspot/share/utilities/concurrentHashTableTasks.inline.hpp index 11c50fbd9cf..190f85084ce 100644 --- a/src/hotspot/share/utilities/concurrentHashTableTasks.inline.hpp +++ b/src/hotspot/share/utilities/concurrentHashTableTasks.inline.hpp @@ -54,7 +54,7 @@ class ConcurrentHashTable::BucketsOperation { // Returns true if you succeeded to claim the range start -> (stop-1). bool claim(size_t* start, size_t* stop) { - size_t claimed = Atomic::add(&_next_to_claim, (size_t)1) - 1; + size_t claimed = Atomic::fetch_and_add(&_next_to_claim, 1u); if (claimed >= _stop_task) { return false; }