Skip to content
Permalink
Browse files

8241673: Shenandoah: refactor anti-false-sharing padding

Reviewed-by: rkennke
  • Loading branch information
shipilev committed Mar 26, 2020
1 parent 8c5531b commit 6be46f8ebbfa2b13d6bfab6d16dbf61ee591a00a
@@ -29,6 +29,7 @@
#include "gc/shenandoah/shenandoahSharedVariables.hpp"
#include "gc/shenandoah/shenandoahLock.hpp"
#include "gc/shenandoah/shenandoahNMethod.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "memory/allocation.hpp"
#include "memory/iterator.hpp"
#include "utilities/globalDefinitions.hpp"
@@ -40,10 +41,10 @@ class ShenandoahParallelCodeHeapIterator {
friend class CodeCache;
private:
CodeHeap* _heap;
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int));
shenandoah_padding(0);
volatile int _claimed_idx;
volatile bool _finished;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);
public:
ShenandoahParallelCodeHeapIterator(CodeHeap* heap);
void parallel_blobs_do(CodeBlobClosure* f);
@@ -28,6 +28,7 @@
#include "memory/allocation.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"

class ShenandoahCollectionSet : public CHeapObj<mtGC> {
friend class ShenandoahHeap;
@@ -46,9 +47,9 @@ class ShenandoahCollectionSet : public CHeapObj<mtGC> {
size_t _used;
size_t _region_count;

DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
shenandoah_padding(0);
volatile jint _current_index;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

public:
ShenandoahCollectionSet(ShenandoahHeap* heap, char* heap_base, size_t size);
@@ -28,6 +28,7 @@
#include "gc/shared/gcCause.hpp"
#include "gc/shared/concurrentGCThread.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "gc/shenandoah/shenandoahSharedVariables.hpp"
#include "runtime/task.hpp"
#include "utilities/ostream.hpp"
@@ -85,9 +86,9 @@ class ShenandoahControlThread: public ConcurrentGCThread {
GCCause::Cause _requested_gc_cause;
ShenandoahHeap::ShenandoahDegenPoint _degen_point;

DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
shenandoah_padding(0);
volatile size_t _allocs_seen;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

bool check_cancellation_or_degen(ShenandoahHeap::ShenandoahDegenPoint point);
void service_concurrent_normal_cycle(GCCause::Cause cause);
@@ -25,6 +25,7 @@
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHEVACOOMHANDLER_HPP

#include "gc/shenandoah/shenandoahPadding.hpp"
#include "memory/allocation.hpp"
#include "utilities/globalDefinitions.hpp"

@@ -81,9 +82,9 @@ class ShenandoahEvacOOMHandler {
private:
static const jint OOM_MARKER_MASK;

DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile jint));
shenandoah_padding(0);
volatile jint _threads_in_evac;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

void wait_for_no_evac_threads();

@@ -54,6 +54,7 @@
#include "gc/shenandoah/shenandoahNormalMode.hpp"
#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "gc/shenandoah/shenandoahPacer.inline.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
#include "gc/shenandoah/shenandoahPassiveMode.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
@@ -1321,9 +1322,9 @@ class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
ShenandoahHeap* const _heap;
ShenandoahHeapRegionClosure* const _blk;

DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
shenandoah_padding(0);
volatile size_t _index;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

public:
ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
@@ -32,6 +32,7 @@
#include "gc/shenandoah/shenandoahAllocRequest.hpp"
#include "gc/shenandoah/shenandoahLock.hpp"
#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "gc/shenandoah/shenandoahSharedVariables.hpp"
#include "gc/shenandoah/shenandoahUnload.hpp"
#include "services/memoryManager.hpp"
@@ -66,9 +67,9 @@ class ShenandoahRegionIterator : public StackObj {
private:
ShenandoahHeap* _heap;

DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
shenandoah_padding(0);
volatile size_t _index;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

// No implicit copying: iterators should be passed by reference to capture the state
NONCOPYABLE(ShenandoahRegionIterator);
@@ -161,11 +162,11 @@ class ShenandoahHeap : public CollectedHeap {
private:
size_t _initial_size;
size_t _minimum_size;
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
shenandoah_padding(0);
volatile size_t _used;
volatile size_t _committed;
volatile size_t _bytes_allocated_since_gc_start;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

public:
void increase_used(size_t bytes);
@@ -30,6 +30,7 @@
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahPacer.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "utilities/sizes.hpp"

class VMStructs;
@@ -224,9 +225,9 @@ class ShenandoahHeapRegion : public CHeapObj<mtGC> {
// Global allocation counter, increased for each allocation under Shenandoah heap lock.
// Padded to avoid false sharing with the read-only fields above.
struct PaddedAllocSeqNum {
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(uint64_t));
shenandoah_padding(0);
uint64_t value;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

PaddedAllocSeqNum() {
// start with 1, reserve 0 for uninitialized value
@@ -265,7 +266,7 @@ class ShenandoahHeapRegion : public CHeapObj<mtGC> {
HeapWord* _update_watermark;

// Claim some space at the end to protect next region
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(0);

public:
ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start, size_t size_words, size_t index, bool committed);
@@ -28,6 +28,7 @@
#include "memory/allocation.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "utilities/globalDefinitions.hpp"

class ShenandoahHeapRegionSet;
@@ -37,9 +38,9 @@ class ShenandoahHeapRegionSetIterator : public StackObj {
const ShenandoahHeapRegionSet* _set;
ShenandoahHeap* const _heap;

DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile jint));
shenandoah_padding(0);
volatile jint _current_index;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

// No implicit copying: iterators should be passed by reference to capture the state
NONCOPYABLE(ShenandoahHeapRegionSetIterator);
@@ -25,6 +25,7 @@
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHLOCK_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHLOCK_HPP

#include "gc/shenandoah/shenandoahPadding.hpp"
#include "memory/allocation.hpp"
#include "runtime/safepoint.hpp"
#include "runtime/thread.hpp"
@@ -33,11 +34,11 @@ class ShenandoahLock {
private:
enum LockState { unlocked = 0, locked = 1 };

DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int));
shenandoah_padding(0);
volatile int _state;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile Thread*));
shenandoah_padding(1);
volatile Thread* _owner;
DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(2);

public:
ShenandoahLock() : _state(unlocked), _owner(NULL) {};
@@ -28,6 +28,7 @@
#include "code/nmethod.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahLock.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "memory/allocation.hpp"
#include "utilities/growableArray.hpp"

@@ -91,9 +92,9 @@ class ShenandoahNMethodTableSnapshot : public CHeapObj<mtGC> {
ShenandoahNMethod** const _array;
const int _length;

DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
shenandoah_padding(0);
volatile size_t _claimed;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

public:
ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table);
@@ -26,6 +26,7 @@
#define SHARE_GC_SHENANDOAH_SHENANDOAHPACER_HPP

#include "gc/shenandoah/shenandoahNumberSeq.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "memory/allocation.hpp"

class ShenandoahHeap;
@@ -50,14 +51,14 @@ class ShenandoahPacer : public CHeapObj<mtGC> {
volatile double _tax_rate;

// Heavily updated, protect from accidental false sharing
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile intptr_t));
shenandoah_padding(0);
volatile intptr_t _budget;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

// Heavily updated, protect from accidental false sharing
DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile intptr_t));
shenandoah_padding(2);
volatile intptr_t _progress;
DEFINE_PAD_MINUS_SIZE(3, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(3);

public:
ShenandoahPacer(ShenandoahHeap* heap) :
@@ -0,0 +1,36 @@
/*
* Copyright (c) 2020, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/

#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHPADDING_INLINE_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHPADDING_INLINE_HPP

// 64 bytes is enough to cover all existing architectures. If we have some
// other platforms, we would need to provide the architecture-specific
// versions here. Shared code provides DEFAULT_CACHE_LINE_SIZE, which is
// inconveniently large by default.

#define shenandoah_padding(id) \
DEFINE_PAD_MINUS_SIZE(id, 64, 0)

#endif // SHARE_GC_SHENANDOAH_SHENANDOAHPADDING_INLINE_HPP
@@ -25,6 +25,7 @@
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSHAREDVARIABLES_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHSHAREDVARIABLES_HPP

#include "gc/shenandoah/shenandoahPadding.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"

@@ -39,9 +40,9 @@ typedef struct ShenandoahSharedFlag {
SET = 1
};

DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile ShenandoahSharedValue));
shenandoah_padding(0);
volatile ShenandoahSharedValue value;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

ShenandoahSharedFlag() {
unset();
@@ -107,9 +108,9 @@ typedef struct ShenandoahSharedFlag {
} ShenandoahSharedFlag;

typedef struct ShenandoahSharedBitmap {
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile ShenandoahSharedValue));
shenandoah_padding(0);
volatile ShenandoahSharedValue value;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

ShenandoahSharedBitmap() {
clear();
@@ -201,9 +202,9 @@ typedef struct ShenandoahSharedBitmap {

template<class T>
struct ShenandoahSharedEnumFlag {
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile ShenandoahSharedValue));
shenandoah_padding(0);
volatile ShenandoahSharedValue value;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

ShenandoahSharedEnumFlag() {
value = 0;
@@ -27,6 +27,7 @@

#include "gc/shared/taskTerminator.hpp"
#include "gc/shared/taskqueue.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"
#include "runtime/mutex.hpp"
@@ -272,9 +273,9 @@ typedef Padded<ShenandoahBufferedOverflowTaskQueue> ShenandoahObjToScanQueue;
template <class T, MEMFLAGS F>
class ParallelClaimableQueueSet: public GenericTaskQueueSet<T, F> {
private:
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile jint));
shenandoah_padding(0);
volatile jint _claimed_index;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
shenandoah_padding(1);

debug_only(uint _reserved; )

0 comments on commit 6be46f8

Please sign in to comment.