Skip to content
Permalink
Browse files
8274615: Support relaxed atomic add for linux-aarch64
Reviewed-by: aph, dholmes
  • Loading branch information
Kim Barrett committed Oct 12, 2021
1 parent 7d2633f commit 8de26361f7d789c7b317536198c891756038a8ea
Showing 4 changed files with 64 additions and 13 deletions.
@@ -37,6 +37,8 @@ typedef uint64_t (*aarch64_atomic_stub_t)(volatile void *ptr, uint64_t arg1, uin
// Pointers to stubs
extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_4_impl;
extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_8_impl;
extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_4_relaxed_impl;
extern aarch64_atomic_stub_t aarch64_atomic_fetch_add_8_relaxed_impl;
extern aarch64_atomic_stub_t aarch64_atomic_xchg_4_impl;
extern aarch64_atomic_stub_t aarch64_atomic_xchg_8_impl;
extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_1_impl;
@@ -6209,10 +6209,16 @@ class StubGenerator: public StubCodeGenerator {
__ ret(lr);
}

void gen_ldaddal_entry(Assembler::operand_size size) {
void gen_ldadd_entry(Assembler::operand_size size, atomic_memory_order order) {
Register prev = r2, addr = c_rarg0, incr = c_rarg1;
__ ldaddal(size, incr, prev, addr);
__ membar(Assembler::StoreStore|Assembler::StoreLoad);
// If not relaxed, then default to conservative. Relaxed is the only
// case we use enough to be worth specializing.
if (order == memory_order_relaxed) {
__ ldadd(size, incr, prev, addr);
} else {
__ ldaddal(size, incr, prev, addr);
__ membar(Assembler::StoreStore|Assembler::StoreLoad);
}
if (size == Assembler::xword) {
__ mov(r0, prev);
} else {
@@ -6242,12 +6248,21 @@ class StubGenerator: public StubCodeGenerator {
StubCodeMark mark(this, "StubRoutines", "atomic entry points");
address first_entry = __ pc();

// All memory_order_conservative
// ADD, memory_order_conservative
AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl);
gen_ldaddal_entry(Assembler::word);
gen_ldadd_entry(Assembler::word, memory_order_conservative);
AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl);
gen_ldaddal_entry(Assembler::xword);
gen_ldadd_entry(Assembler::xword, memory_order_conservative);

// ADD, memory_order_relaxed
AtomicStubMark mark_fetch_add_4_relaxed
(_masm, &aarch64_atomic_fetch_add_4_relaxed_impl);
gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed);
AtomicStubMark mark_fetch_add_8_relaxed
(_masm, &aarch64_atomic_fetch_add_8_relaxed_impl);
gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed);

// XCHG, memory_order_conservative
AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl);
gen_swpal_entry(Assembler::word);
AtomicStubMark mark_xchg_8_impl(_masm, &aarch64_atomic_xchg_8_impl);
@@ -7447,6 +7462,8 @@ void StubGenerator_generate(CodeBuffer* code, bool all) {

DEFAULT_ATOMIC_OP(fetch_add, 4, )
DEFAULT_ATOMIC_OP(fetch_add, 8, )
DEFAULT_ATOMIC_OP(fetch_add, 4, _relaxed)
DEFAULT_ATOMIC_OP(fetch_add, 8, _relaxed)
DEFAULT_ATOMIC_OP(xchg, 4, )
DEFAULT_ATOMIC_OP(xchg, 8, )
DEFAULT_ATOMIC_OP(cmpxchg, 1, )
@@ -47,6 +47,28 @@ aarch64_atomic_fetch_add_4_default_impl:
mov w0, w2
ret

.global aarch64_atomic_fetch_add_8_relaxed_default_impl
.align 5
aarch64_atomic_fetch_add_8_relaxed_default_impl:
prfm pstl1strm, [x0]
0: ldxr x2, [x0]
add x8, x2, x1
stxr w9, x8, [x0]
cbnz w9, 0b
mov x0, x2
ret

.global aarch64_atomic_fetch_add_4_relaxed_default_impl
.align 5
aarch64_atomic_fetch_add_4_relaxed_default_impl:
prfm pstl1strm, [x0]
0: ldxr w2, [x0]
add w8, w2, w1
stxr w9, w8, [x0]
cbnz w9, 0b
mov w0, w2
ret

.globl aarch64_atomic_xchg_4_default_impl
.align 5
aarch64_atomic_xchg_4_default_impl:
@@ -1,5 +1,5 @@
/*
* Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -87,9 +87,14 @@ inline D Atomic::PlatformAdd<4>::fetch_and_add(D volatile* dest, I add_value,
atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(I));
STATIC_ASSERT(4 == sizeof(D));
D old_value
= atomic_fastcall(aarch64_atomic_fetch_add_4_impl, dest, add_value);
return old_value;
aarch64_atomic_stub_t stub;
switch (order) {
case memory_order_relaxed:
stub = aarch64_atomic_fetch_add_4_relaxed_impl; break;
default:
stub = aarch64_atomic_fetch_add_4_impl; break;
}
return atomic_fastcall(stub, dest, add_value);
}

template<>
@@ -98,9 +103,14 @@ inline D Atomic::PlatformAdd<8>::fetch_and_add(D volatile* dest, I add_value,
atomic_memory_order order) const {
STATIC_ASSERT(8 == sizeof(I));
STATIC_ASSERT(8 == sizeof(D));
D old_value
= atomic_fastcall(aarch64_atomic_fetch_add_8_impl, dest, add_value);
return old_value;
aarch64_atomic_stub_t stub;
switch (order) {
case memory_order_relaxed:
stub = aarch64_atomic_fetch_add_8_relaxed_impl; break;
default:
stub = aarch64_atomic_fetch_add_8_impl; break;
}
return atomic_fastcall(stub, dest, add_value);
}

template<>

1 comment on commit 8de2636

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on 8de2636 Oct 12, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.