Skip to content

Commit

Permalink
8320807: [PPC64][ZGC] C1 generates wrong code for atomics
Browse files Browse the repository at this point in the history
Backport-of: 3087e14cde9257680f0406b11942f9cb7739cb7b
  • Loading branch information
TheRealMDoerr committed Dec 14, 2023
1 parent 0e0e897 commit 0106db0
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 79 deletions.
45 changes: 33 additions & 12 deletions src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
Expand Up @@ -2635,6 +2635,13 @@ void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
Unimplemented();
}

// There might be a volatile load before this Unsafe CAS.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ sync();
} else {
__ lwsync();
}

if (is_64bit) {
__ cmpxchgd(BOOL_RESULT, /*current_value=*/R0, cmp_value, new_value, addr,
MacroAssembler::MemBarNone,
Expand Down Expand Up @@ -2996,9 +3003,24 @@ void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr
assert(addr->disp() == 0 && addr->index()->is_illegal(), "use leal!");
const Register Rptr = addr->base()->as_pointer_register(),
Rtmp = tmp->as_register();
Register Rco = noreg;
if (UseCompressedOops && data->is_oop()) {
Rco = __ encode_heap_oop(Rtmp, data->as_register());
Register Robj = noreg;
if (data->is_oop()) {
if (UseCompressedOops) {
Robj = __ encode_heap_oop(Rtmp, data->as_register());
} else {
Robj = data->as_register();
if (Robj == dest->as_register()) { // May happen with ZGC.
__ mr(Rtmp, Robj);
Robj = Rtmp;
}
}
}

// There might be a volatile load before this Unsafe OP.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ sync();
} else {
__ lwsync();
}

Label Lretry;
Expand All @@ -3018,18 +3040,11 @@ void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr
} else if (data->is_oop()) {
assert(code == lir_xchg, "xadd for oops");
const Register Rold = dest->as_register();
assert_different_registers(Rptr, Rold, Robj);
if (UseCompressedOops) {
assert_different_registers(Rptr, Rold, Rco);
__ lwarx(Rold, Rptr, MacroAssembler::cmpxchgx_hint_atomic_update());
__ stwcx_(Rco, Rptr);
__ stwcx_(Robj, Rptr);
} else {
Register Robj = data->as_register();
assert_different_registers(Rptr, Rold, Rtmp);
assert_different_registers(Rptr, Robj, Rtmp);
if (Robj == Rold) { // May happen with ZGC.
__ mr(Rtmp, Robj);
Robj = Rtmp;
}
__ ldarx(Rold, Rptr, MacroAssembler::cmpxchgx_hint_atomic_update());
__ stdcx_(Robj, Rptr);
}
Expand Down Expand Up @@ -3057,6 +3072,12 @@ void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr
if (UseCompressedOops && data->is_oop()) {
__ decode_heap_oop(dest->as_register());
}

if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ isync();
} else {
__ sync();
}
}


Expand Down
35 changes: 0 additions & 35 deletions src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
Expand Up @@ -639,13 +639,6 @@ LIR_Opr LIRGenerator::atomic_cmpxchg(BasicType type, LIR_Opr addr, LIRItem& cmp_
cmp_value.load_item();
new_value.load_item();

// Volatile load may be followed by Unsafe CAS.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar();
} else {
__ membar_release();
}

if (is_reference_type(type)) {
if (UseCompressedOops) {
t1 = new_register(T_OBJECT);
Expand All @@ -670,21 +663,7 @@ LIR_Opr LIRGenerator::atomic_xchg(BasicType type, LIR_Opr addr, LIRItem& value)
LIR_Opr tmp = FrameMap::R0_opr;

value.load_item();

// Volatile load may be followed by Unsafe CAS.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar();
} else {
__ membar_release();
}

__ xchg(addr, value.result(), result, tmp);

if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar_acquire();
} else {
__ membar();
}
return result;
}

Expand All @@ -694,21 +673,7 @@ LIR_Opr LIRGenerator::atomic_add(BasicType type, LIR_Opr addr, LIRItem& value) {
LIR_Opr tmp = FrameMap::R0_opr;

value.load_item();

// Volatile load may be followed by Unsafe CAS.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar(); // To be safe. Unsafe semantics are unclear.
} else {
__ membar_release();
}

__ xadd(addr, value.result(), result, tmp);

if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar_acquire();
} else {
__ membar();
}
return result;
}

Expand Down
45 changes: 15 additions & 30 deletions src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
* Copyright (c) 2012, 2021 SAP SE. All rights reserved.
* Copyright (c) 2018, 2023, Red Hat, Inc. All rights reserved.
* Copyright (c) 2012, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -53,8 +53,13 @@ void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler *masm) {
__ encode_heap_oop(new_val, new_val);
}

// Due to the memory barriers emitted in ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved,
// there is no need to specify stronger memory semantics.
// There might be a volatile load before this Unsafe CAS.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ sync();
} else {
__ lwsync();
}

ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmp_val, new_val, tmp1, tmp2,
false, result);

Expand All @@ -63,6 +68,12 @@ void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler *masm) {
__ decode_heap_oop(new_val);
}

if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ isync();
} else {
__ sync();
}

__ block_comment("} LIR_OpShenandoahCompareAndSwap (shenandaohgc)");
}

Expand All @@ -80,14 +91,6 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI
if (access.is_oop()) {
LIRGenerator* gen = access.gen();

if (ShenandoahCASBarrier) {
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar();
} else {
__ membar_release();
}
}

if (ShenandoahSATBBarrier) {
pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
LIR_OprFact::illegalOpr);
Expand All @@ -104,12 +107,6 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI

__ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));

if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar_acquire();
} else {
__ membar();
}

return result;
}
}
Expand All @@ -125,12 +122,6 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRIt
value.load_item();
LIR_Opr value_opr = value.result();

if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar();
} else {
__ membar_release();
}

if (access.is_oop()) {
value_opr = iu_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators());
}
Expand All @@ -152,11 +143,5 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRIt
}
}

if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar_acquire();
} else {
__ membar();
}

return result;
}
5 changes: 3 additions & 2 deletions src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
Expand Up @@ -340,11 +340,12 @@ void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm,
}
__ ld(R0, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread);
__ cmpxchgd(CCR0, tmp, (intptr_t)0, R0, ref_base,
MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update());
MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
noreg, need_restore ? nullptr : &slow_path);
if (need_restore) {
__ subf(ref_base, ind_or_offs, ref_base);
__ bne(CCR0, slow_path);
}
__ bne(CCR0, slow_path);
} else {
// A non-atomic relocatable object won't get to the medium fast path due to a
// raw null in the young generation. We only get here because the field is bad.
Expand Down

1 comment on commit 0106db0

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.