@@ -35,7 +35,7 @@ source %{
3535
3636static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
3737 if (barrier_data == ZLoadBarrierElided) {
38- return; // Elided.
38+ return;
3939 }
4040 ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
4141 {
@@ -60,6 +60,27 @@ static void z_load_barrier_cmpxchg(MacroAssembler& _masm, const MachNode* node,
6060 __ bind(*stub->continuation());
6161}
6262
63+ static void z_cmpxchg_common(MacroAssembler& _masm, const MachNode* node, Register mem_reg, Register newval, Register tmp) {
64+ // Compare value (oldval) is in rax
65+ const Address mem = Address(mem_reg, 0);
66+
67+ if (node->barrier_data() != ZLoadBarrierElided) {
68+ __ movptr(tmp, rax);
69+ }
70+
71+ __ lock();
72+ __ cmpxchgptr(newval, mem);
73+
74+ if (node->barrier_data() != ZLoadBarrierElided) {
75+ Label good;
76+ z_load_barrier_cmpxchg(_masm, node, mem, rax, tmp, good);
77+ __ movptr(rax, tmp);
78+ __ lock();
79+ __ cmpxchgptr(newval, mem);
80+ __ bind(good);
81+ }
82+ }
83+
6384%}
6485
6586// Load Pointer
@@ -81,7 +102,7 @@ instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr)
81102 ins_pipe(ialu_reg_mem);
82103%}
83104
84- instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{
105+ instruct zCompareAndExchangeP(indirect mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{
85106 match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
86107 predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
87108 effect(KILL cr, TEMP tmp);
@@ -90,26 +111,14 @@ instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP t
90111 "cmpxchgq $newval, $mem" %}
91112
92113 ins_encode %{
93- if (barrier_data() != ZLoadBarrierElided) { // barrier could be elided by ZBarrierSetC2::analyze_dominating_barriers()
94- __ movptr($tmp$$Register, $oldval$$Register);
95- }
96- __ lock();
97- __ cmpxchgptr($newval$$Register, $mem$$Address);
98-
99- if (barrier_data() != ZLoadBarrierElided) {
100- Label good;
101- z_load_barrier_cmpxchg(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register, good);
102- __ movptr($oldval$$Register, $tmp$$Register);
103- __ lock();
104- __ cmpxchgptr($newval$$Register, $mem$$Address);
105- __ bind(good);
106- }
114+ precond($oldval$$Register == rax);
115+ z_cmpxchg_common(_masm, this, $mem$$Register, $newval$$Register, $tmp$$Register);
107116 %}
108117
109118 ins_pipe(pipe_cmpxchg);
110119%}
111120
112- instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{
121+ instruct zCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{
113122 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
114123 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
115124 predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
@@ -121,20 +130,10 @@ instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlags
121130 "movzbl $res, $res" %}
122131
123132 ins_encode %{
124- if (barrier_data() != ZLoadBarrierElided) { // barrier could be elided by ZBarrierSetC2::analyze_dominating_barriers()
125- __ movptr($tmp$$Register, $oldval$$Register);
126- }
127- __ lock();
128- __ cmpxchgptr($newval$$Register, $mem$$Address);
129-
133+ precond($oldval$$Register == rax);
134+ z_cmpxchg_common(_masm, this, $mem$$Register, $newval$$Register, $tmp$$Register);
130135 if (barrier_data() != ZLoadBarrierElided) {
131- Label good;
132- z_load_barrier_cmpxchg(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register, good);
133- __ movptr($oldval$$Register, $tmp$$Register);
134- __ lock();
135- __ cmpxchgptr($newval$$Register, $mem$$Address);
136- __ bind(good);
137- __ cmpptr($tmp$$Register, $oldval$$Register);
136+ __ cmpptr($tmp$$Register, rax);
138137 }
139138 __ setb(Assembler::equal, $res$$Register);
140139 __ movzbl($res$$Register, $res$$Register);
@@ -143,15 +142,15 @@ instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlags
143142 ins_pipe(pipe_cmpxchg);
144143%}
145144
146- instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{
145+ instruct zXChgP(indirect mem, rRegP newval, rFlagsReg cr) %{
147146 match(Set newval (GetAndSetP mem newval));
148147 predicate(UseZGC && n->as_LoadStore()->barrier_data() != 0);
149148 effect(KILL cr);
150149
151150 format %{ "xchgq $newval, $mem" %}
152151
153152 ins_encode %{
154- __ xchgptr($newval$$Register, $mem$$Address );
153+ __ xchgptr($newval$$Register, Address( $mem$$Register, 0) );
155154 z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, barrier_data());
156155 %}
157156
0 commit comments