Skip to content

Commit b9873e1

Browse files
committed
8253180: ZGC: Implementation of JEP 376: ZGC: Concurrent Thread-Stack Processing
Reviewed-by: stefank, pliden, rehn, neliasso, coleenp, smonteith
1 parent a2f6519 commit b9873e1

File tree

131 files changed

+2429
-573
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

131 files changed

+2429
-573
lines changed

src/hotspot/cpu/aarch64/aarch64.ad

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1966,9 +1966,10 @@ void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
19661966
}
19671967

19681968
if (do_polling() && C->is_method_compilation()) {
1969-
st->print("# touch polling page\n\t");
1970-
st->print("ldr rscratch1, [rthread],#polling_page_offset\n\t");
1971-
st->print("ldr zr, [rscratch1]");
1969+
st->print("# test polling word\n\t");
1970+
st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1971+
st->print("cmp sp, rscratch1\n\t");
1972+
st->print("bhi #slow_path");
19721973
}
19731974
}
19741975
#endif
@@ -1985,7 +1986,13 @@ void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
19851986
}
19861987

19871988
if (do_polling() && C->is_method_compilation()) {
1988-
__ fetch_and_read_polling_page(rscratch1, relocInfo::poll_return_type);
1989+
Label dummy_label;
1990+
Label* code_stub = &dummy_label;
1991+
if (!C->output()->in_scratch_emit_size()) {
1992+
code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
1993+
}
1994+
__ relocate(relocInfo::poll_return_type);
1995+
__ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
19891996
}
19901997
}
19911998

src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,19 @@
3838

3939
#define __ ce->masm()->
4040

41+
void C1SafepointPollStub::emit_code(LIR_Assembler* ce) {
42+
__ bind(_entry);
43+
InternalAddress safepoint_pc(ce->masm()->pc() - ce->masm()->offset() + safepoint_offset());
44+
__ adr(rscratch1, safepoint_pc);
45+
__ str(rscratch1, Address(rthread, JavaThread::saved_exception_pc_offset()));
46+
47+
assert(SharedRuntime::polling_page_return_handler_blob() != NULL,
48+
"polling page return stub not created yet");
49+
address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();
50+
51+
__ far_jump(RuntimeAddress(stub));
52+
}
53+
4154
void CounterOverflowStub::emit_code(LIR_Assembler* ce) {
4255
__ bind(_entry);
4356
Metadata *m = _method->as_constant_ptr()->as_metadata();

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -504,7 +504,7 @@ void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
504504
}
505505
}
506506

507-
void LIR_Assembler::return_op(LIR_Opr result) {
507+
void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
508508
assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
509509

510510
// Pop the stack before the safepoint code
@@ -514,7 +514,9 @@ void LIR_Assembler::return_op(LIR_Opr result) {
514514
__ reserved_stack_check();
515515
}
516516

517-
__ fetch_and_read_polling_page(rscratch1, relocInfo::poll_return_type);
517+
code_stub->set_safepoint_offset(__ offset());
518+
__ relocate(relocInfo::poll_return_type);
519+
__ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
518520
__ ret(lr);
519521
}
520522

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* This code is free software; you can redistribute it and/or modify it
6+
* under the terms of the GNU General Public License version 2 only, as
7+
* published by the Free Software Foundation.
8+
*
9+
* This code is distributed in the hope that it will be useful, but WITHOUT
10+
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11+
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12+
* version 2 for more details (a copy is included in the LICENSE file that
13+
* accompanied this code).
14+
*
15+
* You should have received a copy of the GNU General Public License version
16+
* 2 along with this work; if not, write to the Free Software Foundation,
17+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18+
*
19+
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20+
* or visit www.oracle.com if you need additional information or have any
21+
* questions.
22+
*
23+
*/
24+
25+
#include "precompiled.hpp"
26+
#include "asm/macroAssembler.hpp"
27+
#include "opto/compile.hpp"
28+
#include "opto/node.hpp"
29+
#include "opto/output.hpp"
30+
#include "runtime/sharedRuntime.hpp"
31+
32+
#define __ masm.
33+
void C2SafepointPollStubTable::emit_stub_impl(MacroAssembler& masm, C2SafepointPollStub* entry) const {
34+
assert(SharedRuntime::polling_page_return_handler_blob() != NULL,
35+
"polling page return stub not created yet");
36+
address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();
37+
38+
RuntimeAddress callback_addr(stub);
39+
40+
__ bind(entry->_stub_label);
41+
InternalAddress safepoint_pc(masm.pc() - masm.offset() + entry->_safepoint_offset);
42+
__ adr(rscratch1, safepoint_pc);
43+
__ str(rscratch1, Address(rthread, JavaThread::saved_exception_pc_offset()));
44+
__ far_jump(callback_addr);
45+
}
46+
#undef __

src/hotspot/cpu/aarch64/frame_aarch64.cpp

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "runtime/monitorChunk.hpp"
3838
#include "runtime/os.inline.hpp"
3939
#include "runtime/signature.hpp"
40+
#include "runtime/stackWatermarkSet.hpp"
4041
#include "runtime/stubCodeGenerator.hpp"
4142
#include "runtime/stubRoutines.hpp"
4243
#include "vmreg_aarch64.inline.hpp"
@@ -476,8 +477,8 @@ frame frame::sender_for_compiled_frame(RegisterMap* map) const {
476477
}
477478

478479
//------------------------------------------------------------------------------
479-
// frame::sender
480-
frame frame::sender(RegisterMap* map) const {
480+
// frame::sender_raw
481+
frame frame::sender_raw(RegisterMap* map) const {
481482
// Default is we done have to follow them. The sender_for_xxx will
482483
// update it accordingly
483484
map->set_include_argument_oops(false);
@@ -499,6 +500,16 @@ frame frame::sender(RegisterMap* map) const {
499500
return frame(sender_sp(), link(), sender_pc());
500501
}
501502

503+
frame frame::sender(RegisterMap* map) const {
504+
frame result = sender_raw(map);
505+
506+
if (map->process_frames()) {
507+
StackWatermarkSet::on_iteration(map->thread(), result);
508+
}
509+
510+
return result;
511+
}
512+
502513
bool frame::is_interpreted_frame_valid(JavaThread* thread) const {
503514
assert(is_interpreted_frame(), "Not an interpreted frame");
504515
// These are reasonable sanity checks

src/hotspot/cpu/aarch64/frame_aarch64.hpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,4 +161,7 @@
161161

162162
static jint interpreter_frame_expression_stack_direction() { return -1; }
163163

164+
// returns the sending frame, without applying any barriers
165+
frame sender_raw(RegisterMap* map) const;
166+
164167
#endif // CPU_AARCH64_FRAME_AARCH64_HPP

src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,9 @@
2424
#ifndef CPU_AARCH64_GC_Z_ZGLOBALS_AARCH64_HPP
2525
#define CPU_AARCH64_GC_Z_ZGLOBALS_AARCH64_HPP
2626

27-
const size_t ZPlatformGranuleSizeShift = 21; // 2MB
28-
const size_t ZPlatformHeapViews = 3;
29-
const size_t ZPlatformNMethodDisarmedOffset = 4;
30-
const size_t ZPlatformCacheLineSize = 64;
27+
const size_t ZPlatformGranuleSizeShift = 21; // 2MB
28+
const size_t ZPlatformHeapViews = 3;
29+
const size_t ZPlatformCacheLineSize = 64;
3130

3231
size_t ZPlatformAddressOffsetBits();
3332
size_t ZPlatformAddressMetadataShift();

src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -473,7 +473,7 @@ void InterpreterMacroAssembler::dispatch_base(TosState state,
473473

474474
if (needs_thread_local_poll) {
475475
NOT_PRODUCT(block_comment("Thread-local Safepoint poll"));
476-
ldr(rscratch2, Address(rthread, Thread::polling_page_offset()));
476+
ldr(rscratch2, Address(rthread, Thread::polling_word_offset()));
477477
tbnz(rscratch2, exact_log2(SafepointMechanism::poll_bit()), safepoint);
478478
}
479479

@@ -521,6 +521,7 @@ void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {
521521

522522
// remove activation
523523
//
524+
// Apply stack watermark barrier.
524525
// Unlock the receiver if this is a synchronized method.
525526
// Unlock any Java monitors from syncronized blocks.
526527
// Remove the activation from the stack.
@@ -541,6 +542,19 @@ void InterpreterMacroAssembler::remove_activation(
541542
// result check if synchronized method
542543
Label unlocked, unlock, no_unlock;
543544

545+
// The below poll is for the stack watermark barrier. It allows fixing up frames lazily,
546+
// that would normally not be safe to use. Such bad returns into unsafe territory of
547+
// the stack, will call InterpreterRuntime::at_unwind.
548+
Label slow_path;
549+
Label fast_path;
550+
safepoint_poll(slow_path, true /* at_return */, false /* acquire */, false /* in_nmethod */);
551+
br(Assembler::AL, fast_path);
552+
bind(slow_path);
553+
push(state);
554+
call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind));
555+
pop(state);
556+
bind(fast_path);
557+
544558
// get the value of _do_not_unlock_if_synchronized into r3
545559
const Address do_not_unlock_if_synchronized(rthread,
546560
in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

Lines changed: 15 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -288,27 +288,21 @@ address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) {
288288
return address(((uint64_t)insn_addr + (offset << 2)));
289289
}
290290

291-
void MacroAssembler::safepoint_poll(Label& slow_path) {
292-
ldr(rscratch1, Address(rthread, Thread::polling_page_offset()));
293-
tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);
294-
}
295-
296-
// Just like safepoint_poll, but use an acquiring load for thread-
297-
// local polling.
298-
//
299-
// We need an acquire here to ensure that any subsequent load of the
300-
// global SafepointSynchronize::_state flag is ordered after this load
301-
// of the local Thread::_polling page. We don't want this poll to
302-
// return false (i.e. not safepointing) and a later poll of the global
303-
// SafepointSynchronize::_state spuriously to return true.
304-
//
305-
// This is to avoid a race when we're in a native->Java transition
306-
// racing the code which wakes up from a safepoint.
307-
//
308-
void MacroAssembler::safepoint_poll_acquire(Label& slow_path) {
309-
lea(rscratch1, Address(rthread, Thread::polling_page_offset()));
310-
ldar(rscratch1, rscratch1);
311-
tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);
291+
void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod) {
292+
if (acquire) {
293+
lea(rscratch1, Address(rthread, Thread::polling_word_offset()));
294+
ldar(rscratch1, rscratch1);
295+
} else {
296+
ldr(rscratch1, Address(rthread, Thread::polling_word_offset()));
297+
}
298+
if (at_return) {
299+
// Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
300+
// we may safely use the sp instead to perform the stack watermark check.
301+
cmp(in_nmethod ? sp : rfp, rscratch1);
302+
br(Assembler::HI, slow_path);
303+
} else {
304+
tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);
305+
}
312306
}
313307

314308
void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
@@ -4405,13 +4399,6 @@ void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype)
44054399
ldr(dest, Address(rthread, Thread::polling_page_offset()));
44064400
}
44074401

4408-
// Move the address of the polling page into r, then read the polling
4409-
// page.
4410-
address MacroAssembler::fetch_and_read_polling_page(Register r, relocInfo::relocType rtype) {
4411-
get_polling_page(r, rtype);
4412-
return read_polling_page(r, rtype);
4413-
}
4414-
44154402
// Read the polling page. The address of the polling page must
44164403
// already be in r.
44174404
address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) {

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,7 @@ class MacroAssembler: public Assembler {
102102
virtual void check_and_handle_popframe(Register java_thread);
103103
virtual void check_and_handle_earlyret(Register java_thread);
104104

105-
void safepoint_poll(Label& slow_path);
106-
void safepoint_poll_acquire(Label& slow_path);
105+
void safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod);
107106

108107
// Biased locking support
109108
// lock_reg and obj_reg must be loaded up with the appropriate values.
@@ -1231,7 +1230,6 @@ class MacroAssembler: public Assembler {
12311230

12321231
address read_polling_page(Register r, relocInfo::relocType rtype);
12331232
void get_polling_page(Register dest, relocInfo::relocType rtype);
1234-
address fetch_and_read_polling_page(Register r, relocInfo::relocType rtype);
12351233

12361234
// CRC32 code for java.util.zip.CRC32::updateBytes() instrinsic.
12371235
void update_byte_crc32(Register crc, Register val, Register table);

0 commit comments

Comments
 (0)