Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
8253180: ZGC: Implementation of JEP 376: ZGC: Concurrent Thread-Stack…
… Processing

Reviewed-by: stefank, pliden, rehn, neliasso, coleenp, smonteith
  • Loading branch information
fisk committed Oct 9, 2020
1 parent a2f6519 commit b9873e1
Show file tree
Hide file tree
Showing 131 changed files with 2,429 additions and 573 deletions.
15 changes: 11 additions & 4 deletions src/hotspot/cpu/aarch64/aarch64.ad
Expand Up @@ -1966,9 +1966,10 @@ void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
}

if (do_polling() && C->is_method_compilation()) {
st->print("# touch polling page\n\t");
st->print("ldr rscratch1, [rthread],#polling_page_offset\n\t");
st->print("ldr zr, [rscratch1]");
st->print("# test polling word\n\t");
st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
st->print("cmp sp, rscratch1\n\t");
st->print("bhi #slow_path");
}
}
#endif
Expand All @@ -1985,7 +1986,13 @@ void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
}

if (do_polling() && C->is_method_compilation()) {
__ fetch_and_read_polling_page(rscratch1, relocInfo::poll_return_type);
Label dummy_label;
Label* code_stub = &dummy_label;
if (!C->output()->in_scratch_emit_size()) {
code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
}
__ relocate(relocInfo::poll_return_type);
__ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
}
}

Expand Down
13 changes: 13 additions & 0 deletions src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp
Expand Up @@ -38,6 +38,19 @@

#define __ ce->masm()->

void C1SafepointPollStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
InternalAddress safepoint_pc(ce->masm()->pc() - ce->masm()->offset() + safepoint_offset());
__ adr(rscratch1, safepoint_pc);
__ str(rscratch1, Address(rthread, JavaThread::saved_exception_pc_offset()));

assert(SharedRuntime::polling_page_return_handler_blob() != NULL,
"polling page return stub not created yet");
address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();

__ far_jump(RuntimeAddress(stub));
}

void CounterOverflowStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
Metadata *m = _method->as_constant_ptr()->as_metadata();
Expand Down
6 changes: 4 additions & 2 deletions src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
Expand Up @@ -504,7 +504,7 @@ void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
}
}

void LIR_Assembler::return_op(LIR_Opr result) {
void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");

// Pop the stack before the safepoint code
Expand All @@ -514,7 +514,9 @@ void LIR_Assembler::return_op(LIR_Opr result) {
__ reserved_stack_check();
}

__ fetch_and_read_polling_page(rscratch1, relocInfo::poll_return_type);
code_stub->set_safepoint_offset(__ offset());
__ relocate(relocInfo::poll_return_type);
__ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
__ ret(lr);
}

Expand Down
46 changes: 46 additions & 0 deletions src/hotspot/cpu/aarch64/c2_safepointPollStubTable_aarch64.cpp
@@ -0,0 +1,46 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/

#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "opto/compile.hpp"
#include "opto/node.hpp"
#include "opto/output.hpp"
#include "runtime/sharedRuntime.hpp"

#define __ masm.
void C2SafepointPollStubTable::emit_stub_impl(MacroAssembler& masm, C2SafepointPollStub* entry) const {
assert(SharedRuntime::polling_page_return_handler_blob() != NULL,
"polling page return stub not created yet");
address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();

RuntimeAddress callback_addr(stub);

__ bind(entry->_stub_label);
InternalAddress safepoint_pc(masm.pc() - masm.offset() + entry->_safepoint_offset);
__ adr(rscratch1, safepoint_pc);
__ str(rscratch1, Address(rthread, JavaThread::saved_exception_pc_offset()));
__ far_jump(callback_addr);
}
#undef __
15 changes: 13 additions & 2 deletions src/hotspot/cpu/aarch64/frame_aarch64.cpp
Expand Up @@ -37,6 +37,7 @@
#include "runtime/monitorChunk.hpp"
#include "runtime/os.inline.hpp"
#include "runtime/signature.hpp"
#include "runtime/stackWatermarkSet.hpp"
#include "runtime/stubCodeGenerator.hpp"
#include "runtime/stubRoutines.hpp"
#include "vmreg_aarch64.inline.hpp"
Expand Down Expand Up @@ -476,8 +477,8 @@ frame frame::sender_for_compiled_frame(RegisterMap* map) const {
}

//------------------------------------------------------------------------------
// frame::sender
frame frame::sender(RegisterMap* map) const {
// frame::sender_raw
frame frame::sender_raw(RegisterMap* map) const {
// Default is we done have to follow them. The sender_for_xxx will
// update it accordingly
map->set_include_argument_oops(false);
Expand All @@ -499,6 +500,16 @@ frame frame::sender(RegisterMap* map) const {
return frame(sender_sp(), link(), sender_pc());
}

frame frame::sender(RegisterMap* map) const {
frame result = sender_raw(map);

if (map->process_frames()) {
StackWatermarkSet::on_iteration(map->thread(), result);
}

return result;
}

bool frame::is_interpreted_frame_valid(JavaThread* thread) const {
assert(is_interpreted_frame(), "Not an interpreted frame");
// These are reasonable sanity checks
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/cpu/aarch64/frame_aarch64.hpp
Expand Up @@ -161,4 +161,7 @@

static jint interpreter_frame_expression_stack_direction() { return -1; }

// returns the sending frame, without applying any barriers
frame sender_raw(RegisterMap* map) const;

#endif // CPU_AARCH64_FRAME_AARCH64_HPP
7 changes: 3 additions & 4 deletions src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp
Expand Up @@ -24,10 +24,9 @@
#ifndef CPU_AARCH64_GC_Z_ZGLOBALS_AARCH64_HPP
#define CPU_AARCH64_GC_Z_ZGLOBALS_AARCH64_HPP

const size_t ZPlatformGranuleSizeShift = 21; // 2MB
const size_t ZPlatformHeapViews = 3;
const size_t ZPlatformNMethodDisarmedOffset = 4;
const size_t ZPlatformCacheLineSize = 64;
const size_t ZPlatformGranuleSizeShift = 21; // 2MB
const size_t ZPlatformHeapViews = 3;
const size_t ZPlatformCacheLineSize = 64;

size_t ZPlatformAddressOffsetBits();
size_t ZPlatformAddressMetadataShift();
Expand Down
16 changes: 15 additions & 1 deletion src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
Expand Up @@ -473,7 +473,7 @@ void InterpreterMacroAssembler::dispatch_base(TosState state,

if (needs_thread_local_poll) {
NOT_PRODUCT(block_comment("Thread-local Safepoint poll"));
ldr(rscratch2, Address(rthread, Thread::polling_page_offset()));
ldr(rscratch2, Address(rthread, Thread::polling_word_offset()));
tbnz(rscratch2, exact_log2(SafepointMechanism::poll_bit()), safepoint);
}

Expand Down Expand Up @@ -521,6 +521,7 @@ void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {

// remove activation
//
// Apply stack watermark barrier.
// Unlock the receiver if this is a synchronized method.
// Unlock any Java monitors from syncronized blocks.
// Remove the activation from the stack.
Expand All @@ -541,6 +542,19 @@ void InterpreterMacroAssembler::remove_activation(
// result check if synchronized method
Label unlocked, unlock, no_unlock;

// The below poll is for the stack watermark barrier. It allows fixing up frames lazily,
// that would normally not be safe to use. Such bad returns into unsafe territory of
// the stack, will call InterpreterRuntime::at_unwind.
Label slow_path;
Label fast_path;
safepoint_poll(slow_path, true /* at_return */, false /* acquire */, false /* in_nmethod */);
br(Assembler::AL, fast_path);
bind(slow_path);
push(state);
call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind));
pop(state);
bind(fast_path);

// get the value of _do_not_unlock_if_synchronized into r3
const Address do_not_unlock_if_synchronized(rthread,
in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
Expand Down
43 changes: 15 additions & 28 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
Expand Up @@ -288,27 +288,21 @@ address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) {
return address(((uint64_t)insn_addr + (offset << 2)));
}

void MacroAssembler::safepoint_poll(Label& slow_path) {
ldr(rscratch1, Address(rthread, Thread::polling_page_offset()));
tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);
}

// Just like safepoint_poll, but use an acquiring load for thread-
// local polling.
//
// We need an acquire here to ensure that any subsequent load of the
// global SafepointSynchronize::_state flag is ordered after this load
// of the local Thread::_polling page. We don't want this poll to
// return false (i.e. not safepointing) and a later poll of the global
// SafepointSynchronize::_state spuriously to return true.
//
// This is to avoid a race when we're in a native->Java transition
// racing the code which wakes up from a safepoint.
//
void MacroAssembler::safepoint_poll_acquire(Label& slow_path) {
lea(rscratch1, Address(rthread, Thread::polling_page_offset()));
ldar(rscratch1, rscratch1);
tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);
void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod) {
if (acquire) {
lea(rscratch1, Address(rthread, Thread::polling_word_offset()));
ldar(rscratch1, rscratch1);
} else {
ldr(rscratch1, Address(rthread, Thread::polling_word_offset()));
}
if (at_return) {
// Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
// we may safely use the sp instead to perform the stack watermark check.
cmp(in_nmethod ? sp : rfp, rscratch1);
br(Assembler::HI, slow_path);
} else {
tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);
}
}

void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
Expand Down Expand Up @@ -4405,13 +4399,6 @@ void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype)
ldr(dest, Address(rthread, Thread::polling_page_offset()));
}

// Move the address of the polling page into r, then read the polling
// page.
address MacroAssembler::fetch_and_read_polling_page(Register r, relocInfo::relocType rtype) {
get_polling_page(r, rtype);
return read_polling_page(r, rtype);
}

// Read the polling page. The address of the polling page must
// already be in r.
address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) {
Expand Down
4 changes: 1 addition & 3 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
Expand Up @@ -102,8 +102,7 @@ class MacroAssembler: public Assembler {
virtual void check_and_handle_popframe(Register java_thread);
virtual void check_and_handle_earlyret(Register java_thread);

void safepoint_poll(Label& slow_path);
void safepoint_poll_acquire(Label& slow_path);
void safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod);

// Biased locking support
// lock_reg and obj_reg must be loaded up with the appropriate values.
Expand Down Expand Up @@ -1231,7 +1230,6 @@ class MacroAssembler: public Assembler {

address read_polling_page(Register r, relocInfo::relocType rtype);
void get_polling_page(Register dest, relocInfo::relocType rtype);
address fetch_and_read_polling_page(Register r, relocInfo::relocType rtype);

// CRC32 code for java.util.zip.CRC32::updateBytes() instrinsic.
void update_byte_crc32(Register crc, Register val, Register table);
Expand Down
11 changes: 10 additions & 1 deletion src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
Expand Up @@ -1877,7 +1877,16 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// check for safepoint operation in progress and/or pending suspend requests
Label safepoint_in_progress, safepoint_in_progress_done;
{
__ safepoint_poll_acquire(safepoint_in_progress);
// We need an acquire here to ensure that any subsequent load of the
// global SafepointSynchronize::_state flag is ordered after this load
// of the thread-local polling word. We don't want this poll to
// return false (i.e. not safepointing) and a later poll of the global
// SafepointSynchronize::_state spuriously to return true.
//
// This is to avoid a race when we're in a native->Java transition
// racing the code which wakes up from a safepoint.

__ safepoint_poll(safepoint_in_progress, true /* at_return */, true /* acquire */, false /* in_nmethod */);
__ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
__ cbnzw(rscratch1, safepoint_in_progress);
__ bind(safepoint_in_progress_done);
Expand Down
15 changes: 12 additions & 3 deletions src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
Expand Up @@ -980,7 +980,7 @@ address TemplateInterpreterGenerator::generate_CRC32_update_entry() {

Label slow_path;
// If we need a safepoint check, generate full interpreter entry.
__ safepoint_poll(slow_path);
__ safepoint_poll(slow_path, false /* at_return */, false /* acquire */, false /* in_nmethod */);

// We don't generate local frame and don't align stack because
// we call stub code and there is no safepoint on this path.
Expand Down Expand Up @@ -1029,7 +1029,7 @@ address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractI

Label slow_path;
// If we need a safepoint check, generate full interpreter entry.
__ safepoint_poll(slow_path);
__ safepoint_poll(slow_path, false /* at_return */, false /* acquire */, false /* in_nmethod */);

// We don't generate local frame and don't align stack because
// we call stub code and there is no safepoint on this path.
Expand Down Expand Up @@ -1388,7 +1388,16 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
// check for safepoint operation in progress and/or pending suspend requests
{
Label L, Continue;
__ safepoint_poll_acquire(L);

// We need an acquire here to ensure that any subsequent load of the
// global SafepointSynchronize::_state flag is ordered after this load
// of the thread-local polling word. We don't want this poll to
// return false (i.e. not safepointing) and a later poll of the global
// SafepointSynchronize::_state spuriously to return true.
//
// This is to avoid a race when we're in a native->Java transition
// racing the code which wakes up from a safepoint.
__ safepoint_poll(L, true /* at_return */, true /* acquire */, false /* in_nmethod */);
__ ldrw(rscratch2, Address(rthread, JavaThread::suspend_flags_offset()));
__ cbz(rscratch2, Continue);
__ bind(L);
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
Expand Up @@ -128,6 +128,7 @@ class VM_Version : public Abstract_VM_Version {
static int get_initial_sve_vector_length() { return _initial_sve_vector_length; };

static bool supports_fast_class_init_checks() { return true; }
constexpr static bool supports_stack_watermark_barrier() { return true; }
};

#endif // CPU_AARCH64_VM_VERSION_AARCH64_HPP
4 changes: 4 additions & 0 deletions src/hotspot/cpu/arm/c1_CodeStubs_arm.cpp
Expand Up @@ -38,6 +38,10 @@

#define __ ce->masm()->

void C1SafepointPollStub::emit_code(LIR_Assembler* ce) {
ShouldNotReachHere();
}

void CounterOverflowStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
ce->store_parameter(_bci, 0);
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
Expand Up @@ -283,7 +283,7 @@ int LIR_Assembler::emit_deopt_handler() {
}


void LIR_Assembler::return_op(LIR_Opr result) {
void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
// Pop the frame before safepoint polling
__ remove_frame(initial_frame_size_in_bytes());
__ read_polling_page(Rtemp, relocInfo::poll_return_type);
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/cpu/arm/interp_masm_arm.cpp
Expand Up @@ -580,7 +580,7 @@ void InterpreterMacroAssembler::dispatch_base(TosState state,

if (needs_thread_local_poll) {
NOT_PRODUCT(block_comment("Thread-local Safepoint poll"));
ldr(Rtemp, Address(Rthread, Thread::polling_page_offset()));
ldr(Rtemp, Address(Rthread, Thread::polling_word_offset()));
tbnz(Rtemp, exact_log2(SafepointMechanism::poll_bit()), safepoint);
}

Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/cpu/arm/macroAssembler_arm.cpp
Expand Up @@ -1914,7 +1914,7 @@ void MacroAssembler::resolve(DecoratorSet decorators, Register obj) {
}

void MacroAssembler::safepoint_poll(Register tmp1, Label& slow_path) {
ldr_u32(tmp1, Address(Rthread, Thread::polling_page_offset()));
ldr_u32(tmp1, Address(Rthread, Thread::polling_word_offset()));
tst(tmp1, exact_log2(SafepointMechanism::poll_bit()));
b(slow_path, eq);
}
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp