Skip to content
Permalink
Browse files

8229971: Arm32: implementation for Thread-local handshakes

Reviewed-by: rehn, dsamersoff
  • Loading branch information
Boris Ulasevich
Boris Ulasevich committed Jan 30, 2020
1 parent c4b708b commit d65f9665dfa2bd95886b08f8eb7676e07a40fa49
@@ -1,5 +1,5 @@
//
// Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -351,10 +351,7 @@ void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

// If this does safepoint polling, then do it here
if (do_polling() && ra_->C->is_method_compilation()) {
// mov_slow here is usually one or two instruction
__ mov_address(Rtemp, (address)os::get_polling_page());
__ relocate(relocInfo::poll_return_type);
__ ldr(Rtemp, Address(Rtemp));
__ read_polling_page(Rtemp, relocInfo::poll_return_type);
}
}

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -285,23 +285,16 @@ int LIR_Assembler::emit_deopt_handler() {
void LIR_Assembler::return_op(LIR_Opr result) {
// Pop the frame before safepoint polling
__ remove_frame(initial_frame_size_in_bytes());

// mov_slow here is usually one or two instruction
__ mov_address(Rtemp, os::get_polling_page());
__ relocate(relocInfo::poll_return_type);
__ ldr(Rtemp, Address(Rtemp));
__ read_polling_page(Rtemp, relocInfo::poll_return_type);
__ ret();
}


int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
__ mov_address(Rtemp, os::get_polling_page());
if (info != NULL) {
add_debug_info_for_branch(info);
}
int offset = __ offset();
__ relocate(relocInfo::poll_type);
__ ldr(Rtemp, Address(Rtemp));
__ read_polling_page(Rtemp, relocInfo::poll_type);
return offset;
}

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -62,4 +62,6 @@ const bool HaveVFP = true;
#endif
#endif

#define THREAD_LOCAL_POLL

#endif // CPU_ARM_GLOBALDEFINITIONS_ARM_HPP
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -42,6 +42,7 @@
#include "runtime/basicLock.hpp"
#include "runtime/biasedLocking.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/safepointMechanism.hpp"
#include "runtime/sharedRuntime.hpp"

//--------------------------------------------------------------------
@@ -556,7 +557,7 @@ void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {

void InterpreterMacroAssembler::dispatch_base(TosState state,
DispatchTableMode table_mode,
bool verifyoop) {
bool verifyoop, bool generate_poll) {
if (VerifyActivationFrameSize) {
Label L;
sub(Rtemp, FP, SP);
@@ -571,6 +572,18 @@ void InterpreterMacroAssembler::dispatch_base(TosState state,
interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
}

Label safepoint;
address* const safepoint_table = Interpreter::safept_table(state);
address* const table = Interpreter::dispatch_table(state);
bool needs_thread_local_poll = generate_poll &&
SafepointMechanism::uses_thread_local_poll() && table != safepoint_table;

if (needs_thread_local_poll) {
NOT_PRODUCT(block_comment("Thread-local Safepoint poll"));
ldr(Rtemp, Address(Rthread, Thread::polling_page_offset()));
tbnz(Rtemp, exact_log2(SafepointMechanism::poll_bit()), safepoint);
}

if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) {
zap_high_non_significant_bits(R0_tos);
}
@@ -600,12 +613,18 @@ void InterpreterMacroAssembler::dispatch_base(TosState state,
indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
}

if (needs_thread_local_poll) {
bind(safepoint);
lea(Rtemp, ExternalAddress((address)safepoint_table));
indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
}

nop(); // to avoid filling CPU pipeline with invalid instructions
nop();
}

void InterpreterMacroAssembler::dispatch_only(TosState state) {
dispatch_base(state, DispatchDefault);
void InterpreterMacroAssembler::dispatch_only(TosState state, bool generate_poll) {
dispatch_base(state, DispatchDefault, true, generate_poll);
}


@@ -617,10 +636,10 @@ void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
dispatch_base(state, DispatchNormal, false);
}

void InterpreterMacroAssembler::dispatch_next(TosState state, int step) {
void InterpreterMacroAssembler::dispatch_next(TosState state, int step, bool generate_poll) {
// load next bytecode and advance Rbcp
ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed));
dispatch_base(state, DispatchDefault);
dispatch_base(state, DispatchDefault, true, generate_poll);
}

void InterpreterMacroAssembler::narrow(Register result) {
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -54,7 +54,7 @@ class InterpreterMacroAssembler: public MacroAssembler {

// base routine for all dispatches
typedef enum { DispatchDefault, DispatchNormal } DispatchTableMode;
void dispatch_base(TosState state, DispatchTableMode table_mode, bool verifyoop = true);
void dispatch_base(TosState state, DispatchTableMode table_mode, bool verifyoop = true, bool generate_poll = false);

public:
InterpreterMacroAssembler(CodeBuffer* code);
@@ -160,10 +160,10 @@ class InterpreterMacroAssembler: public MacroAssembler {
// Dispatching
void dispatch_prolog(TosState state, int step = 0);
void dispatch_epilog(TosState state, int step = 0);
void dispatch_only(TosState state); // dispatch by R3_bytecode
void dispatch_only_normal(TosState state); // dispatch normal table by R3_bytecode
void dispatch_only(TosState state, bool generate_poll = false); // dispatch by R3_bytecode
void dispatch_only_normal(TosState state); // dispatch normal table by R3_bytecode
void dispatch_only_noverify(TosState state);
void dispatch_next(TosState state, int step = 0); // load R3_bytecode from [Rbcp + step] and dispatch by R3_bytecode
void dispatch_next(TosState state, int step = 0, bool generate_poll = false); // load R3_bytecode from [Rbcp + step] and dispatch by R3_bytecode

// jump to an invoked target
void prepare_to_jump_from_interpreted();
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -2053,4 +2053,32 @@ void MacroAssembler::fast_unlock(Register Roop, Register Rbox, Register Rscratch
bind(done);

}

void MacroAssembler::safepoint_poll(Register tmp1, Label& slow_path) {
if (SafepointMechanism::uses_thread_local_poll()) {
ldr_u32(tmp1, Address(Rthread, Thread::polling_page_offset()));
tst(tmp1, exact_log2(SafepointMechanism::poll_bit()));
b(slow_path, eq);
} else {
ldr_global_s32(tmp1, SafepointSynchronize::address_of_state());
cmp(tmp1, SafepointSynchronize::_not_synchronized);
b(slow_path, ne);
}
}

void MacroAssembler::get_polling_page(Register dest) {
if (SafepointMechanism::uses_thread_local_poll()) {
ldr(dest, Address(Rthread, Thread::polling_page_offset()));
} else {
mov_address(dest, os::get_polling_page());
}
}

void MacroAssembler::read_polling_page(Register dest, relocInfo::relocType rtype) {
get_polling_page(dest);
relocate(rtype);
ldr(dest, Address(dest));
}


#endif // COMPILER2
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1074,7 +1074,9 @@ class MacroAssembler: public Assembler {
void fast_unlock(Register obj, Register box, Register scratch, Register scratch2);
#endif


void safepoint_poll(Register tmp1, Label& slow_path);
void get_polling_page(Register dest);
void read_polling_page(Register dest, relocInfo::relocType rtype);
};


@@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,6 +34,7 @@
#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/safepointMechanism.hpp"
#include "runtime/vframeArray.hpp"
#include "utilities/align.hpp"
#include "vmreg_arm.inline.hpp"
@@ -1219,20 +1220,18 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
}

// Do a safepoint check while thread is in transition state
InlinedAddress safepoint_state(SafepointSynchronize::address_of_state());
Label call_safepoint_runtime, return_to_java;
__ mov(Rtemp, _thread_in_native_trans);
__ ldr_literal(R2, safepoint_state);
__ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));

// make sure the store is observed before reading the SafepointSynchronize state and further mem refs
__ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);

__ ldr_s32(R2, Address(R2));
__ safepoint_poll(R2, call_safepoint_runtime);
__ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
__ cmp(R2, SafepointSynchronize::_not_synchronized);
__ cond_cmp(R3, 0, eq);
__ cmp(R3, 0);
__ b(call_safepoint_runtime, ne);

__ bind(return_to_java);

// Perform thread state transition and reguard stack yellow pages if needed
@@ -1303,8 +1302,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
pop_result_registers(masm, ret_type);
__ b(return_to_java);

__ bind_literal(safepoint_state);

// Reguard stack pages. Save native results around a call to C runtime.
__ bind(reguard);
push_result_registers(masm, ret_type);
@@ -1806,15 +1803,29 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t
oop_maps->add_gc_map(pc_offset, map);
__ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call

// Check for pending exception
__ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
__ cmp(Rtemp, 0);

if (!cause_return) {
if (SafepointMechanism::uses_thread_local_poll()) {
// If our stashed return pc was modified by the runtime we avoid touching it
__ ldr(R3_tmp, Address(Rthread, JavaThread::saved_exception_pc_offset()));
__ ldr(R2_tmp, Address(SP, RegisterSaver::LR_offset * wordSize));
__ cmp(R2_tmp, R3_tmp);
// Adjust return pc forward to step over the safepoint poll instruction
__ add(R2_tmp, R2_tmp, 4, eq);
__ str(R2_tmp, Address(SP, RegisterSaver::LR_offset * wordSize), eq);
}

// Check for pending exception
__ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
__ cmp(Rtemp, 0);

RegisterSaver::restore_live_registers(masm, false);
__ pop(PC, eq);
__ pop(Rexception_pc);
} else {
// Check for pending exception
__ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
__ cmp(Rtemp, 0);

RegisterSaver::restore_live_registers(masm);
__ bx(LR, eq);
__ mov(Rexception_pc, LR);
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -959,8 +959,6 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
// Force this write out before the read below
__ membar(MacroAssembler::StoreLoad, Rtemp);

__ ldr_global_s32(Rtemp, SafepointSynchronize::address_of_state());

// Protect the return value in the interleaved code: save it to callee-save registers.
__ mov(Rsaved_result_lo, R0);
__ mov(Rsaved_result_hi, R1);
@@ -973,12 +971,16 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
#endif // __ABI_HARD__

{
__ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
__ cmp(Rtemp, SafepointSynchronize::_not_synchronized);
__ cond_cmp(R3, 0, eq);
Label call, skip_call;
__ safepoint_poll(Rtemp, call);
__ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
__ cmp(R3, 0);
__ b(skip_call, eq);
__ bind(call);
__ mov(R0, Rthread);
__ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none);
__ bind(skip_call);

__ mov(R0, Rthread, ne);
__ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none, ne);
#if R9_IS_SCRATCHED
__ restore_method();
#endif

0 comments on commit d65f966

Please sign in to comment.