Skip to content
Permalink
Browse files
8274912: Eagerly generate native invokers
Reviewed-by: ngasson, mcimadamore
  • Loading branch information
JornVernee committed Oct 15, 2021
1 parent e5e373e commit 1aa1c29c16cba9f4389c32fd55b5e4653e6a5fff
Showing with 1,876 additions and 1,384 deletions.
  1. +102 −5 src/hotspot/cpu/aarch64/foreign_globals_aarch64.cpp
  2. +201 −10 src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
  3. +20 −2 src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
  4. +20 −5 src/hotspot/cpu/aarch64/methodHandles_aarch64.cpp
  5. +3 −0 src/hotspot/cpu/aarch64/methodHandles_aarch64.hpp
  6. +15 −453 src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
  7. +255 −0 src/hotspot/cpu/aarch64/universalNativeInvoker_aarch64.cpp
  8. +0 −13 src/hotspot/cpu/aarch64/vmreg_aarch64.cpp
  9. +5 −0 src/hotspot/cpu/arm/foreign_globals_arm.cpp
  10. +0 −10 src/hotspot/cpu/arm/sharedRuntime_arm.cpp
  11. +14 −0 src/hotspot/cpu/arm/universalNativeInvoker_arm.cpp
  12. +0 −5 src/hotspot/cpu/arm/vmreg_arm.cpp
  13. +5 −0 src/hotspot/cpu/ppc/foreign_globals_ppc.cpp
  14. +10 −4 src/hotspot/cpu/ppc/methodHandles_ppc.cpp
  15. +3 −0 src/hotspot/cpu/ppc/methodHandles_ppc.hpp
  16. +4 −11 src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
  17. +14 −0 src/hotspot/cpu/ppc/universalNativeInvoker_ppc.cpp
  18. +0 −5 src/hotspot/cpu/ppc/vmreg_ppc.cpp
  19. +5 −0 src/hotspot/cpu/s390/foreign_globals_s390.cpp
  20. +11 −5 src/hotspot/cpu/s390/methodHandles_s390.cpp
  21. +3 −0 src/hotspot/cpu/s390/methodHandles_s390.hpp
  22. +4 −11 src/hotspot/cpu/s390/sharedRuntime_s390.cpp
  23. +14 −0 src/hotspot/cpu/s390/universalNativeInvoker_s390.cpp
  24. +0 −5 src/hotspot/cpu/s390/vmreg_s390.cpp
  25. +101 −10 src/hotspot/cpu/x86/foreign_globals_x86.cpp
  26. +2 −0 src/hotspot/cpu/x86/foreign_globals_x86.hpp
  27. +4 −3 src/hotspot/cpu/x86/macroAssembler_x86.cpp
  28. +21 −7 src/hotspot/cpu/x86/methodHandles_x86.cpp
  29. +3 −0 src/hotspot/cpu/x86/methodHandles_x86.hpp
  30. +0 −10 src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
  31. +6 −267 src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
  32. +14 −0 src/hotspot/cpu/x86/universalNativeInvoker_x86_32.cpp
  33. +266 −0 src/hotspot/cpu/x86/universalNativeInvoker_x86_64.cpp
  34. +49 −330 src/hotspot/cpu/x86/universalUpcallHandler_x86_64.cpp
  35. +0 −15 src/hotspot/cpu/x86/vmreg_x86.cpp
  36. +5 −0 src/hotspot/cpu/zero/foreign_globals_zero.cpp
  37. +14 −0 src/hotspot/cpu/zero/universalNativeInvoker_zero.cpp
  38. +0 −5 src/hotspot/cpu/zero/vmreg_zero.cpp
  39. +2 −4 src/hotspot/share/ci/ciEnv.cpp
  40. +1 −2 src/hotspot/share/ci/ciEnv.hpp
  41. +7 −1 src/hotspot/share/classfile/javaClasses.cpp
  42. +3 −0 src/hotspot/share/classfile/javaClasses.hpp
  43. +1 −3 src/hotspot/share/code/codeCache.cpp
  44. +5 −33 src/hotspot/share/code/nmethod.cpp
  45. +3 −11 src/hotspot/share/code/nmethod.hpp
  46. +0 −2 src/hotspot/share/code/vmreg.hpp
  47. +1 −1 src/hotspot/share/jvmci/jvmciRuntime.cpp
  48. +50 −0 src/hotspot/share/oops/oopCast.inline.hpp
  49. +13 −5 src/hotspot/share/opto/callGenerator.cpp
  50. +5 −2 src/hotspot/share/opto/callnode.cpp
  51. +3 −6 src/hotspot/share/opto/callnode.hpp
  52. +0 −6 src/hotspot/share/opto/compile.cpp
  53. +0 −6 src/hotspot/share/opto/compile.hpp
  54. +13 −27 src/hotspot/share/opto/graphKit.cpp
  55. +1 −2 src/hotspot/share/opto/output.cpp
  56. +284 −6 src/hotspot/share/prims/foreign_globals.cpp
  57. +82 −5 src/hotspot/share/prims/foreign_globals.hpp
  58. +6 −17 src/hotspot/share/prims/foreign_globals.inline.hpp
  59. +102 −2 src/hotspot/share/prims/nativeEntryPoint.cpp
  60. +6 −1 src/hotspot/share/prims/universalNativeInvoker.cpp
  61. +9 −0 src/hotspot/share/prims/universalNativeInvoker.hpp
  62. +3 −0 src/hotspot/share/runtime/globals.hpp
  63. +0 −13 src/hotspot/share/runtime/sharedRuntime.hpp
  64. +8 −0 src/hotspot/share/utilities/growableArray.hpp
  65. +2 −2 src/java.base/share/classes/java/lang/invoke/MethodHandleImpl.java
  66. +11 −24 src/java.base/share/classes/java/lang/invoke/NativeMethodHandle.java
  67. +1 −2 src/java.base/share/classes/jdk/internal/access/JavaLangInvokeAccess.java
  68. +27 −3 src/java.base/share/classes/jdk/internal/invoke/NativeEntryPoint.java
  69. +10 −7 src/jdk.incubator.foreign/share/classes/jdk/internal/foreign/abi/ProgrammableInvoker.java
  70. +24 −0 test/jdk/java/foreign/TestUpcall.java
@@ -23,11 +23,15 @@
*/

#include "precompiled.hpp"
#include "code/vmreg.inline.hpp"
#include "runtime/jniHandles.hpp"
#include "runtime/jniHandles.inline.hpp"
#include "oops/typeArrayOop.inline.hpp"
#include "oops/oopCast.inline.hpp"
#include "opto/matcher.hpp"
#include "prims/foreign_globals.hpp"
#include "prims/foreign_globals.inline.hpp"
#include "utilities/formatBuffer.hpp"

bool ABIDescriptor::is_volatile_reg(Register reg) const {
return _integer_argument_registers.contains(reg)
@@ -47,15 +51,15 @@ const ABIDescriptor ForeignGlobals::parse_abi_descriptor_impl(jobject jabi) cons
ABIDescriptor abi;
const Register (*to_Register)(int) = as_Register;

objArrayOop inputStorage = cast<objArrayOop>(abi_oop->obj_field(ABI.inputStorage_offset));
objArrayOop inputStorage = oop_cast<objArrayOop>(abi_oop->obj_field(ABI.inputStorage_offset));
loadArray(inputStorage, INTEGER_TYPE, abi._integer_argument_registers, to_Register);
loadArray(inputStorage, VECTOR_TYPE, abi._vector_argument_registers, as_FloatRegister);

objArrayOop outputStorage = cast<objArrayOop>(abi_oop->obj_field(ABI.outputStorage_offset));
objArrayOop outputStorage = oop_cast<objArrayOop>(abi_oop->obj_field(ABI.outputStorage_offset));
loadArray(outputStorage, INTEGER_TYPE, abi._integer_return_registers, to_Register);
loadArray(outputStorage, VECTOR_TYPE, abi._vector_return_registers, as_FloatRegister);

objArrayOop volatileStorage = cast<objArrayOop>(abi_oop->obj_field(ABI.volatileStorage_offset));
objArrayOop volatileStorage = oop_cast<objArrayOop>(abi_oop->obj_field(ABI.volatileStorage_offset));
loadArray(volatileStorage, INTEGER_TYPE, abi._integer_additional_volatile_registers, to_Register);
loadArray(volatileStorage, VECTOR_TYPE, abi._vector_additional_volatile_registers, as_FloatRegister);

@@ -73,11 +77,11 @@ const BufferLayout ForeignGlobals::parse_buffer_layout_impl(jobject jlayout) con
layout.stack_args = layout_oop->long_field(BL.stack_args_offset);
layout.arguments_next_pc = layout_oop->long_field(BL.arguments_next_pc_offset);

typeArrayOop input_offsets = cast<typeArrayOop>(layout_oop->obj_field(BL.input_type_offsets_offset));
typeArrayOop input_offsets = oop_cast<typeArrayOop>(layout_oop->obj_field(BL.input_type_offsets_offset));
layout.arguments_integer = (size_t) input_offsets->long_at(INTEGER_TYPE);
layout.arguments_vector = (size_t) input_offsets->long_at(VECTOR_TYPE);

typeArrayOop output_offsets = cast<typeArrayOop>(layout_oop->obj_field(BL.output_type_offsets_offset));
typeArrayOop output_offsets = oop_cast<typeArrayOop>(layout_oop->obj_field(BL.output_type_offsets_offset));
layout.returns_integer = (size_t) output_offsets->long_at(INTEGER_TYPE);
layout.returns_vector = (size_t) output_offsets->long_at(VECTOR_TYPE);

@@ -90,3 +94,96 @@ const CallRegs ForeignGlobals::parse_call_regs_impl(jobject jconv) const {
ShouldNotCallThis();
return {};
}

enum class RegType {
INTEGER = 0,
VECTOR = 1,
STACK = 3
};

VMReg ForeignGlobals::vmstorage_to_vmreg(int type, int index) {
switch(static_cast<RegType>(type)) {
case RegType::INTEGER: return ::as_Register(index)->as_VMReg();
case RegType::VECTOR: return ::as_FloatRegister(index)->as_VMReg();
case RegType::STACK: return VMRegImpl::stack2reg(index LP64_ONLY(* 2));
}
return VMRegImpl::Bad();
}

int RegSpiller::pd_reg_size(VMReg reg) {
if (reg->is_Register()) {
return 8;
} else if (reg->is_FloatRegister()) {
bool use_sve = Matcher::supports_scalable_vector();
if (use_sve) {
return Matcher::scalable_vector_reg_size(T_BYTE);
}
return 16;
}
return 0; // stack and BAD
}

void RegSpiller::pd_store_reg(MacroAssembler* masm, int offset, VMReg reg) {
if (reg->is_Register()) {
masm->spill(reg->as_Register(), true, offset);
} else if (reg->is_FloatRegister()) {
bool use_sve = Matcher::supports_scalable_vector();
if (use_sve) {
masm->spill_sve_vector(reg->as_FloatRegister(), offset, Matcher::scalable_vector_reg_size(T_BYTE));
} else {
masm->spill(reg->as_FloatRegister(), masm->Q, offset);
}
} else {
// stack and BAD
}
}

void RegSpiller::pd_load_reg(MacroAssembler* masm, int offset, VMReg reg) {
if (reg->is_Register()) {
masm->unspill(reg->as_Register(), true, offset);
} else if (reg->is_FloatRegister()) {
bool use_sve = Matcher::supports_scalable_vector();
if (use_sve) {
masm->unspill_sve_vector(reg->as_FloatRegister(), offset, Matcher::scalable_vector_reg_size(T_BYTE));
} else {
masm->unspill(reg->as_FloatRegister(), masm->Q, offset);
}
} else {
// stack and BAD
}
}

void ArgumentShuffle::pd_generate(MacroAssembler* masm) const {
for (int i = 0; i < _moves.length(); i++) {
Move move = _moves.at(i);
BasicType arg_bt = move.bt;
VMRegPair from_vmreg = move.from;
VMRegPair to_vmreg = move.to;

masm->block_comment(err_msg("bt=%s", null_safe_string(type2name(arg_bt))));
switch (arg_bt) {
case T_BOOLEAN:
case T_BYTE:
case T_SHORT:
case T_CHAR:
case T_INT:
masm->move32_64(from_vmreg, to_vmreg);
break;

case T_FLOAT:
masm->float_move(from_vmreg, to_vmreg);
break;

case T_DOUBLE:
masm->double_move(from_vmreg, to_vmreg);
break;

case T_LONG :
masm->long_move(from_vmreg, to_vmreg);
break;

default:
fatal("found in upcall args: %s", type2name(arg_bt));
}
}
}
@@ -29,6 +29,7 @@
#include "jvm.h"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "gc/shared/cardTableBarrierSet.hpp"
@@ -292,20 +293,30 @@ address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) {
return address(((uint64_t)insn_addr + (offset << 2)));
}

void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod) {
void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod, Register tmp) {
if (acquire) {
lea(rscratch1, Address(rthread, JavaThread::polling_word_offset()));
ldar(rscratch1, rscratch1);
lea(tmp, Address(rthread, JavaThread::polling_word_offset()));
ldar(tmp, tmp);
} else {
ldr(rscratch1, Address(rthread, JavaThread::polling_word_offset()));
ldr(tmp, Address(rthread, JavaThread::polling_word_offset()));
}
if (at_return) {
// Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
// we may safely use the sp instead to perform the stack watermark check.
cmp(in_nmethod ? sp : rfp, rscratch1);
cmp(in_nmethod ? sp : rfp, tmp);
br(Assembler::HI, slow_path);
} else {
tbnz(rscratch1, log2i_exact(SafepointMechanism::poll_bit()), slow_path);
tbnz(tmp, log2i_exact(SafepointMechanism::poll_bit()), slow_path);
}
}

void MacroAssembler::rt_call(address dest, Register tmp) {
CodeBlob *cb = CodeCache::find_blob(dest);
if (cb) {
far_call(RuntimeAddress(dest));
} else {
lea(tmp, RuntimeAddress(dest));
blr(tmp);
}
}

@@ -5105,13 +5116,13 @@ void MacroAssembler::cache_wbsync(bool is_pre) {
}
}

void MacroAssembler::verify_sve_vector_length() {
void MacroAssembler::verify_sve_vector_length(Register tmp) {
// Make sure that native code does not change SVE vector length.
if (!UseSVE) return;
Label verify_ok;
movw(rscratch1, zr);
sve_inc(rscratch1, B);
subsw(zr, rscratch1, VM_Version::get_initial_sve_vector_length());
movw(tmp, zr);
sve_inc(tmp, B);
subsw(zr, tmp, VM_Version::get_initial_sve_vector_length());
br(EQ, verify_ok);
stop("Error: SVE vector length has changed since jvm startup");
bind(verify_ok);
@@ -5154,3 +5165,183 @@ void MacroAssembler::verify_cross_modify_fence_not_required() {
}
}
#endif

// The java_calling_convention describes stack locations as ideal slots on
// a frame with no abi restrictions. Since we must observe abi restrictions
// (like the placement of the register window) the slots must be biased by
// the following value.
static int reg2offset_in(VMReg r) {
// Account for saved rfp and lr
// This should really be in_preserve_stack_slots
return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
}

static int reg2offset_out(VMReg r) {
return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
}

// On 64 bit we will store integer like items to the stack as
// 64 bits items (Aarch64 abi) even though java would only store
// 32bits for a parameter. On 32bit it will simply be 32 bits
// So this routine will do 32->32 on 32bit and 32->64 on 64bit
void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst) {
if (src.first()->is_stack()) {
if (dst.first()->is_stack()) {
// stack to stack
ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
str(rscratch1, Address(sp, reg2offset_out(dst.first())));
} else {
// stack to reg
ldrsw(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first())));
}
} else if (dst.first()->is_stack()) {
// reg to stack
// Do we really have to sign extend???
// __ movslq(src.first()->as_Register(), src.first()->as_Register());
str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
} else {
if (dst.first() != src.first()) {
sxtw(dst.first()->as_Register(), src.first()->as_Register());
}
}
}

// An oop arg. Must pass a handle not the oop itself
void MacroAssembler::object_move(
OopMap* map,
int oop_handle_offset,
int framesize_in_slots,
VMRegPair src,
VMRegPair dst,
bool is_receiver,
int* receiver_offset) {

// must pass a handle. First figure out the location we use as a handle

Register rHandle = dst.first()->is_stack() ? rscratch2 : dst.first()->as_Register();

// See if oop is NULL if it is we need no handle

if (src.first()->is_stack()) {

// Oop is already on the stack as an argument
int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
if (is_receiver) {
*receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
}

ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
lea(rHandle, Address(rfp, reg2offset_in(src.first())));
// conditionally move a NULL
cmp(rscratch1, zr);
csel(rHandle, zr, rHandle, Assembler::EQ);
} else {

// Oop is in an a register we must store it to the space we reserve
// on the stack for oop_handles and pass a handle if oop is non-NULL

const Register rOop = src.first()->as_Register();
int oop_slot;
if (rOop == j_rarg0)
oop_slot = 0;
else if (rOop == j_rarg1)
oop_slot = 1;
else if (rOop == j_rarg2)
oop_slot = 2;
else if (rOop == j_rarg3)
oop_slot = 3;
else if (rOop == j_rarg4)
oop_slot = 4;
else if (rOop == j_rarg5)
oop_slot = 5;
else if (rOop == j_rarg6)
oop_slot = 6;
else {
assert(rOop == j_rarg7, "wrong register");
oop_slot = 7;
}

oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
int offset = oop_slot*VMRegImpl::stack_slot_size;

map->set_oop(VMRegImpl::stack2reg(oop_slot));
// Store oop in handle area, may be NULL
str(rOop, Address(sp, offset));
if (is_receiver) {
*receiver_offset = offset;
}

cmp(rOop, zr);
lea(rHandle, Address(sp, offset));
// conditionally move a NULL
csel(rHandle, zr, rHandle, Assembler::EQ);
}

// If arg is on the stack then place it otherwise it is already in correct reg.
if (dst.first()->is_stack()) {
str(rHandle, Address(sp, reg2offset_out(dst.first())));
}
}

// A float arg may have to do float reg int reg conversion
void MacroAssembler::float_move(VMRegPair src, VMRegPair dst) {
assert(src.first()->is_stack() && dst.first()->is_stack() ||
src.first()->is_reg() && dst.first()->is_reg(), "Unexpected error");
if (src.first()->is_stack()) {
if (dst.first()->is_stack()) {
ldrw(rscratch1, Address(rfp, reg2offset_in(src.first())));
strw(rscratch1, Address(sp, reg2offset_out(dst.first())));
} else {
ShouldNotReachHere();
}
} else if (src.first() != dst.first()) {
if (src.is_single_phys_reg() && dst.is_single_phys_reg())
fmovs(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
else
ShouldNotReachHere();
}
}

// A long move
void MacroAssembler::long_move(VMRegPair src, VMRegPair dst) {
if (src.first()->is_stack()) {
if (dst.first()->is_stack()) {
// stack to stack
ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
str(rscratch1, Address(sp, reg2offset_out(dst.first())));
} else {
// stack to reg
ldr(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first())));
}
} else if (dst.first()->is_stack()) {
// reg to stack
// Do we really have to sign extend???
// __ movslq(src.first()->as_Register(), src.first()->as_Register());
str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
} else {
if (dst.first() != src.first()) {
mov(dst.first()->as_Register(), src.first()->as_Register());
}
}
}


// A double move
void MacroAssembler::double_move(VMRegPair src, VMRegPair dst) {
assert(src.first()->is_stack() && dst.first()->is_stack() ||
src.first()->is_reg() && dst.first()->is_reg(), "Unexpected error");
if (src.first()->is_stack()) {
if (dst.first()->is_stack()) {
ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
str(rscratch1, Address(sp, reg2offset_out(dst.first())));
} else {
ShouldNotReachHere();
}
} else if (src.first() != dst.first()) {
if (src.is_single_phys_reg() && dst.is_single_phys_reg())
fmovd(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
else
ShouldNotReachHere();
}
}

0 comments on commit 1aa1c29

Please sign in to comment.