Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
232 changes: 78 additions & 154 deletions src/hotspot/cpu/aarch64/aarch64.ad

Large diffs are not rendered by default.

28 changes: 14 additions & 14 deletions src/hotspot/cpu/aarch64/aarch64_vector.ad
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ source %{
PRegister Pg, const Address &adr);

// Predicated load/store, with optional ptrue to all elements of given predicate register.
static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
static void loadStoreA_predicated(C2_MacroAssembler* masm, bool is_store, FloatRegister reg,
PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
int opcode, Register base, int index, int size, int disp) {
sve_mem_insn_predicate insn;
Expand All @@ -119,7 +119,7 @@ source %{
ShouldNotReachHere();
}
int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
(masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
(masm->*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
} else {
assert(false, "unimplemented");
ShouldNotReachHere();
Expand Down Expand Up @@ -422,7 +422,7 @@ instruct loadV(vReg dst, vmemA mem) %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ false,
loadStoreA_predicated(masm, /* is_store */ false,
$dst$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -439,7 +439,7 @@ instruct storeV(vReg src, vmemA mem) %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ true,
loadStoreA_predicated(masm, /* is_store */ true,
$src$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -454,7 +454,7 @@ instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
format %{ "loadV_masked $dst, $pg, $mem" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ false, $dst$$FloatRegister,
loadStoreA_predicated(masm, /* is_store */ false, $dst$$FloatRegister,
$pg$$PRegister, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -467,7 +467,7 @@ instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
format %{ "storeV_masked $mem, $pg, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ true, $src$$FloatRegister,
loadStoreA_predicated(masm, /* is_store */ true, $src$$FloatRegister,
$pg$$PRegister, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand Down Expand Up @@ -4929,7 +4929,7 @@ instruct vloadmask_loadV(pReg dst, indirect mem, vReg tmp, rFlagsReg cr) %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, $tmp$$FloatRegister,
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
Expand All @@ -4950,7 +4950,7 @@ instruct vloadmask_loadV_masked(pReg dst, indirect mem, pRegGov pg,
// Load valid mask values which are boolean type, and extend them to the
// defined vector element type. Convert the vector to predicate.
BasicType bt = Matcher::vector_element_basic_type(this);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, $tmp$$FloatRegister,
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
$pg$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
Expand All @@ -4977,7 +4977,7 @@ instruct vloadmask_loadVMasked(pReg dst, vmemA mem, pRegGov pg, vReg tmp, rFlags
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, $tmp$$FloatRegister,
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
Expand Down Expand Up @@ -5005,7 +5005,7 @@ instruct vloadmask_loadVMasked_masked(pReg dst, vmemA mem, pRegGov pg1, pRegGov
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, $tmp$$FloatRegister,
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
$pg2$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
Expand All @@ -5030,7 +5030,7 @@ instruct storeV_vstoremask(indirect mem, pReg src, immI_gt_1 esize, vReg tmp) %{
assert(type2aelembytes(bt) == (int)$esize$$constant, "unsupported type");
Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, $tmp$$FloatRegister,
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -5052,7 +5052,7 @@ instruct storeV_vstoremask_masked(indirect mem, pReg src, immI_gt_1 esize,
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
__ sve_gen_mask_imm($pgtmp$$PRegister, bt, Matcher::vector_length(this, $src));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, $tmp$$FloatRegister,
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
$pgtmp$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -5078,7 +5078,7 @@ instruct storeVMasked_vstoremask(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esiz
assert(type2aelembytes(bt) == (int)$esize$$constant, "unsupported type.");
Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, $tmp$$FloatRegister,
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -5105,7 +5105,7 @@ instruct storeVMasked_vstoremask_masked(vmemA mem, pReg src, pRegGov pg, immI_gt
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
__ sve_gen_mask_imm($pgtmp$$PRegister, bt, Matcher::vector_length(this, $src));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, $tmp$$FloatRegister,
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
$pgtmp$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand Down
28 changes: 14 additions & 14 deletions src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ source %{
PRegister Pg, const Address &adr);

// Predicated load/store, with optional ptrue to all elements of given predicate register.
static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
static void loadStoreA_predicated(C2_MacroAssembler* masm, bool is_store, FloatRegister reg,
PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
int opcode, Register base, int index, int size, int disp) {
sve_mem_insn_predicate insn;
Expand All @@ -109,7 +109,7 @@ source %{
ShouldNotReachHere();
}
int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
(masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
(masm->*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
} else {
assert(false, "unimplemented");
ShouldNotReachHere();
Expand Down Expand Up @@ -361,7 +361,7 @@ instruct loadV(vReg dst, vmemA mem) %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ false,
loadStoreA_predicated(masm, /* is_store */ false,
$dst$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -378,7 +378,7 @@ instruct storeV(vReg src, vmemA mem) %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ true,
loadStoreA_predicated(masm, /* is_store */ true,
$src$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -393,7 +393,7 @@ instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
format %{ "loadV_masked $dst, $pg, $mem" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ false, $dst$$FloatRegister,
loadStoreA_predicated(masm, /* is_store */ false, $dst$$FloatRegister,
$pg$$PRegister, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -406,7 +406,7 @@ instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
format %{ "storeV_masked $mem, $pg, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), /* is_store */ true, $src$$FloatRegister,
loadStoreA_predicated(masm, /* is_store */ true, $src$$FloatRegister,
$pg$$PRegister, bt, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand Down Expand Up @@ -3321,7 +3321,7 @@ instruct vloadmask_loadV(pReg dst, indirect mem, vReg tmp, rFlagsReg cr) %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, $tmp$$FloatRegister,
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
Expand All @@ -3342,7 +3342,7 @@ instruct vloadmask_loadV_masked(pReg dst, indirect mem, pRegGov pg,
// Load valid mask values which are boolean type, and extend them to the
// defined vector element type. Convert the vector to predicate.
BasicType bt = Matcher::vector_element_basic_type(this);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, $tmp$$FloatRegister,
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
$pg$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
Expand All @@ -3369,7 +3369,7 @@ instruct vloadmask_loadVMasked(pReg dst, vmemA mem, pRegGov pg, vReg tmp, rFlags
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, $tmp$$FloatRegister,
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
Expand Down Expand Up @@ -3397,7 +3397,7 @@ instruct vloadmask_loadVMasked_masked(pReg dst, vmemA mem, pRegGov pg1, pRegGov
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, $tmp$$FloatRegister,
loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
$pg2$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
__ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
Expand All @@ -3422,7 +3422,7 @@ instruct storeV_vstoremask(indirect mem, pReg src, immI_gt_1 esize, vReg tmp) %{
assert(type2aelembytes(bt) == (int)$esize$$constant, "unsupported type");
Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, $tmp$$FloatRegister,
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -3444,7 +3444,7 @@ instruct storeV_vstoremask_masked(indirect mem, pReg src, immI_gt_1 esize,
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
__ sve_gen_mask_imm($pgtmp$$PRegister, bt, Matcher::vector_length(this, $src));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, $tmp$$FloatRegister,
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
$pgtmp$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -3470,7 +3470,7 @@ instruct storeVMasked_vstoremask(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esiz
assert(type2aelembytes(bt) == (int)$esize$$constant, "unsupported type.");
Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, $tmp$$FloatRegister,
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
ptrue, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand All @@ -3497,7 +3497,7 @@ instruct storeVMasked_vstoremask_masked(vmemA mem, pReg src, pRegGov pg, immI_gt
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
__ sve_gen_mask_imm($pgtmp$$PRegister, bt, Matcher::vector_length(this, $src));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, $tmp$$FloatRegister,
loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
$pgtmp$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
%}
Expand Down
9 changes: 3 additions & 6 deletions src/hotspot/cpu/aarch64/ad_encode.m4
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ define(choose, `loadStore($1, &MacroAssembler::$3, $2, $4,
%}')dnl
define(access, `
$3Register $1_reg = as_$3Register($$1$$reg);
$4choose(C2_MacroAssembler(&cbuf), $1_reg,$2,$mem->opcode(),
$4choose(masm, $1_reg,$2,$mem->opcode(),
as_Register($mem$$base),$mem$$index,$mem$$scale,$mem$$disp,$5)')dnl
define(load,`
// This encoding class is generated automatically from ad_encode.m4.
Expand Down Expand Up @@ -59,8 +59,7 @@ define(STORE0,`
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
enc_class aarch64_enc_$2`'0(memory$4 mem) %{
C2_MacroAssembler _masm(&cbuf);
choose(_masm,zr,$2,$mem->opcode(),
choose(masm,zr,$2,$mem->opcode(),
as_$3Register($mem$$base),$mem$$index,$mem$$scale,$mem$$disp,$4)')dnl
STORE(iRegI,strb,,,1)
STORE0(iRegI,strb,,1)
Expand All @@ -72,7 +71,6 @@ STORE(iRegL,str,,
`// we sometimes get asked to store the stack pointer into the
// current thread -- we cannot do that directly on AArch64
if (src_reg == r31_sp) {
C2_MacroAssembler _masm(&cbuf);
assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
__ mov(rscratch2, sp);
src_reg = rscratch2;
Expand All @@ -85,8 +83,7 @@ STORE(vRegD,strd,Float,,8)
// This encoding class is generated automatically from ad_encode.m4.
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
C2_MacroAssembler _masm(&cbuf);
__ membar(Assembler::StoreStore);
loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
%}
14 changes: 5 additions & 9 deletions src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,24 +34,20 @@

// ----------------------------------------------------------------------------

#define __ _masm.
address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
precond(cbuf.stubs()->start() != badAddress);
precond(cbuf.stubs()->end() != badAddress);
#define __ masm->
address CompiledDirectCall::emit_to_interp_stub(MacroAssembler *masm, address mark) {
precond(__ code()->stubs()->start() != badAddress);
precond(__ code()->stubs()->end() != badAddress);

// Stub is fixed up when the corresponding call is converted from
// calling compiled code to calling interpreted code.
// mov rmethod, 0
// jmp -4 # to self

if (mark == nullptr) {
mark = cbuf.insts_mark(); // Get mark within main instrs section.
mark = __ inst_mark(); // Get mark within main instrs section.
}

// Note that the code buffer's insts_mark is always relative to insts.
// That's why we must use the macroassembler to generate a stub.
MacroAssembler _masm(&cbuf);

address base = __ start_a_stub(to_interp_stub_size());
int offset = __ offset();
if (base == nullptr) {
Expand Down
Loading