Skip to content

Commit

Permalink
8317352: [Lilliput] Sync with upstreaming PRs
Browse files Browse the repository at this point in the history
Reviewed-by: shade
  • Loading branch information
rkennke committed Oct 11, 2023
1 parent 0a73377 commit 39f08e7
Show file tree
Hide file tree
Showing 144 changed files with 1,909 additions and 1,691 deletions.
14 changes: 2 additions & 12 deletions src/hotspot/cpu/aarch64/aarch64.ad
Original file line number Diff line number Diff line change
Expand Up @@ -7337,7 +7337,7 @@ instruct loadNKlass(iRegNNoSp dst, memory4 mem)
ins_pipe(iload_reg_mem);
%}

instruct loadNKlassLilliput(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
%{
match(Set dst (LoadNKlass mem));
effect(KILL cr);
Expand All @@ -7348,17 +7348,7 @@ instruct loadNKlassLilliput(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
ins_encode %{
assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
assert($mem$$index$$Register == noreg, "expect no index");
Register dst = $dst$$Register;
Register obj = $mem$$base$$Register;
C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
Compile::current()->output()->add_stub(stub);
__ ldr(dst, Address(obj, oopDesc::mark_offset_in_bytes()));
// NOTE: We can't use tbnz here, because the target is sometimes too far away
// and cannot be encoded.
__ tst(dst, markWord::monitor_value);
__ br(Assembler::NE, stub->entry());
__ bind(stub->continuation());
__ lsr(dst, dst, markWord::klass_shift);
__ load_nklass_compact($dst$$Register, $mem$$base$$Register);
%}
ins_pipe(pipe_slow);
%}
Expand Down
12 changes: 1 addition & 11 deletions src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2289,8 +2289,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {

Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());

// test for null
if (flags & LIR_OpArrayCopy::src_null_check) {
Expand Down Expand Up @@ -2351,15 +2349,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// We don't know the array types are compatible
if (basic_type != T_OBJECT) {
// Simple test for basic type arrays
if (UseCompressedClassPointers) {
__ load_nklass(tmp, src);
__ load_nklass(rscratch1, dst);
__ cmpw(tmp, rscratch1);
} else {
__ ldr(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
__ ldr(rscratch1, Address(dst, oopDesc::klass_offset_in_bytes()));
__ cmp(tmp, rscratch1);
}
__ cmp_klass(src, dst, tmp, rscratch1);
__ br(Assembler::NE, *stub->entry());
} else {
// For object arrays, if src is a sub class of dst then we can
Expand Down
19 changes: 12 additions & 7 deletions src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -217,12 +217,6 @@ void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int
subs(len_in_bytes, len_in_bytes, hdr_size_in_bytes);
br(Assembler::EQ, done);

// Zero first 4 bytes, if start offset is not word aligned.
if (!is_aligned(hdr_size_in_bytes, BytesPerWord)) {
strw(zr, Address(obj, hdr_size_in_bytes));
hdr_size_in_bytes += BytesPerInt;
}

// zero_words() takes ptr in r10 and count in words in r11
mov(rscratch1, len_in_bytes);
lea(t1, Address(obj, hdr_size_in_bytes));
Expand Down Expand Up @@ -303,8 +297,19 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,

initialize_header(obj, klass, len, t1, t2);

// Clear leading 4 bytes, if necessary.
// TODO: This could perhaps go into initialize_body() and also clear the leading 4 bytes
// for non-array objects, thereby replacing the klass-gap clearing code in initialize_header().
int base_offset = base_offset_in_bytes;
if (!is_aligned(base_offset, BytesPerWord)) {
assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
strw(zr, Address(obj, base_offset));
base_offset += BytesPerInt;
}
assert(is_aligned(base_offset, BytesPerWord), "must be word-aligned");

// clear rest of allocated space
initialize_body(obj, arr_size, base_offset_in_bytes, t1, t2);
initialize_body(obj, arr_size, base_offset, t1, t2);
if (Compilation::current()->bailed_out()) {
return;
}
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,7 @@ static OopMap* save_live_registers(StubAssembler* sasm,
} else {
__ add(sp, sp, -32 * wordSize);
}

return generate_oop_map(sasm, save_fpu_registers);
}

Expand Down
12 changes: 12 additions & 0 deletions src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2058,3 +2058,15 @@ bool C2_MacroAssembler::in_scratch_emit_size() {
}
return MacroAssembler::in_scratch_emit_size();
}

void C2_MacroAssembler::load_nklass_compact(Register dst, Register obj) {
C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
Compile::current()->output()->add_stub(stub);
ldr(dst, Address(obj, oopDesc::mark_offset_in_bytes()));
// NOTE: We can't use tbnz here, because the target is sometimes too far away
// and cannot be encoded.
tst(dst, markWord::monitor_value);
br(Assembler::NE, stub->entry());
bind(stub->continuation());
lsr(dst, dst, markWord::klass_shift);
}
2 changes: 2 additions & 0 deletions src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -172,4 +172,6 @@
void vector_signum_sve(FloatRegister dst, FloatRegister src, FloatRegister zero,
FloatRegister one, FloatRegister vtmp, PRegister pgtmp, SIMD_RegVariant T);

void load_nklass_compact(Register dst, Register obj);

#endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP
36 changes: 24 additions & 12 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4314,12 +4314,7 @@ void MacroAssembler::load_method_holder(Register holder, Register method) {
// Loads the obj's Klass* into dst.
// Preserves all registers (incl src, rscratch1 and rscratch2).
void MacroAssembler::load_nklass(Register dst, Register src) {
assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");

if (!UseCompactObjectHeaders) {
ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
return;
}
assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");

Label fast;

Expand All @@ -4336,12 +4331,11 @@ void MacroAssembler::load_nklass(Register dst, Register src) {
}

void MacroAssembler::load_klass(Register dst, Register src) {
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_nklass(dst, src);
} else {
ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
if (UseCompactObjectHeaders) {
load_nklass(dst, src);
decode_klass_not_null(dst);
} else if (UseCompressedClassPointers) {
ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst);
} else {
ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
Expand Down Expand Up @@ -4402,9 +4396,26 @@ void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp)
cmp(trial_klass, tmp);
}

void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) {
if (UseCompactObjectHeaders) {
load_nklass(tmp1, src);
load_nklass(tmp2, dst);
cmpw(tmp1, tmp2);
} else if (UseCompressedClassPointers) {
ldrw(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
ldrw(tmp2, Address(dst, oopDesc::klass_offset_in_bytes()));
cmpw(tmp1, tmp2);
} else {
ldr(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
ldr(tmp2, Address(dst, oopDesc::klass_offset_in_bytes()));
cmp(tmp1, tmp2);
}
}

void MacroAssembler::store_klass(Register dst, Register src) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
assert(!UseCompactObjectHeaders, "not with compact headers");
if (UseCompressedClassPointers) {
encode_klass_not_null(src);
strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
Expand All @@ -4414,6 +4425,7 @@ void MacroAssembler::store_klass(Register dst, Register src) {
}

void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "not with compact headers");
if (UseCompressedClassPointers) {
// Store to klass gap in destination
strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -854,6 +854,7 @@ class MacroAssembler: public Assembler {
void load_klass(Register dst, Register src);
void store_klass(Register dst, Register src);
void cmp_klass(Register oop, Register trial_klass, Register tmp);
void cmp_klass(Register src, Register dst, Register tmp1, Register tmp2);

void resolve_weak_handle(Register result, Register tmp1, Register tmp2);
void resolve_oop_handle(Register result, Register tmp1, Register tmp2);
Expand Down
18 changes: 12 additions & 6 deletions src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3607,16 +3607,21 @@ void TemplateTable::_new() {

// The object is initialized before the header. If the object size is
// zero, go directly to the header initialization.
__ sub(r3, r3, oopDesc::base_offset_in_bytes());
if (UseCompactObjectHeaders) {
assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
__ sub(r3, r3, oopDesc::base_offset_in_bytes());
} else {
__ sub(r3, r3, sizeof(oopDesc));
}
__ cbz(r3, initialize_header);

// Initialize object fields
{
__ add(r2, r0, oopDesc::base_offset_in_bytes());
if (!is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong)) {
__ strw(zr, Address(__ post(r2, BytesPerInt)));
__ sub(r3, r3, BytesPerInt);
__ cbz(r3, initialize_header);
if (UseCompactObjectHeaders) {
assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
__ add(r2, r0, oopDesc::base_offset_in_bytes());
} else {
__ add(r2, r0, sizeof(oopDesc));
}
Label loop;
__ bind(loop);
Expand All @@ -3633,6 +3638,7 @@ void TemplateTable::_new() {
} else {
__ mov(rscratch1, (intptr_t)markWord::prototype().value());
__ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
__ store_klass_gap(r0, zr); // zero klass gap for compressed oops
__ store_klass(r0, r4); // store klass last
}
{
Expand Down
8 changes: 4 additions & 4 deletions src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3195,8 +3195,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {

Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());

// length and pos's are all sign extended at this point on 64bit

Expand Down Expand Up @@ -3321,6 +3319,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
store_parameter(src, 4);

#ifndef _LP64
Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
__ movptr(tmp, dst_klass_addr);
__ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset()));
__ push(tmp);
Expand Down Expand Up @@ -3537,13 +3536,14 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
// Fast-path: shift and decode Klass*.
__ shrq(result, markWord::klass_shift);
__ decode_klass_not_null(result, tmp);
} else
if (UseCompressedClassPointers) {
} else if (UseCompressedClassPointers) {
__ movl(result, Address(obj, oopDesc::klass_offset_in_bytes()));
__ decode_klass_not_null(result, rscratch1);
} else
#endif
{
__ movptr(result, Address(obj, oopDesc::klass_offset_in_bytes()));
}
}

void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
Expand Down
56 changes: 34 additions & 22 deletions src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -164,21 +164,20 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i

void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
assert_different_registers(obj, klass, len, t1, t2);
#ifdef _LP64
if (UseCompactObjectHeaders) {
movptr(t1, Address(klass, Klass::prototype_header_offset()));
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1);
} else {
} else if (UseCompressedClassPointers) { // Take care not to kill klass
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
#ifdef _LP64
if (UseCompressedClassPointers) { // Take care not to kill klass
movptr(t1, klass);
encode_klass_not_null(t1, rscratch1);
movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
} else
movptr(t1, klass);
encode_klass_not_null(t1, rscratch1);
movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
} else
#endif
{
movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
}
{
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
}
if (len->is_valid()) {
movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len);
Expand Down Expand Up @@ -218,39 +217,40 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0,
"con_size_in_bytes is not multiple of alignment");
const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;

if (UseCompactObjectHeaders) {
assert(hdr_size_in_bytes == 8, "check object headers size");
}
initialize_header(obj, klass, noreg, t1, t2);

if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
// clear rest of allocated space
const Register t1_zero = t1;
const Register index = t2;
const int threshold = 6 * BytesPerWord; // approximate break even point for code size (see comments below)
int hdr_size_aligned = align_up(hdr_size_in_bytes, BytesPerWord); // klass gap is already cleared by init_header().
if (var_size_in_bytes != noreg) {
mov(index, var_size_in_bytes);
initialize_body(obj, index, hdr_size_aligned, t1_zero);
initialize_body(obj, index, hdr_size_in_bytes, t1_zero);
} else if (con_size_in_bytes <= threshold) {
// use explicit null stores
// code size = 2 + 3*n bytes (n = number of fields to clear)
xorptr(t1_zero, t1_zero); // use t1_zero reg to clear memory (shorter code)
for (int i = hdr_size_aligned; i < con_size_in_bytes; i += BytesPerWord)
for (int i = hdr_size_in_bytes; i < con_size_in_bytes; i += BytesPerWord)
movptr(Address(obj, i), t1_zero);
} else if (con_size_in_bytes > hdr_size_aligned) {
} else if (con_size_in_bytes > hdr_size_in_bytes) {
// use loop to null out the fields
// code size = 16 bytes for even n (n = number of fields to clear)
// initialize last object field first if odd number of fields
xorptr(t1_zero, t1_zero); // use t1_zero reg to clear memory (shorter code)
movptr(index, (con_size_in_bytes - hdr_size_aligned) >> 3);
movptr(index, (con_size_in_bytes - hdr_size_in_bytes) >> 3);
// initialize last object field if constant size is odd
if (((con_size_in_bytes - hdr_size_aligned) & 4) != 0)
if (((con_size_in_bytes - hdr_size_in_bytes) & 4) != 0)
movptr(Address(obj, con_size_in_bytes - (1*BytesPerWord)), t1_zero);
// initialize remaining object fields: rdx is a multiple of 2
{ Label loop;
bind(loop);
movptr(Address(obj, index, Address::times_8, hdr_size_aligned - (1*BytesPerWord)),
movptr(Address(obj, index, Address::times_8, hdr_size_in_bytes - (1*BytesPerWord)),
t1_zero);
NOT_LP64(movptr(Address(obj, index, Address::times_8, hdr_size_aligned - (2*BytesPerWord)),
NOT_LP64(movptr(Address(obj, index, Address::times_8, hdr_size_in_bytes - (2*BytesPerWord)),
t1_zero);)
decrement(index);
jcc(Assembler::notZero, loop);
Expand Down Expand Up @@ -287,9 +287,22 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,

initialize_header(obj, klass, len, t1, t2);

// Clear leading 4 bytes, if necessary.
// TODO: This could perhaps go into initialize_body() and also clear the leading 4 bytes
// for non-array objects, thereby replacing the klass-gap clearing code in initialize_header().
int base_offset = base_offset_in_bytes;
#ifdef _LP64
if (!is_aligned(base_offset, BytesPerWord)) {
assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
movl(Address(obj, base_offset), 0);
base_offset += BytesPerInt;
}
#endif
assert(is_aligned(base_offset, BytesPerWord), "must be word aligned");

// clear rest of allocated space
const Register len_zero = len;
initialize_body(obj, arr_size, base_offset_in_bytes, len_zero);
initialize_body(obj, arr_size, base_offset, len_zero);

if (CURRENT_ENV->dtrace_alloc_probes()) {
assert(obj == rax, "must be");
Expand All @@ -304,8 +317,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
verify_oop(receiver);
// explicit null check not needed since load from [klass_offset] causes a trap
// check against inline cache
assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
// check against inline cache. This is checked in Universe::genesis().
int start_offset = offset();

if (UseCompressedClassPointers) {
Expand Down

0 comments