Skip to content

Commit

Permalink
8290706: Remove the support for inline contiguous allocations
Browse files Browse the repository at this point in the history
Reviewed-by: eosterlund, aph, rrich, fyang, thartmann
  • Loading branch information
shipilev committed Jul 26, 2022
1 parent 7318b22 commit 8159a1a
Show file tree
Hide file tree
Showing 45 changed files with 48 additions and 1,006 deletions.
2 changes: 1 addition & 1 deletion src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i
if (UseTLAB) {
tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
} else {
eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
b(slow_case);
}
}

Expand Down
95 changes: 0 additions & 95 deletions src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -656,56 +656,6 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
__ set_info("fast new_instance init check", dont_gc_arguments);
}

// If TLAB is disabled, see if there is support for inlining contiguous
// allocations.
// Otherwise, just go to the slow path.
if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
!UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
Label slow_path;
Register obj_size = r19;
Register t1 = r10;
Register t2 = r11;
assert_different_registers(klass, obj, obj_size, t1, t2);

__ stp(r19, zr, Address(__ pre(sp, -2 * wordSize)));

if (id == fast_new_instance_init_check_id) {
// make sure the klass is initialized
__ ldrb(rscratch1, Address(klass, InstanceKlass::init_state_offset()));
__ cmpw(rscratch1, InstanceKlass::fully_initialized);
__ br(Assembler::NE, slow_path);
}

#ifdef ASSERT
// assert object can be fast path allocated
{
Label ok, not_ok;
__ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
__ cmp(obj_size, (u1)0);
__ br(Assembler::LE, not_ok); // make sure it's an instance (LH > 0)
__ tstw(obj_size, Klass::_lh_instance_slow_path_bit);
__ br(Assembler::EQ, ok);
__ bind(not_ok);
__ stop("assert(can be fast path allocated)");
__ should_not_reach_here();
__ bind(ok);
}
#endif // ASSERT

// get the instance size (size is positive so movl is fine for 64bit)
__ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));

__ eden_allocate(obj, obj_size, 0, t1, slow_path);

__ initialize_object(obj, klass, obj_size, 0, t1, t2, /* is_tlab_allocated */ false);
__ verify_oop(obj);
__ ldp(r19, zr, Address(__ post(sp, 2 * wordSize)));
__ ret(lr);

__ bind(slow_path);
__ ldp(r19, zr, Address(__ post(sp, 2 * wordSize)));
}

__ enter();
OopMap* map = save_live_registers(sasm);
int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
Expand Down Expand Up @@ -771,51 +721,6 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
}
#endif // ASSERT

// If TLAB is disabled, see if there is support for inlining contiguous
// allocations.
// Otherwise, just go to the slow path.
if (!UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
Register arr_size = r5;
Register t1 = r10;
Register t2 = r11;
Label slow_path;
assert_different_registers(length, klass, obj, arr_size, t1, t2);

// check that array length is small enough for fast path.
__ mov(rscratch1, C1_MacroAssembler::max_array_allocation_length);
__ cmpw(length, rscratch1);
__ br(Assembler::HI, slow_path);

// get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
// since size is positive ldrw does right thing on 64bit
__ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
// since size is positive movw does right thing on 64bit
__ movw(arr_size, length);
__ lslvw(arr_size, length, t1);
__ ubfx(t1, t1, Klass::_lh_header_size_shift,
exact_log2(Klass::_lh_header_size_mask + 1));
__ add(arr_size, arr_size, t1);
__ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
__ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);

__ eden_allocate(obj, arr_size, 0, t1, slow_path); // preserves arr_size

__ initialize_header(obj, klass, length, t1, t2);
__ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
__ andr(t1, t1, Klass::_lh_header_size_mask);
__ sub(arr_size, arr_size, t1); // body length
__ add(t1, t1, obj); // body start
__ initialize_body(t1, arr_size, 0, t1, t2);
__ membar(Assembler::StoreStore);
__ verify_oop(obj);

__ ret(lr);

__ bind(slow_path);
}

__ enter();
OopMap* map = save_live_registers(sasm);
int call_offset;
Expand Down
57 changes: 0 additions & 57 deletions src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -160,63 +160,6 @@ void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
// verify_tlab();
}

// Defines obj, preserves var_size_in_bytes
void BarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register obj,
Register var_size_in_bytes,
int con_size_in_bytes,
Register t1,
Label& slow_case) {
assert_different_registers(obj, var_size_in_bytes, t1);
if (!Universe::heap()->supports_inline_contig_alloc()) {
__ b(slow_case);
} else {
Register end = t1;
Register heap_end = rscratch2;
Label retry;
__ bind(retry);
{
uint64_t offset;
__ adrp(rscratch1, ExternalAddress((address) Universe::heap()->end_addr()), offset);
__ ldr(heap_end, Address(rscratch1, offset));
}

ExternalAddress heap_top((address) Universe::heap()->top_addr());

// Get the current top of the heap
{
uint64_t offset;
__ adrp(rscratch1, heap_top, offset);
// Use add() here after ARDP, rather than lea().
// lea() does not generate anything if its offset is zero.
// However, relocs expect to find either an ADD or a load/store
// insn after an ADRP. add() always generates an ADD insn, even
// for add(Rn, Rn, 0).
__ add(rscratch1, rscratch1, offset);
__ ldaxr(obj, rscratch1);
}

// Adjust it my the size of our new object
if (var_size_in_bytes == noreg) {
__ lea(end, Address(obj, con_size_in_bytes));
} else {
__ lea(end, Address(obj, var_size_in_bytes));
}

// if end < obj then we wrapped around high memory
__ cmp(end, obj);
__ br(Assembler::LO, slow_case);

__ cmp(end, heap_end);
__ br(Assembler::HI, slow_case);

// If heap_top hasn't been changed by some other thread, update it.
__ stlxr(rscratch2, end, rscratch1);
__ cbnzw(rscratch2, retry);

incr_allocated_bytes(masm, var_size_in_bytes, con_size_in_bytes, t1);
}
}

void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm,
Register var_size_in_bytes,
int con_size_in_bytes,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,6 @@ class BarrierSetAssembler: public CHeapObj<mtGC> {
Label& slow_case // continuation point if fast allocation fails
);

void eden_allocate(MacroAssembler* masm,
Register obj, // result: pointer to object after successful allocation
Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
int con_size_in_bytes, // object size in bytes if known at compile time
Register t1, // temp register
Label& slow_case // continuation point if fast allocation fails
);
virtual void barrier_stubs_init() {}

virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::stw_instruction_and_data_patch; }
Expand Down
10 changes: 0 additions & 10 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4518,16 +4518,6 @@ void MacroAssembler::tlab_allocate(Register obj,
bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
}

// Defines obj, preserves var_size_in_bytes
void MacroAssembler::eden_allocate(Register obj,
Register var_size_in_bytes,
int con_size_in_bytes,
Register t1,
Label& slow_case) {
BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->eden_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
}

void MacroAssembler::verify_tlab() {
#ifdef ASSERT
if (UseTLAB && VerifyOops) {
Expand Down
7 changes: 0 additions & 7 deletions src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -909,13 +909,6 @@ class MacroAssembler: public Assembler {
void java_round_float(Register dst, FloatRegister src, FloatRegister ftmp);

// allocation
void eden_allocate(
Register obj, // result: pointer to object after successful allocation
Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
int con_size_in_bytes, // object size in bytes if known at compile time
Register t1, // temp register
Label& slow_case // continuation point if fast allocation fails
);
void tlab_allocate(
Register obj, // result: pointer to object after successful allocation
Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
Expand Down
23 changes: 0 additions & 23 deletions src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3468,7 +3468,6 @@ void TemplateTable::_new() {
Label slow_case;
Label done;
Label initialize_header;
Label initialize_object; // including clearing the fields

__ get_cpool_and_tags(r4, r0);
// Make sure the class we're about to instantiate has been resolved.
Expand Down Expand Up @@ -3501,43 +3500,21 @@ void TemplateTable::_new() {
// If TLAB is enabled:
// Try to allocate in the TLAB.
// If fails, go to the slow path.
// Else If inline contiguous allocations are enabled:
// Try to allocate in eden.
// If fails due to heap end, go to slow path.
//
// If TLAB is enabled OR inline contiguous is enabled:
// Initialize the allocation.
// Exit.
//
// Go to slow path.
const bool allow_shared_alloc =
Universe::heap()->supports_inline_contig_alloc();

if (UseTLAB) {
__ tlab_allocate(r0, r3, 0, noreg, r1, slow_case);

if (ZeroTLAB) {
// the fields have been already cleared
__ b(initialize_header);
} else {
// initialize both the header and fields
__ b(initialize_object);
}
} else {
// Allocation in the shared Eden, if allowed.
//
// r3: instance size in bytes
if (allow_shared_alloc) {
__ eden_allocate(r0, r3, 0, r10, slow_case);
}
}

// If UseTLAB or allow_shared_alloc are true, the object is created above and
// there is an initialize need. Otherwise, skip and go to the slow path.
if (UseTLAB || allow_shared_alloc) {
// The object is initialized before the header. If the object size is
// zero, go directly to the header initialization.
__ bind(initialize_object);
__ sub(r3, r3, sizeof(oopDesc));
__ cbz(r3, initialize_header);

Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ void C1_MacroAssembler::try_allocate(Register obj, Register obj_end, Register tm
if (UseTLAB) {
tlab_allocate(obj, obj_end, tmp1, size_expression, slow_case);
} else {
eden_allocate(obj, obj_end, tmp1, tmp2, size_expression, slow_case);
b(slow_case);
}
}

Expand Down
75 changes: 0 additions & 75 deletions src/hotspot/cpu/arm/c1_Runtime1_arm.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -419,40 +419,6 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
const Register result = R0;
const Register klass = R1;

// If TLAB is disabled, see if there is support for inlining contiguous
// allocations.
// Otherwise, just go to the slow path.
if (!UseTLAB && Universe::heap()->supports_inline_contig_alloc() && id != new_instance_id) {
Label slow_case, slow_case_no_pop;

// Make sure the class is fully initialized
if (id == fast_new_instance_init_check_id) {
__ ldrb(result, Address(klass, InstanceKlass::init_state_offset()));
__ cmp(result, InstanceKlass::fully_initialized);
__ b(slow_case_no_pop, ne);
}

// Free some temporary registers
const Register obj_size = R4;
const Register tmp1 = R5;
const Register tmp2 = LR;
const Register obj_end = Rtemp;

__ raw_push(R4, R5, LR);

__ ldr_u32(obj_size, Address(klass, Klass::layout_helper_offset()));
__ eden_allocate(result, obj_end, tmp1, tmp2, obj_size, slow_case); // initializes result and obj_end
__ initialize_object(result, obj_end, klass, noreg /* len */, tmp1, tmp2,
instanceOopDesc::header_size() * HeapWordSize, -1,
/* is_tlab_allocated */ false);
__ raw_pop_and_ret(R4, R5);

__ bind(slow_case);
__ raw_pop(R4, R5, LR);

__ bind(slow_case_no_pop);
}

OopMap* map = save_live_registers(sasm);
int call_offset = __ call_RT(result, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
oop_maps = new OopMapSet();
Expand Down Expand Up @@ -489,47 +455,6 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
const Register klass = R1;
const Register length = R2;

// If TLAB is disabled, see if there is support for inlining contiguous
// allocations.
// Otherwise, just go to the slow path.
if (!UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
Label slow_case, slow_case_no_pop;

__ cmp_32(length, C1_MacroAssembler::max_array_allocation_length);
__ b(slow_case_no_pop, hs);

// Free some temporary registers
const Register arr_size = R4;
const Register tmp1 = R5;
const Register tmp2 = LR;
const Register tmp3 = Rtemp;
const Register obj_end = tmp3;

__ raw_push(R4, R5, LR);

// Get the allocation size: round_up((length << (layout_helper & 0xff)) + header_size)
__ ldr_u32(tmp1, Address(klass, Klass::layout_helper_offset()));
__ mov(arr_size, MinObjAlignmentInBytesMask);
__ and_32(tmp2, tmp1, (unsigned int)(Klass::_lh_header_size_mask << Klass::_lh_header_size_shift));

__ add(arr_size, arr_size, AsmOperand(length, lsl, tmp1));

__ add(arr_size, arr_size, AsmOperand(tmp2, lsr, Klass::_lh_header_size_shift));
__ align_reg(arr_size, arr_size, MinObjAlignmentInBytes);

// eden_allocate destroys tmp2, so reload header_size after allocation
// eden_allocate initializes result and obj_end
__ eden_allocate(result, obj_end, tmp1, tmp2, arr_size, slow_case);
__ ldrb(tmp2, Address(klass, in_bytes(Klass::layout_helper_offset()) +
Klass::_lh_header_size_shift / BitsPerByte));
__ initialize_object(result, obj_end, klass, length, tmp1, tmp2, tmp2, -1, /* is_tlab_allocated */ false);
__ raw_pop_and_ret(R4, R5);

__ bind(slow_case);
__ raw_pop(R4, R5, LR);
__ bind(slow_case_no_pop);
}

OopMap* map = save_live_registers(sasm);
int call_offset;
if (id == new_type_array_id) {
Expand Down

1 comment on commit 8159a1a

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.