Skip to content

Commit 974ff6b

Browse files
committed
8270947: AArch64: C1: use zero_words to initialize all objects
Backport-of: 6c68ce2d396c6fe02201daf2bdb8c164de807cc1
1 parent dbf694e commit 974ff6b

File tree

8 files changed

+28831
-165
lines changed

8 files changed

+28831
-165
lines changed

src/hotspot/cpu/aarch64/aarch64.ad

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15196,12 +15196,12 @@ instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlag
1519615196
ins_pipe(pipe_class_memory);
1519715197
%}
1519815198

15199-
instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15199+
instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
1520015200
%{
1520115201
predicate((uint64_t)n->in(2)->get_long()
1520215202
< (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
1520315203
match(Set dummy (ClearArray cnt base));
15204-
effect(USE_KILL base);
15204+
effect(TEMP temp, USE_KILL base, KILL cr);
1520515205

1520615206
ins_cost(4 * INSN_COST);
1520715207
format %{ "ClearArray $cnt, $base" %}

src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1115,8 +1115,8 @@ void LIRGenerator::do_NewInstance(NewInstance* x) {
11151115
CodeEmitInfo* info = state_for(x, x->state());
11161116
LIR_Opr reg = result_register_for(x->type());
11171117
new_instance(reg, x->klass(), x->is_unresolved(),
1118-
FrameMap::r2_oop_opr,
1119-
FrameMap::r5_oop_opr,
1118+
FrameMap::r10_oop_opr,
1119+
FrameMap::r11_oop_opr,
11201120
FrameMap::r4_oop_opr,
11211121
LIR_OprFact::illegalOpr,
11221122
FrameMap::r3_metadata_opr, info);
@@ -1131,8 +1131,8 @@ void LIRGenerator::do_NewTypeArray(NewTypeArray* x) {
11311131
length.load_item_force(FrameMap::r19_opr);
11321132

11331133
LIR_Opr reg = result_register_for(x->type());
1134-
LIR_Opr tmp1 = FrameMap::r2_oop_opr;
1135-
LIR_Opr tmp2 = FrameMap::r4_oop_opr;
1134+
LIR_Opr tmp1 = FrameMap::r10_oop_opr;
1135+
LIR_Opr tmp2 = FrameMap::r11_oop_opr;
11361136
LIR_Opr tmp3 = FrameMap::r5_oop_opr;
11371137
LIR_Opr tmp4 = reg;
11381138
LIR_Opr klass_reg = FrameMap::r3_metadata_opr;
@@ -1160,8 +1160,8 @@ void LIRGenerator::do_NewObjectArray(NewObjectArray* x) {
11601160
CodeEmitInfo* info = state_for(x, x->state());
11611161

11621162
LIR_Opr reg = result_register_for(x->type());
1163-
LIR_Opr tmp1 = FrameMap::r2_oop_opr;
1164-
LIR_Opr tmp2 = FrameMap::r4_oop_opr;
1163+
LIR_Opr tmp1 = FrameMap::r10_oop_opr;
1164+
LIR_Opr tmp2 = FrameMap::r11_oop_opr;
11651165
LIR_Opr tmp3 = FrameMap::r5_oop_opr;
11661166
LIR_Opr tmp4 = reg;
11671167
LIR_Opr klass_reg = FrameMap::r3_metadata_opr;

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

Lines changed: 18 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
* Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
3-
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
3+
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
44
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
*
66
* This code is free software; you can redistribute it and/or modify it
@@ -203,20 +203,24 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
203203
}
204204

205205
// preserves obj, destroys len_in_bytes
206-
void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1) {
206+
//
207+
// Scratch registers: t1 = r10, t2 = r11
208+
//
209+
void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1, Register t2) {
207210
assert(hdr_size_in_bytes >= 0, "header size must be positive or 0");
211+
assert(t1 == r10 && t2 == r11, "must be");
212+
208213
Label done;
209214

210215
// len_in_bytes is positive and ptr sized
211216
subs(len_in_bytes, len_in_bytes, hdr_size_in_bytes);
212217
br(Assembler::EQ, done);
213218

214-
// Preserve obj
215-
if (hdr_size_in_bytes)
216-
add(obj, obj, hdr_size_in_bytes);
217-
zero_memory(obj, len_in_bytes, t1);
218-
if (hdr_size_in_bytes)
219-
sub(obj, obj, hdr_size_in_bytes);
219+
// zero_words() takes ptr in r10 and count in words in r11
220+
mov(rscratch1, len_in_bytes);
221+
lea(t1, Address(obj, hdr_size_in_bytes));
222+
lsr(t2, rscratch1, LogBytesPerWord);
223+
zero_words(t1, t2);
220224

221225
bind(done);
222226
}
@@ -231,6 +235,7 @@ void C1_MacroAssembler::allocate_object(Register obj, Register t1, Register t2,
231235
initialize_object(obj, klass, noreg, object_size * HeapWordSize, t1, t2, UseTLAB);
232236
}
233237

238+
// Scratch registers: t1 = r10, t2 = r11
234239
void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, bool is_tlab_allocated) {
235240
assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0,
236241
"con_size_in_bytes is not multiple of alignment");
@@ -241,45 +246,13 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
241246
if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
242247
// clear rest of allocated space
243248
const Register index = t2;
244-
const int threshold = 16 * BytesPerWord; // approximate break even point for code size (see comments below)
245249
if (var_size_in_bytes != noreg) {
246250
mov(index, var_size_in_bytes);
247-
initialize_body(obj, index, hdr_size_in_bytes, t1);
248-
} else if (con_size_in_bytes <= threshold) {
249-
// use explicit null stores
250-
int i = hdr_size_in_bytes;
251-
if (i < con_size_in_bytes && (con_size_in_bytes % (2 * BytesPerWord))) {
252-
str(zr, Address(obj, i));
253-
i += BytesPerWord;
254-
}
255-
for (; i < con_size_in_bytes; i += 2 * BytesPerWord)
256-
stp(zr, zr, Address(obj, i));
251+
initialize_body(obj, index, hdr_size_in_bytes, t1, t2);
257252
} else if (con_size_in_bytes > hdr_size_in_bytes) {
258-
block_comment("zero memory");
259-
// use loop to null out the fields
260-
261-
int words = (con_size_in_bytes - hdr_size_in_bytes) / BytesPerWord;
262-
mov(index, words / 8);
263-
264-
const int unroll = 8; // Number of str(zr) instructions we'll unroll
265-
int remainder = words % unroll;
266-
lea(rscratch1, Address(obj, hdr_size_in_bytes + remainder * BytesPerWord));
267-
268-
Label entry_point, loop;
269-
b(entry_point);
270-
271-
bind(loop);
272-
sub(index, index, 1);
273-
for (int i = -unroll; i < 0; i++) {
274-
if (-i == remainder)
275-
bind(entry_point);
276-
str(zr, Address(rscratch1, i * wordSize));
277-
}
278-
if (remainder == 0)
279-
bind(entry_point);
280-
add(rscratch1, rscratch1, unroll * wordSize);
281-
cbnz(index, loop);
282-
253+
con_size_in_bytes -= hdr_size_in_bytes;
254+
lea(t1, Address(obj, hdr_size_in_bytes));
255+
zero_words(t1, con_size_in_bytes / BytesPerWord);
283256
}
284257
}
285258

@@ -314,8 +287,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
314287
initialize_header(obj, klass, len, t1, t2);
315288

316289
// clear rest of allocated space
317-
const Register len_zero = len;
318-
initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
290+
initialize_body(obj, arr_size, header_size * BytesPerWord, t1, t2);
319291

320292
membar(StoreStore);
321293

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
2-
* Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
3-
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
2+
* Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
3+
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
44
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
*
66
* This code is free software; you can redistribute it and/or modify it
@@ -48,7 +48,7 @@ using MacroAssembler::null_check;
4848
);
4949

5050
void initialize_header(Register obj, Register klass, Register len, Register t1, Register t2);
51-
void initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1);
51+
void initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1, Register t2);
5252

5353
void float_cmp(bool is_float, int unordered_result,
5454
FloatRegister f0, FloatRegister f1,

src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
* Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
3-
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
3+
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
44
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
*
66
* This code is free software; you can redistribute it and/or modify it
@@ -656,9 +656,9 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
656656
if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
657657
!UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
658658
Label slow_path;
659-
Register obj_size = r2;
660-
Register t1 = r19;
661-
Register t2 = r4;
659+
Register obj_size = r19;
660+
Register t1 = r10;
661+
Register t2 = r11;
662662
assert_different_registers(klass, obj, obj_size, t1, t2);
663663

664664
__ stp(r19, zr, Address(__ pre(sp, -2 * wordSize)));
@@ -769,9 +769,9 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
769769
// allocations.
770770
// Otherwise, just go to the slow path.
771771
if (!UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
772-
Register arr_size = r4;
773-
Register t1 = r2;
774-
Register t2 = r5;
772+
Register arr_size = r5;
773+
Register t1 = r10;
774+
Register t2 = r11;
775775
Label slow_path;
776776
assert_different_registers(length, klass, obj, arr_size, t1, t2);
777777

@@ -801,7 +801,7 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
801801
__ andr(t1, t1, Klass::_lh_header_size_mask);
802802
__ sub(arr_size, arr_size, t1); // body length
803803
__ add(t1, t1, obj); // body start
804-
__ initialize_body(t1, arr_size, 0, t2);
804+
__ initialize_body(t1, arr_size, 0, t1, t2);
805805
__ membar(Assembler::StoreStore);
806806
__ verify_oop(obj);
807807

0 commit comments

Comments
 (0)