Skip to content

Commit 0d91f0a

Browse files
author
Jatin Bhateja
committed
8252848: Optimize small primitive arrayCopy operations through partial inlining using AVX-512 masked instructions
Reviewed-by: neliasso, kvn
1 parent 66943fe commit 0d91f0a

25 files changed

+470
-45
lines changed

‎src/hotspot/cpu/x86/assembler_x86.cpp‎

Lines changed: 6 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2706,34 +2706,18 @@ void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, bool mer
27062706
emit_operand(dst, src);
27072707
}
27082708

2709-
void Assembler::evmovdqu(XMMRegister dst, KRegister mask, Address src, int vector_len, int type) {
2710-
assert(VM_Version::supports_avx512vlbw(), "");
2711-
assert(type == T_BYTE || type == T_SHORT || type == T_CHAR || type == T_INT || type == T_LONG, "");
2712-
InstructionMark im(this);
2713-
bool wide = type == T_SHORT || type == T_CHAR || type == T_LONG;
2714-
int prefix = (type == T_BYTE || type == T_SHORT || type == T_CHAR) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2715-
InstructionAttr attributes(vector_len, /* vex_w */ wide, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2716-
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2717-
attributes.set_embedded_opmask_register_specifier(mask);
2718-
attributes.set_is_evex_instruction();
2719-
vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2720-
emit_int8(0x6F);
2721-
emit_operand(dst, src);
2722-
}
2723-
2724-
void Assembler::evmovdqu(Address dst, KRegister mask, XMMRegister src, int vector_len, int type) {
2709+
void Assembler::evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
27252710
assert(VM_Version::supports_avx512vlbw(), "");
27262711
assert(src != xnoreg, "sanity");
2727-
assert(type == T_BYTE || type == T_SHORT || type == T_CHAR || type == T_INT || type == T_LONG, "");
27282712
InstructionMark im(this);
2729-
bool wide = type == T_SHORT || type == T_CHAR || type == T_LONG;
2730-
int prefix = (type == T_BYTE || type == T_SHORT || type == T_CHAR) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2731-
InstructionAttr attributes(vector_len, /* vex_w */ wide, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2713+
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
27322714
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2733-
attributes.reset_is_clear_context();
27342715
attributes.set_embedded_opmask_register_specifier(mask);
27352716
attributes.set_is_evex_instruction();
2736-
vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2717+
if (merge) {
2718+
attributes.reset_is_clear_context();
2719+
}
2720+
vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
27372721
emit_int8(0x7F);
27382722
emit_operand(src, dst);
27392723
}

‎src/hotspot/cpu/x86/assembler_x86.hpp‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1549,6 +1549,7 @@ class Assembler : public AbstractAssembler {
15491549
void evmovdqub(XMMRegister dst, Address src, bool merge, int vector_len);
15501550
void evmovdqub(XMMRegister dst, XMMRegister src, bool merge, int vector_len);
15511551
void evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
1552+
void evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
15521553
void evmovdquw(Address dst, XMMRegister src, bool merge, int vector_len);
15531554
void evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
15541555
void evmovdquw(XMMRegister dst, Address src, bool merge, int vector_len);
@@ -1566,10 +1567,6 @@ class Assembler : public AbstractAssembler {
15661567
void evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
15671568
void evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
15681569

1569-
// Generic move instructions.
1570-
void evmovdqu(Address dst, KRegister mask, XMMRegister src, int vector_len, int type);
1571-
void evmovdqu(XMMRegister dst, KRegister mask, Address src, int vector_len, int type);
1572-
15731570
// Move lower 64bit to high 64bit in 128bit register
15741571
void movlhps(XMMRegister dst, XMMRegister src);
15751572

‎src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1891,6 +1891,20 @@ void C2_MacroAssembler::reduce8L(int opcode, Register dst, Register src1, XMMReg
18911891
reduce_operation_256(T_LONG, opcode, vtmp2, vtmp2, src2);
18921892
reduce4L(opcode, dst, src1, vtmp2, vtmp1, vtmp2);
18931893
}
1894+
1895+
void C2_MacroAssembler::genmask(Register dst, Register len, Register temp) {
1896+
if (ArrayCopyPartialInlineSize <= 32) {
1897+
mov64(dst, 1);
1898+
shlxq(dst, dst, len);
1899+
decq(dst);
1900+
} else {
1901+
mov64(dst, -1);
1902+
movq(temp, len);
1903+
negptr(temp);
1904+
addptr(temp, 64);
1905+
shrxq(dst, dst, temp);
1906+
}
1907+
}
18941908
#endif // _LP64
18951909

18961910
void C2_MacroAssembler::reduce2F(int opcode, XMMRegister dst, XMMRegister src, XMMRegister vtmp) {
@@ -1937,6 +1951,15 @@ void C2_MacroAssembler::reduce8D(int opcode, XMMRegister dst, XMMRegister src, X
19371951
reduce4D(opcode, dst, vtmp1, vtmp1, vtmp2);
19381952
}
19391953

1954+
void C2_MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, int vector_len) {
1955+
MacroAssembler::evmovdqu(type, kmask, dst, src, vector_len);
1956+
}
1957+
1958+
void C2_MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, int vector_len) {
1959+
MacroAssembler::evmovdqu(type, kmask, dst, src, vector_len);
1960+
}
1961+
1962+
19401963
void C2_MacroAssembler::reduceFloatMinMax(int opcode, int vlen, bool is_dst_valid,
19411964
XMMRegister dst, XMMRegister src,
19421965
XMMRegister tmp, XMMRegister atmp, XMMRegister btmp,

‎src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,9 @@
120120
void evgather(BasicType typ, XMMRegister dst, KRegister mask, Register base, XMMRegister idx, int vector_len);
121121
void evscatter(BasicType typ, Register base, XMMRegister idx, KRegister mask, XMMRegister src, int vector_len);
122122

123+
void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, int vector_len);
124+
void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, int vector_len);
125+
123126
// extract
124127
void extract(BasicType typ, Register dst, XMMRegister src, int idx);
125128
XMMRegister get_lane(BasicType typ, XMMRegister dst, XMMRegister src, int elemindex);
@@ -139,6 +142,7 @@
139142
void reduceI(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
140143
#ifdef _LP64
141144
void reduceL(int opcode, int vlen, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2);
145+
void genmask(Register dst, Register len, Register temp);
142146
#endif // _LP64
143147

144148
// dst = reduce(op, src2) using vtmp as temps

‎src/hotspot/cpu/x86/macroAssembler_x86.cpp‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8000,6 +8000,56 @@ void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len
80008000
bind(done);
80018001
}
80028002

8003+
8004+
void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, int vector_len) {
8005+
switch(type) {
8006+
case T_BYTE:
8007+
case T_BOOLEAN:
8008+
evmovdqub(dst, kmask, src, false, vector_len);
8009+
break;
8010+
case T_CHAR:
8011+
case T_SHORT:
8012+
evmovdquw(dst, kmask, src, false, vector_len);
8013+
break;
8014+
case T_INT:
8015+
case T_FLOAT:
8016+
evmovdqul(dst, kmask, src, false, vector_len);
8017+
break;
8018+
case T_LONG:
8019+
case T_DOUBLE:
8020+
evmovdquq(dst, kmask, src, false, vector_len);
8021+
break;
8022+
default:
8023+
fatal("Unexpected type argument %s", type2name(type));
8024+
break;
8025+
}
8026+
}
8027+
8028+
void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, int vector_len) {
8029+
switch(type) {
8030+
case T_BYTE:
8031+
case T_BOOLEAN:
8032+
evmovdqub(dst, kmask, src, true, vector_len);
8033+
break;
8034+
case T_CHAR:
8035+
case T_SHORT:
8036+
evmovdquw(dst, kmask, src, true, vector_len);
8037+
break;
8038+
case T_INT:
8039+
case T_FLOAT:
8040+
evmovdqul(dst, kmask, src, true, vector_len);
8041+
break;
8042+
case T_LONG:
8043+
case T_DOUBLE:
8044+
evmovdquq(dst, kmask, src, true, vector_len);
8045+
break;
8046+
default:
8047+
fatal("Unexpected type argument %s", type2name(type));
8048+
break;
8049+
}
8050+
}
8051+
8052+
80038053
#ifdef _LP64
80048054
void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
80058055
Label done;

‎src/hotspot/cpu/x86/macroAssembler_x86.hpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1094,10 +1094,14 @@ class MacroAssembler: public Assembler {
10941094
void vmovdqu(XMMRegister dst, AddressLiteral src, Register scratch_reg = rscratch1);
10951095

10961096
// AVX512 Unaligned
1097+
void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, int vector_len);
1098+
void evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, int vector_len);
1099+
10971100
void evmovdqub(Address dst, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqub(dst, src, merge, vector_len); }
10981101
void evmovdqub(XMMRegister dst, Address src, bool merge, int vector_len) { Assembler::evmovdqub(dst, src, merge, vector_len); }
10991102
void evmovdqub(XMMRegister dst, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqub(dst, src, merge, vector_len); }
11001103
void evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
1104+
void evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
11011105
void evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register scratch_reg);
11021106

11031107
void evmovdquw(Address dst, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdquw(dst, src, merge, vector_len); }

‎src/hotspot/cpu/x86/macroAssembler_x86_arrayCopy_avx3.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -200,8 +200,8 @@ void MacroAssembler::copy64_masked_avx(Register dst, Register src, XMMRegister x
200200
mov64(temp, -1);
201201
shrxq(temp, temp, length);
202202
kmovql(mask, temp);
203-
evmovdqu(xmm, mask, Address(src, index, scale, offset), Assembler::AVX_512bit, type[shift]);
204-
evmovdqu(Address(dst, index, scale, offset), mask, xmm, Assembler::AVX_512bit, type[shift]);
203+
evmovdqu(type[shift], mask, xmm, Address(src, index, scale, offset), Assembler::AVX_512bit);
204+
evmovdqu(type[shift], mask, Address(dst, index, scale, offset), xmm, Assembler::AVX_512bit);
205205
}
206206
}
207207

@@ -216,8 +216,8 @@ void MacroAssembler::copy32_masked_avx(Register dst, Register src, XMMRegister x
216216
shlxq(temp, temp, length);
217217
decq(temp);
218218
kmovql(mask, temp);
219-
evmovdqu(xmm, mask, Address(src, index, scale, offset), Assembler::AVX_256bit, type[shift]);
220-
evmovdqu(Address(dst, index, scale, offset), mask, xmm, Assembler::AVX_256bit, type[shift]);
219+
evmovdqu(type[shift], mask, xmm, Address(src, index, scale, offset), Assembler::AVX_256bit);
220+
evmovdqu(type[shift], mask, Address(dst, index, scale, offset), xmm, Assembler::AVX_256bit);
221221
}
222222

223223

‎src/hotspot/cpu/x86/vm_version_x86.cpp‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1362,6 +1362,7 @@ void VM_Version::get_processor_features() {
13621362
MaxLoopPad = 11;
13631363
}
13641364
#endif // COMPILER2
1365+
13651366
if (FLAG_IS_DEFAULT(UseXMMForArrayCopy)) {
13661367
UseXMMForArrayCopy = true; // use SSE2 movq on new Intel cpus
13671368
}
@@ -1399,6 +1400,38 @@ void VM_Version::get_processor_features() {
13991400
if (FLAG_IS_DEFAULT(AllocatePrefetchInstr) && supports_3dnow_prefetch()) {
14001401
FLAG_SET_DEFAULT(AllocatePrefetchInstr, 3);
14011402
}
1403+
#ifdef COMPILER2
1404+
if (UseAVX > 2) {
1405+
if (FLAG_IS_DEFAULT(ArrayCopyPartialInlineSize) ||
1406+
(!FLAG_IS_DEFAULT(ArrayCopyPartialInlineSize) &&
1407+
ArrayCopyPartialInlineSize != 0 &&
1408+
ArrayCopyPartialInlineSize != 32 &&
1409+
ArrayCopyPartialInlineSize != 16 &&
1410+
ArrayCopyPartialInlineSize != 64)) {
1411+
int inline_size = 0;
1412+
if (MaxVectorSize >= 64 && AVX3Threshold == 0) {
1413+
inline_size = 64;
1414+
} else if (MaxVectorSize >= 32) {
1415+
inline_size = 32;
1416+
} else if (MaxVectorSize >= 16) {
1417+
inline_size = 16;
1418+
}
1419+
if(!FLAG_IS_DEFAULT(ArrayCopyPartialInlineSize)) {
1420+
warning("Setting ArrayCopyPartialInlineSize as %d", inline_size);
1421+
}
1422+
ArrayCopyPartialInlineSize = inline_size;
1423+
}
1424+
1425+
if (ArrayCopyPartialInlineSize > MaxVectorSize) {
1426+
ArrayCopyPartialInlineSize = MaxVectorSize >= 16 ? MaxVectorSize : 0;
1427+
if (ArrayCopyPartialInlineSize) {
1428+
warning("Setting ArrayCopyPartialInlineSize as MaxVectorSize" INTX_FORMAT ")", MaxVectorSize);
1429+
} else {
1430+
warning("Setting ArrayCopyPartialInlineSize as " INTX_FORMAT, ArrayCopyPartialInlineSize);
1431+
}
1432+
}
1433+
}
1434+
#endif
14021435
}
14031436

14041437
#ifdef _LP64

‎src/hotspot/cpu/x86/x86.ad‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1521,6 +1521,13 @@ const bool Matcher::match_rule_supported(int opcode) {
15211521
return false;
15221522
}
15231523
break;
1524+
case Op_VectorMaskGen:
1525+
case Op_LoadVectorMasked:
1526+
case Op_StoreVectorMasked:
1527+
if (UseAVX < 3) {
1528+
return false;
1529+
}
1530+
break;
15241531
#ifndef _LP64
15251532
case Op_AddReductionVF:
15261533
case Op_AddReductionVD:
@@ -1594,6 +1601,16 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
15941601
return false;
15951602
}
15961603
break;
1604+
case Op_VectorMaskGen:
1605+
case Op_LoadVectorMasked:
1606+
case Op_StoreVectorMasked:
1607+
if (!VM_Version::supports_avx512bw()) {
1608+
return false;
1609+
}
1610+
if ((size_in_bits != 512) && !VM_Version::supports_avx512vl()) {
1611+
return false;
1612+
}
1613+
break;
15971614
case Op_CMoveVD:
15981615
if (vlen != 4) {
15991616
return false; // implementation limitation (only vcmov4D_reg is present)
@@ -7894,3 +7911,50 @@ instruct vprorate(vec dst, vec src, vec shift) %{
78947911
ins_pipe( pipe_slow );
78957912
%}
78967913

7914+
#ifdef _LP64
7915+
// ---------------------------------- Masked Block Copy ------------------------------------
7916+
7917+
instruct vmasked_load64(vec dst, memory mem, rRegL mask) %{
7918+
match(Set dst (LoadVectorMasked mem mask));
7919+
format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
7920+
ins_encode %{
7921+
BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
7922+
int vector_len = vector_length_encoding(this);
7923+
__ kmovql(k2, $mask$$Register);
7924+
__ evmovdqu(elmType, k2, $dst$$XMMRegister, $mem$$Address, vector_len);
7925+
%}
7926+
ins_pipe( pipe_slow );
7927+
%}
7928+
7929+
instruct vmask_gen(rRegL dst, rRegL len, rRegL tempLen) %{
7930+
match(Set dst (VectorMaskGen len));
7931+
effect(TEMP_DEF dst, TEMP tempLen);
7932+
format %{ "vector_mask_gen $len \t! vector mask generator" %}
7933+
ins_encode %{
7934+
__ genmask($dst$$Register, $len$$Register, $tempLen$$Register);
7935+
%}
7936+
ins_pipe( pipe_slow );
7937+
%}
7938+
7939+
instruct vmask_gen_imm(rRegL dst, immL len) %{
7940+
match(Set dst (VectorMaskGen len));
7941+
format %{ "vector_mask_gen $len \t! vector mask generator" %}
7942+
ins_encode %{
7943+
__ mov64($dst$$Register, (0xFFFFFFFFFFFFFFFFUL >> (64 -$len$$constant)));
7944+
%}
7945+
ins_pipe( pipe_slow );
7946+
%}
7947+
7948+
instruct vmasked_store64(memory mem, vec src, rRegL mask) %{
7949+
match(Set mem (StoreVectorMasked mem (Binary src mask)));
7950+
format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
7951+
ins_encode %{
7952+
const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
7953+
BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
7954+
int vector_len = vector_length_encoding(src_node);
7955+
__ kmovql(k2, $mask$$Register);
7956+
__ evmovdqu(elmType, k2, $mem$$Address, $src$$XMMRegister, vector_len);
7957+
%}
7958+
ins_pipe( pipe_slow );
7959+
%}
7960+
#endif // _LP64

‎src/hotspot/share/adlc/forms.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,7 @@ Form::DataType Form::is_load_from_memory(const char *opType) const {
269269
if( strcmp(opType,"LoadS")==0 ) return Form::idealS;
270270
if( strcmp(opType,"LoadVector")==0 ) return Form::idealV;
271271
if( strcmp(opType,"LoadVectorGather")==0 ) return Form::idealV;
272+
if( strcmp(opType,"LoadVectorMasked")==0 ) return Form::idealV;
272273
assert( strcmp(opType,"Load") != 0, "Must type Loads" );
273274
return Form::none;
274275
}
@@ -286,6 +287,7 @@ Form::DataType Form::is_store_to_memory(const char *opType) const {
286287
if( strcmp(opType,"StoreNKlass")==0) return Form::idealNKlass;
287288
if( strcmp(opType,"StoreVector")==0 ) return Form::idealV;
288289
if( strcmp(opType,"StoreVectorScatter")==0 ) return Form::idealV;
290+
if( strcmp(opType,"StoreVectorMasked")==0 ) return Form::idealV;
289291
assert( strcmp(opType,"Store") != 0, "Must type Stores" );
290292
return Form::none;
291293
}

0 commit comments

Comments
 (0)