Skip to content

8338023: Support two vector selectFrom API #20508

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 20 commits into from
Closed

8338023: Support two vector selectFrom API #20508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
45bfe6c
8338023: Support two vector selectFrom API
Aug 8, 2024
82c0b0a
Adding Benchmark
Aug 8, 2024
055fb22
Review suggestions incorporated.
Aug 19, 2024
e24632c
Pass explicit wrap argument to selectFrom API with default value set …
Aug 21, 2024
d7ad688
Defaulting to index wrapping scheme.
Aug 23, 2024
6cb1a46
Removing redundant checkIndex routine
Aug 23, 2024
408a869
Review comments resolutions.
Aug 27, 2024
8d71f17
Adding descriptive comments
Aug 29, 2024
d3ee310
Review resolutions.
Sep 6, 2024
1c00f41
Documentation suggestions from Paul.
Sep 13, 2024
7c80bfc
Disabling VectorLoadShuffle bypassing optimization to comply with rea…
Sep 16, 2024
d72c82e
Review comments resolution.
Sep 17, 2024
2953004
Jcheck clearance
Sep 17, 2024
31a5864
Incorporating review and documentation suggestions.
Sep 18, 2024
42ca80c
Handling NPOT vector length for AArch64 SVE with vector sizes varying…
Sep 24, 2024
7327736
Review comments resolutions.
Oct 1, 2024
130fa3e
Merge branch 'master' of http://github.com/openjdk/jdk into JDK-8338023
Oct 3, 2024
6215ab9
Review comments resolution.
Oct 3, 2024
1cca8e2
Sharpening intrinsic exit check.
Oct 3, 2024
79ee29c
Updating tests to use floorMod
Oct 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 56 additions & 16 deletions src/hotspot/cpu/x86/assembler_x86.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4738,22 +4738,6 @@ void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_l
emit_int24(0x01, (0xC0 | encode), imm8);
}

void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(VM_Version::supports_evex(), "");
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int16(0x76, (0xC0 | encode));
}

void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(VM_Version::supports_avx512_vbmi(), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int16(0x7D, (0xC0 | encode));
}

void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) {
assert(VM_Version::supports_avx512_vbmi(), "");
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
Expand Down Expand Up @@ -16103,3 +16087,59 @@ void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_b
_input_size_in_bits = input_size_in_bits;
}
}

void Assembler::evpermi2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(VM_Version::supports_avx512_vbmi() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int16(0x75, (0xC0 | encode));
}

void Assembler::evpermi2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(VM_Version::supports_avx512bw() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int16(0x75, (0xC0 | encode));
}

void Assembler::evpermi2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int16(0x76, (0xC0 | encode));
}

void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int16(0x76, (0xC0 | encode));
}

void Assembler::evpermi2ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int16(0x77, (0xC0 | encode));
}

void Assembler::evpermi2pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), "");
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int16(0x77, (0xC0 | encode));
}

void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
assert(VM_Version::supports_avx512_vbmi(), "");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_is_evex_instruction();
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int16(0x7D, (0xC0 | encode));
}
7 changes: 6 additions & 1 deletion src/hotspot/cpu/x86/assembler_x86.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1962,9 +1962,14 @@ class Assembler : public AbstractAssembler {
void vpermilps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len);
void vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len);
void evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len);
void evpermi2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpermi2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpermi2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpermi2ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpermi2pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len);

void pause();

Expand Down
27 changes: 27 additions & 0 deletions src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6475,3 +6475,30 @@ void C2_MacroAssembler::vector_rearrange_int_float(BasicType bt, XMMRegister dst
vpermps(dst, shuffle, src, vlen_enc);
}
}

void C2_MacroAssembler::select_from_two_vectors_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1,
XMMRegister src2, int vlen_enc) {
switch(elem_bt) {
case T_BYTE:
evpermi2b(dst, src1, src2, vlen_enc);
break;
case T_SHORT:
evpermi2w(dst, src1, src2, vlen_enc);
break;
case T_INT:
evpermi2d(dst, src1, src2, vlen_enc);
break;
case T_LONG:
evpermi2q(dst, src1, src2, vlen_enc);
break;
case T_FLOAT:
evpermi2ps(dst, src1, src2, vlen_enc);
break;
case T_DOUBLE:
evpermi2pd(dst, src1, src2, vlen_enc);
break;
default:
fatal("Unsupported type %s", type2name(elem_bt));
break;
}
}
2 changes: 2 additions & 0 deletions src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -505,4 +505,6 @@
void vgather8b_offset(BasicType elem_bt, XMMRegister dst, Register base, Register idx_base,
Register offset, Register rtmp, int vlen_enc);

void select_from_two_vectors_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);

#endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP
27 changes: 27 additions & 0 deletions src/hotspot/cpu/x86/x86.ad
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1935,6 +1935,20 @@ bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
return false;
}
break;
case Op_SelectFromTwoVector:
if (size_in_bits < 128 || (size_in_bits < 512 && !VM_Version::supports_avx512vl())) {
return false;
}
if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
return false;
}
if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
return false;
}
if ((bt == T_INT || bt == T_FLOAT || bt == T_DOUBLE) && !VM_Version::supports_evex()) {
return false;
}
break;
case Op_MaskAll:
if (!VM_Version::supports_evex()) {
return false;
Expand Down Expand Up @@ -10468,3 +10482,16 @@ instruct DoubleClassCheck_reg_reg_vfpclass(rRegI dst, regD src, kReg ktmp, rFlag
%}
ins_pipe(pipe_slow);
%}


instruct vector_selectfrom_twovectors_reg_evex(vec index, vec src1, vec src2)
%{
match(Set index (SelectFromTwoVector (Binary index src1) src2));
format %{ "select_from_two_vector $index, $src1, $src2 \t!" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
BasicType bt = Matcher::vector_element_basic_type(this);
__ select_from_two_vectors_evex(bt, $index$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
%}
ins_pipe(pipe_slow);
%}
2 changes: 1 addition & 1 deletion src/hotspot/share/adlc/formssel.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4356,7 +4356,7 @@ bool MatchRule::is_vector() const {
"Replicate","ReverseV","ReverseBytesV",
"RoundDoubleModeV","RotateLeftV" , "RotateRightV", "LoadVector","StoreVector",
"LoadVectorGather", "StoreVectorScatter", "LoadVectorGatherMasked", "StoreVectorScatterMasked",
"VectorTest", "VectorLoadMask", "VectorStoreMask", "VectorBlend", "VectorInsert",
"SelectFromTwoVector", "VectorTest", "VectorLoadMask", "VectorStoreMask", "VectorBlend", "VectorInsert",
"VectorRearrange", "VectorLoadShuffle", "VectorLoadConst",
"VectorCastB2X", "VectorCastS2X", "VectorCastI2X",
"VectorCastL2X", "VectorCastF2X", "VectorCastD2X", "VectorCastF2HF", "VectorCastHF2F",
Expand Down
11 changes: 11 additions & 0 deletions src/hotspot/share/classfile/vmIntrinsics.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -979,6 +979,17 @@ class methodHandle;
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
do_name(vector_ternary_op_name, "ternaryOp") \
\
do_intrinsic(_VectorSelectFromTwoVectorOp, jdk_internal_vm_vector_VectorSupport, vector_select_from_op_name, vector_select_from_op_sig, F_S) \
do_signature(vector_select_from_op_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$SelectFromTwoVector;)" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
do_name(vector_select_from_op_name, "selectFromTwoVectorOp") \
\
do_intrinsic(_VectorFromBitsCoerced, jdk_internal_vm_vector_VectorSupport, vector_frombits_coerced_name, vector_frombits_coerced_sig, F_S) \
do_signature(vector_frombits_coerced_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/share/opto/c2compiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -816,6 +816,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
case vmIntrinsics::_VectorLoadMaskedOp:
case vmIntrinsics::_VectorStoreOp:
case vmIntrinsics::_VectorStoreMaskedOp:
case vmIntrinsics::_VectorSelectFromTwoVectorOp:
case vmIntrinsics::_VectorGatherOp:
case vmIntrinsics::_VectorScatterOp:
case vmIntrinsics::_VectorReductionCoerced:
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/share/opto/classes.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,7 @@ macro(Digit)
macro(LowerCase)
macro(UpperCase)
macro(Whitespace)
macro(SelectFromTwoVector)
macro(VectorBox)
macro(VectorBoxAllocate)
macro(VectorUnbox)
Expand Down
2 changes: 2 additions & 0 deletions src/hotspot/share/opto/library_call.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -753,6 +753,8 @@ bool LibraryCallKit::try_to_inline(int predicate) {
return inline_vector_extract();
case vmIntrinsics::_VectorCompressExpand:
return inline_vector_compress_expand();
case vmIntrinsics::_VectorSelectFromTwoVectorOp:
return inline_vector_select_from_two_vectors();
case vmIntrinsics::_IndexVector:
return inline_index_vector();
case vmIntrinsics::_IndexPartiallyInUpperRange:
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/share/opto/library_call.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,7 @@ class LibraryCallKit : public GraphKit {
bool inline_vector_compress_expand();
bool inline_index_vector();
bool inline_index_partially_in_upper_range();
bool inline_vector_select_from_two_vectors();

Node* gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2);

Expand Down
1 change: 1 addition & 0 deletions src/hotspot/share/opto/matcher.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2413,6 +2413,7 @@ void Matcher::find_shared_post_visit(Node* n, uint opcode) {
n->del_req(4);
break;
}
case Op_SelectFromTwoVector:
case Op_LoopLimit: {
Node* pair1 = new BinaryNode(n->in(1), n->in(2));
n->set_req(1, pair1);
Expand Down
Loading