Skip to content

Commit d4aacdb

Browse files
e1iue1iu
committed
8306136: [vectorapi] Intrinsics of VectorMask.laneIsSet()
Reviewed-by: psandoz, xgong
1 parent 783de32 commit d4aacdb

40 files changed

+1070
-78
lines changed

‎src/hotspot/cpu/aarch64/aarch64_vector.ad

+49
Original file line numberDiff line numberDiff line change
@@ -4587,6 +4587,55 @@ instruct insertD_gt128b(vReg dst, vReg src, vRegD val, immI idx,
45874587
%}
45884588

45894589
// ------------------------------ Extract --------------------------------------
4590+
4591+
// BOOLEAN
4592+
4593+
instruct extractUB_ireg(iRegINoSp dst, vReg src, iRegI idx, vReg tmp) %{
4594+
match(Set dst (ExtractUB src idx));
4595+
effect(TEMP tmp);
4596+
format %{ "extractUB_ireg $dst, $src, $idx\t# variable index. KILL $tmp" %}
4597+
ins_encode %{
4598+
// Input "src" is a vector of boolean represented as
4599+
// bytes with 0x00/0x01 as element values.
4600+
// "idx" is expected to be in range.
4601+
4602+
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
4603+
__ mov($tmp$$FloatRegister, __ B, 0, $idx$$Register);
4604+
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
4605+
__ tbl($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
4606+
$src$$FloatRegister, 1, $tmp$$FloatRegister);
4607+
} else {
4608+
assert(UseSVE > 0, "must be sve");
4609+
__ sve_tbl($tmp$$FloatRegister, __ B, $src$$FloatRegister, $tmp$$FloatRegister);
4610+
}
4611+
__ smov($dst$$Register, $tmp$$FloatRegister, __ B, 0);
4612+
%}
4613+
ins_pipe(pipe_slow);
4614+
%}
4615+
4616+
instruct extractUB_index_lt16(iRegINoSp dst, vReg src, immI idx) %{
4617+
predicate(n->in(2)->get_int() < 16);
4618+
match(Set dst (ExtractUB src idx));
4619+
format %{ "extractUB_index_lt16 $dst, $src, $idx\t# index < 16" %}
4620+
ins_encode %{
4621+
__ smov($dst$$Register, $src$$FloatRegister, __ B, (int)($idx$$constant));
4622+
%}
4623+
ins_pipe(pipe_slow);
4624+
%}
4625+
4626+
instruct extractUB_index_ge16(iRegINoSp dst, vReg src, immI idx, vReg tmp) %{
4627+
predicate(n->in(2)->get_int() >= 16);
4628+
match(Set dst (ExtractUB src idx));
4629+
effect(TEMP tmp);
4630+
format %{ "extractUB_index_ge16 $dst, $src, $idx\t# index >=16. KILL $tmp" %}
4631+
ins_encode %{
4632+
assert(UseSVE > 0, "must be sve");
4633+
__ sve_extract_integral($dst$$Register, T_BYTE, $src$$FloatRegister,
4634+
(int)($idx$$constant), $tmp$$FloatRegister);
4635+
%}
4636+
ins_pipe(pipe_slow);
4637+
%}
4638+
45904639
// BYTE
45914640

45924641
instruct extractB_index_lt16(iRegINoSp dst, vReg src, immI idx) %{

‎src/hotspot/cpu/aarch64/aarch64_vector_ad.m4

+28
Original file line numberDiff line numberDiff line change
@@ -3118,6 +3118,34 @@ instruct extract$1_index_ge$2($3 dst, vReg src, immI idx, vReg tmp) %{
31183118
ins_pipe(pipe_slow);
31193119
%}')dnl
31203120
dnl
3121+
3122+
// BOOLEAN
3123+
3124+
instruct extractUB_ireg(iRegINoSp dst, vReg src, iRegI idx, vReg tmp) %{
3125+
match(Set dst (ExtractUB src idx));
3126+
effect(TEMP tmp);
3127+
format %{ "extractUB_ireg $dst, $src, $idx\t# variable index. KILL $tmp" %}
3128+
ins_encode %{
3129+
// Input "src" is a vector of boolean represented as
3130+
// bytes with 0x00/0x01 as element values.
3131+
// "idx" is expected to be in range.
3132+
3133+
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
3134+
__ mov($tmp$$FloatRegister, __ B, 0, $idx$$Register);
3135+
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
3136+
__ tbl($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
3137+
$src$$FloatRegister, 1, $tmp$$FloatRegister);
3138+
} else {
3139+
assert(UseSVE > 0, "must be sve");
3140+
__ sve_tbl($tmp$$FloatRegister, __ B, $src$$FloatRegister, $tmp$$FloatRegister);
3141+
}
3142+
__ smov($dst$$Register, $tmp$$FloatRegister, __ B, 0);
3143+
%}
3144+
ins_pipe(pipe_slow);
3145+
%}
3146+
EXTRACT_INT_SMALL(UB, 16, iRegINoSp, smov, B)
3147+
EXTRACT_INT_LARGE(UB, 16, iRegINoSp, T_BYTE)
3148+
31213149
// BYTE
31223150
EXTRACT_INT_SMALL(B, 16, iRegINoSp, smov, B)
31233151
EXTRACT_INT_LARGE(B, 16, iRegINoSp, T_BYTE)

‎src/hotspot/share/classfile/vmIntrinsics.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -1104,7 +1104,7 @@ class methodHandle;
11041104
do_signature(vector_extract_sig, "(Ljava/lang/Class;" \
11051105
"Ljava/lang/Class;" \
11061106
"I" \
1107-
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
1107+
"Ljdk/internal/vm/vector/VectorSupport$VectorPayload;" \
11081108
"I" \
11091109
"Ljdk/internal/vm/vector/VectorSupport$VecExtractOp;)" \
11101110
"J") \

‎src/hotspot/share/opto/vectorIntrinsics.cpp

+84-42
Original file line numberDiff line numberDiff line change
@@ -2687,11 +2687,12 @@ bool LibraryCallKit::inline_vector_insert() {
26872687
}
26882688

26892689
// public static
2690-
// <V extends Vector<E>,
2690+
// <VM extends VectorPayload,
26912691
// E>
2692-
// long extract(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2693-
// V vec, int ix,
2694-
// VecExtractOp<V> defaultImpl)
2692+
// long extract(Class<? extends VM> vClass, Class<E> eClass,
2693+
// int length,
2694+
// VM vm, int i,
2695+
// VecExtractOp<VM> defaultImpl)
26952696
bool LibraryCallKit::inline_vector_extract() {
26962697
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
26972698
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
@@ -2701,13 +2702,12 @@ bool LibraryCallKit::inline_vector_extract() {
27012702
if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
27022703
return false; // dead code
27032704
}
2704-
if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2705+
if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
27052706
if (C->print_intrinsics()) {
2706-
tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2707+
tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s",
27072708
NodeClassNames[argument(0)->Opcode()],
27082709
NodeClassNames[argument(1)->Opcode()],
2709-
NodeClassNames[argument(2)->Opcode()],
2710-
NodeClassNames[argument(4)->Opcode()]);
2710+
NodeClassNames[argument(2)->Opcode()]);
27112711
}
27122712
return false; // not enough info for intrinsification
27132713
}
@@ -2726,51 +2726,93 @@ bool LibraryCallKit::inline_vector_extract() {
27262726
}
27272727
BasicType elem_bt = elem_type->basic_type();
27282728
int num_elem = vlen->get_con();
2729-
int vopc = ExtractNode::opcode(elem_bt);
2730-
if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2731-
if (C->print_intrinsics()) {
2732-
tty->print_cr(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2733-
num_elem, type2name(elem_bt));
2734-
}
2735-
return false; // not supported
2736-
}
27372729

27382730
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
27392731
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
27402732

2741-
Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2742-
if (opd == nullptr) {
2743-
return false;
2744-
}
2733+
Node* opd = nullptr;
27452734

2746-
ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2747-
Node* operation = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2735+
if (is_vector_mask(vbox_klass)) {
2736+
// vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
27482737

2749-
Node* bits = nullptr;
2750-
switch (elem_bt) {
2751-
case T_BYTE:
2752-
case T_SHORT:
2753-
case T_INT: {
2754-
bits = gvn().transform(new ConvI2LNode(operation));
2755-
break;
2738+
Node* pos = argument(4); // can be variable
2739+
if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2740+
// Transform mask to vector with type of boolean and utilize ExtractUB node.
2741+
opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2742+
if (opd == nullptr) {
2743+
return false;
2744+
}
2745+
opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2746+
opd = gvn().transform(new ExtractUBNode(opd, pos));
2747+
opd = gvn().transform(new ConvI2LNode(opd));
2748+
} else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2749+
opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2750+
if (opd == nullptr) {
2751+
return false;
2752+
}
2753+
// VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2754+
if (opd->bottom_type()->isa_vectmask() == nullptr) {
2755+
opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2756+
}
2757+
// ((toLong() >>> pos) & 1L
2758+
opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2759+
opd = gvn().transform(new URShiftLNode(opd, pos));
2760+
opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2761+
} else {
2762+
if (C->print_intrinsics()) {
2763+
tty->print_cr(" ** Rejected mask extraction because architecture does not support it");
2764+
}
2765+
return false; // not supported
27562766
}
2757-
case T_FLOAT: {
2758-
bits = gvn().transform(new MoveF2INode(operation));
2759-
bits = gvn().transform(new ConvI2LNode(bits));
2760-
break;
2767+
} else {
2768+
// vbox_klass is vector. This is used for Vector.lane(int).
2769+
if (!idx->is_con()) {
2770+
if (C->print_intrinsics()) {
2771+
tty->print_cr(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2772+
}
2773+
return false; // not enough info for intrinsification
27612774
}
2762-
case T_DOUBLE: {
2763-
bits = gvn().transform(new MoveD2LNode(operation));
2764-
break;
2775+
2776+
int vopc = ExtractNode::opcode(elem_bt);
2777+
if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2778+
if (C->print_intrinsics()) {
2779+
tty->print_cr(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2780+
num_elem, type2name(elem_bt));
2781+
}
2782+
return false; // not supported
27652783
}
2766-
case T_LONG: {
2767-
bits = operation; // no conversion needed
2768-
break;
2784+
2785+
opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2786+
if (opd == nullptr) {
2787+
return false;
27692788
}
2770-
default: fatal("%s", type2name(elem_bt));
2771-
}
2789+
ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
27722790

2773-
set_result(bits);
2791+
opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2792+
switch (elem_bt) {
2793+
case T_BYTE:
2794+
case T_SHORT:
2795+
case T_INT: {
2796+
opd = gvn().transform(new ConvI2LNode(opd));
2797+
break;
2798+
}
2799+
case T_FLOAT: {
2800+
opd = gvn().transform(new MoveF2INode(opd));
2801+
opd = gvn().transform(new ConvI2LNode(opd));
2802+
break;
2803+
}
2804+
case T_DOUBLE: {
2805+
opd = gvn().transform(new MoveD2LNode(opd));
2806+
break;
2807+
}
2808+
case T_LONG: {
2809+
// no conversion needed
2810+
break;
2811+
}
2812+
default: fatal("%s", type2name(elem_bt));
2813+
}
2814+
}
2815+
set_result(opd);
27742816
return true;
27752817
}
27762818

‎src/hotspot/share/opto/vectornode.cpp

+3-2
Original file line numberDiff line numberDiff line change
@@ -1202,9 +1202,10 @@ int ExtractNode::opcode(BasicType bt) {
12021202
}
12031203
}
12041204

1205-
// Extract a scalar element of vector.
1205+
// Extract a scalar element of vector by constant position.
12061206
Node* ExtractNode::make(Node* v, ConINode* pos, BasicType bt) {
1207-
assert(pos->get_int() < Matcher::max_vector_size(bt), "pos in range");
1207+
assert(pos->get_int() >= 0 &&
1208+
pos->get_int() < Matcher::max_vector_size(bt), "pos in range");
12081209
switch (bt) {
12091210
case T_BOOLEAN: return new ExtractUBNode(v, pos);
12101211
case T_BYTE: return new ExtractBNode(v, pos);

‎src/hotspot/share/opto/vectornode.hpp

+10-14
Original file line numberDiff line numberDiff line change
@@ -1280,12 +1280,8 @@ class VectorLoadConstNode : public VectorNode {
12801280
// Extract a scalar from a vector at position "pos"
12811281
class ExtractNode : public Node {
12821282
public:
1283-
ExtractNode(Node* src, ConINode* pos) : Node(nullptr, src, (Node*)pos) {
1284-
assert(in(2)->get_int() >= 0, "positive constants");
1285-
}
1283+
ExtractNode(Node* src, Node* pos) : Node(nullptr, src, pos) {}
12861284
virtual int Opcode() const;
1287-
uint pos() const { return in(2)->get_int(); }
1288-
12891285
static Node* make(Node* v, ConINode* pos, BasicType bt);
12901286
static int opcode(BasicType bt);
12911287
};
@@ -1294,7 +1290,7 @@ class ExtractNode : public Node {
12941290
// Extract a byte from a vector at position "pos"
12951291
class ExtractBNode : public ExtractNode {
12961292
public:
1297-
ExtractBNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1293+
ExtractBNode(Node* src, Node* pos) : ExtractNode(src, pos) {}
12981294
virtual int Opcode() const;
12991295
virtual const Type* bottom_type() const { return TypeInt::BYTE; }
13001296
virtual uint ideal_reg() const { return Op_RegI; }
@@ -1304,17 +1300,17 @@ class ExtractBNode : public ExtractNode {
13041300
// Extract a boolean from a vector at position "pos"
13051301
class ExtractUBNode : public ExtractNode {
13061302
public:
1307-
ExtractUBNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1303+
ExtractUBNode(Node* src, Node* pos) : ExtractNode(src, pos) {}
13081304
virtual int Opcode() const;
1309-
virtual const Type* bottom_type() const { return TypeInt::UBYTE; }
1305+
virtual const Type* bottom_type() const { return TypeInt::BOOL; }
13101306
virtual uint ideal_reg() const { return Op_RegI; }
13111307
};
13121308

13131309
//------------------------------ExtractCNode-----------------------------------
13141310
// Extract a char from a vector at position "pos"
13151311
class ExtractCNode : public ExtractNode {
13161312
public:
1317-
ExtractCNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1313+
ExtractCNode(Node* src, Node* pos) : ExtractNode(src, pos) {}
13181314
virtual int Opcode() const;
13191315
virtual const Type *bottom_type() const { return TypeInt::CHAR; }
13201316
virtual uint ideal_reg() const { return Op_RegI; }
@@ -1324,7 +1320,7 @@ class ExtractCNode : public ExtractNode {
13241320
// Extract a short from a vector at position "pos"
13251321
class ExtractSNode : public ExtractNode {
13261322
public:
1327-
ExtractSNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1323+
ExtractSNode(Node* src, Node* pos) : ExtractNode(src, pos) {}
13281324
virtual int Opcode() const;
13291325
virtual const Type *bottom_type() const { return TypeInt::SHORT; }
13301326
virtual uint ideal_reg() const { return Op_RegI; }
@@ -1334,7 +1330,7 @@ class ExtractSNode : public ExtractNode {
13341330
// Extract an int from a vector at position "pos"
13351331
class ExtractINode : public ExtractNode {
13361332
public:
1337-
ExtractINode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1333+
ExtractINode(Node* src, Node* pos) : ExtractNode(src, pos) {}
13381334
virtual int Opcode() const;
13391335
virtual const Type *bottom_type() const { return TypeInt::INT; }
13401336
virtual uint ideal_reg() const { return Op_RegI; }
@@ -1344,7 +1340,7 @@ class ExtractINode : public ExtractNode {
13441340
// Extract a long from a vector at position "pos"
13451341
class ExtractLNode : public ExtractNode {
13461342
public:
1347-
ExtractLNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1343+
ExtractLNode(Node* src, Node* pos) : ExtractNode(src, pos) {}
13481344
virtual int Opcode() const;
13491345
virtual const Type *bottom_type() const { return TypeLong::LONG; }
13501346
virtual uint ideal_reg() const { return Op_RegL; }
@@ -1354,7 +1350,7 @@ class ExtractLNode : public ExtractNode {
13541350
// Extract a float from a vector at position "pos"
13551351
class ExtractFNode : public ExtractNode {
13561352
public:
1357-
ExtractFNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1353+
ExtractFNode(Node* src, Node* pos) : ExtractNode(src, pos) {}
13581354
virtual int Opcode() const;
13591355
virtual const Type *bottom_type() const { return Type::FLOAT; }
13601356
virtual uint ideal_reg() const { return Op_RegF; }
@@ -1364,7 +1360,7 @@ class ExtractFNode : public ExtractNode {
13641360
// Extract a double from a vector at position "pos"
13651361
class ExtractDNode : public ExtractNode {
13661362
public:
1367-
ExtractDNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1363+
ExtractDNode(Node* src, Node* pos) : ExtractNode(src, pos) {}
13681364
virtual int Opcode() const;
13691365
virtual const Type *bottom_type() const { return Type::DOUBLE; }
13701366
virtual uint ideal_reg() const { return Op_RegD; }

‎src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java

+7-7
Original file line numberDiff line numberDiff line change
@@ -306,20 +306,20 @@ long reductionCoerced(int oprId,
306306

307307
/* ============================================================================ */
308308

309-
public interface VecExtractOp<V extends Vector<?>> {
310-
long apply(V v, int i);
309+
public interface VecExtractOp<VM extends VectorPayload> {
310+
long apply(VM vm, int i);
311311
}
312312

313313
@IntrinsicCandidate
314314
public static
315-
<V extends Vector<E>,
315+
<VM extends VectorPayload,
316316
E>
317-
long extract(Class<? extends V> vClass, Class<E> eClass,
317+
long extract(Class<? extends VM> vClass, Class<E> eClass,
318318
int length,
319-
V v, int i,
320-
VecExtractOp<V> defaultImpl) {
319+
VM vm, int i,
320+
VecExtractOp<VM> defaultImpl) {
321321
assert isNonCapturingLambda(defaultImpl) : defaultImpl;
322-
return defaultImpl.apply(v, i);
322+
return defaultImpl.apply(vm, i);
323323
}
324324

325325
/* ============================================================================ */

0 commit comments

Comments
 (0)