Skip to content

Commit 0e7348d

Browse files
author
Jatin Bhateja
committed
8273949: Intrinsic creation for VectorMask.toLong operation.
Reviewed-by: psandoz, sviswanathan, eliu
1 parent b44662a commit 0e7348d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+645
-251
lines changed

‎src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4058,13 +4058,14 @@ void C2_MacroAssembler::masked_op(int ideal_opc, int mask_len, KRegister dst,
40584058
}
40594059

40604060
#ifdef _LP64
4061-
void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp,
4062-
Register tmp, KRegister ktmp, int masklen, int vec_enc) {
4063-
assert(VM_Version::supports_avx512vlbw(), "");
4064-
vpxor(xtmp, xtmp, xtmp, vec_enc);
4065-
vpsubb(xtmp, xtmp, mask, vec_enc);
4066-
evpmovb2m(ktmp, xtmp, vec_enc);
4067-
kmovql(tmp, ktmp);
4061+
void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, KRegister mask,
4062+
Register tmp, int masklen, int vec_enc) {
4063+
if(VM_Version::supports_avx512bw()) {
4064+
kmovql(tmp, mask);
4065+
} else {
4066+
assert(masklen <= 16, "");
4067+
kmovwl(tmp, mask);
4068+
}
40684069
switch(opc) {
40694070
case Op_VectorMaskTrueCount:
40704071
popcntq(dst, tmp);

‎src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -224,8 +224,7 @@
224224

225225
public:
226226
#ifdef _LP64
227-
void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp, Register tmp,
228-
KRegister ktmp, int masklen, int vec_enc);
227+
void vector_mask_operation(int opc, Register dst, KRegister mask, Register tmp, int masklen, int vec_enc);
229228

230229
void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp, XMMRegister xtmp1,
231230
Register tmp, int masklen, int vec_enc);

‎src/hotspot/cpu/x86/x86.ad

Lines changed: 54 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1558,6 +1558,7 @@ const bool Matcher::match_rule_supported(int opcode) {
15581558
case Op_VectorMaskFirstTrue:
15591559
case Op_VectorMaskLastTrue:
15601560
case Op_VectorMaskTrueCount:
1561+
case Op_VectorMaskToLong:
15611562
if (!is_LP64 || UseAVX < 1) {
15621563
return false;
15631564
}
@@ -8659,58 +8660,87 @@ instruct vmasked_store64(memory mem, vec src, kReg mask) %{
86598660
ins_pipe( pipe_slow );
86608661
%}
86618662

8662-
instruct vmask_truecount_evex(rRegI dst, vec mask, rRegL tmp, kReg ktmp, vec xtmp) %{
8663-
predicate(VM_Version::supports_avx512vlbw());
8663+
instruct vmask_tolong_evex(rRegL dst, kReg mask) %{
8664+
predicate(n->in(1)->bottom_type()->isa_vectmask());
8665+
match(Set dst (VectorMaskToLong mask));
8666+
format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
8667+
ins_encode %{
8668+
if (VM_Version::supports_avx512vlbw()) {
8669+
__ kmovql($dst$$Register, $mask$$KRegister);
8670+
} else {
8671+
int mask_len = Matcher::vector_length(this, $mask);
8672+
assert(mask_len <= 16, "");
8673+
__ kmovwl($dst$$Register, $mask$$KRegister);
8674+
}
8675+
%}
8676+
ins_pipe( pipe_slow );
8677+
%}
8678+
8679+
instruct vmask_tolong_avx(rRegL dst, vec mask, vec xtmp) %{
8680+
predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL &&
8681+
n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN);
8682+
match(Set dst (VectorMaskToLong mask));
8683+
format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
8684+
effect(TEMP xtmp);
8685+
ins_encode %{
8686+
int vlen_enc = vector_length_encoding(this, $mask);
8687+
__ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
8688+
__ vpsubb($xtmp$$XMMRegister, $xtmp$$XMMRegister, $mask$$XMMRegister, vlen_enc);
8689+
__ vpmovmskb($dst$$Register, $xtmp$$XMMRegister, vlen_enc);
8690+
%}
8691+
ins_pipe( pipe_slow );
8692+
%}
8693+
8694+
instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
8695+
predicate(n->in(1)->bottom_type()->isa_vectmask());
86648696
match(Set dst (VectorMaskTrueCount mask));
8665-
effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp, TEMP xtmp);
8666-
format %{ "vector_truecount_evex $mask \t! vector mask true count" %}
8697+
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
8698+
format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
86678699
ins_encode %{
86688700
int opcode = this->ideal_Opcode();
86698701
int vlen_enc = vector_length_encoding(this, $mask);
86708702
int mask_len = Matcher::vector_length(this, $mask);
8671-
__ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
8672-
$tmp$$Register, $ktmp$$KRegister, mask_len, vlen_enc);
8703+
__ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister, $tmp$$Register, mask_len, vlen_enc);
86738704
%}
86748705
ins_pipe( pipe_slow );
86758706
%}
86768707

8677-
instruct vmask_first_or_last_true_evex(rRegI dst, vec mask, rRegL tmp, kReg ktmp, vec xtmp, rFlagsReg cr) %{
8678-
predicate(VM_Version::supports_avx512vlbw());
8679-
match(Set dst (VectorMaskFirstTrue mask));
8680-
match(Set dst (VectorMaskLastTrue mask));
8681-
effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp, TEMP xtmp, KILL cr);
8682-
format %{ "vector_mask_first_or_last_true_evex $mask \t! vector first/last true location" %}
8708+
instruct vmask_truecount_avx(rRegI dst, vec mask, rRegL tmp, vec xtmp, vec xtmp1, rFlagsReg cr) %{
8709+
predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
8710+
match(Set dst (VectorMaskTrueCount mask));
8711+
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, TEMP xtmp1, KILL cr);
8712+
format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp and $xtmp1 as TEMP" %}
86838713
ins_encode %{
86848714
int opcode = this->ideal_Opcode();
86858715
int vlen_enc = vector_length_encoding(this, $mask);
86868716
int mask_len = Matcher::vector_length(this, $mask);
86878717
__ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
8688-
$tmp$$Register, $ktmp$$KRegister, mask_len, vlen_enc);
8718+
$xtmp1$$XMMRegister, $tmp$$Register, mask_len, vlen_enc);
86898719
%}
86908720
ins_pipe( pipe_slow );
86918721
%}
86928722

8693-
instruct vmask_truecount_avx(rRegI dst, vec mask, rRegL tmp, vec xtmp, vec xtmp1) %{
8694-
predicate(!VM_Version::supports_avx512vlbw());
8695-
match(Set dst (VectorMaskTrueCount mask));
8696-
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, TEMP xtmp1);
8697-
format %{ "vector_truecount_avx $mask \t! vector mask true count" %}
8723+
instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
8724+
predicate(n->in(1)->bottom_type()->isa_vectmask());
8725+
match(Set dst (VectorMaskFirstTrue mask));
8726+
match(Set dst (VectorMaskLastTrue mask));
8727+
effect(TEMP_DEF dst, TEMP tmp, KILL cr);
8728+
format %{ "vector_mask_first_or_last_true_evex $dst, $mask \t! using $tmp as TEMP" %}
86988729
ins_encode %{
86998730
int opcode = this->ideal_Opcode();
87008731
int vlen_enc = vector_length_encoding(this, $mask);
87018732
int mask_len = Matcher::vector_length(this, $mask);
8702-
__ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
8703-
$xtmp1$$XMMRegister, $tmp$$Register, mask_len, vlen_enc);
8733+
__ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister, $tmp$$Register, mask_len, vlen_enc);
87048734
%}
87058735
ins_pipe( pipe_slow );
87068736
%}
87078737

87088738
instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, rRegL tmp, vec xtmp, vec xtmp1, rFlagsReg cr) %{
8709-
predicate(!VM_Version::supports_avx512vlbw());
8739+
predicate(n->in(1)->bottom_type()->isa_vectmask() == NULL);
87108740
match(Set dst (VectorMaskFirstTrue mask));
87118741
match(Set dst (VectorMaskLastTrue mask));
87128742
effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, TEMP xtmp1, KILL cr);
8713-
format %{ "vector_mask_first_or_last_true_avx $mask \t! vector first/last true location" %}
8743+
format %{ "vector_mask_first_or_last_true_avx $dst, $mask \t! using $tmp, $xtmp and $xtmp1 as TEMP" %}
87148744
ins_encode %{
87158745
int opcode = this->ideal_Opcode();
87168746
int vlen_enc = vector_length_encoding(this, $mask);
@@ -9297,7 +9327,7 @@ instruct evcmp_masked(kReg dst, vec src1, vec src2, immI8 cond, kReg mask, rRegP
92979327
#ifdef _LP64
92989328
instruct mask_all_evexI_imm(kReg dst, immI cnt, rRegL tmp) %{
92999329
match(Set dst (MaskAll cnt));
9300-
effect(TEMP tmp);
9330+
effect(TEMP_DEF dst, TEMP tmp);
93019331
format %{ "mask_all_evexI $dst, $cnt \t! using $tmp as TEMP" %}
93029332
ins_encode %{
93039333
int vec_len = Matcher::vector_length(this);
@@ -9317,7 +9347,7 @@ instruct mask_all_evexI_imm(kReg dst, immI cnt, rRegL tmp) %{
93179347

93189348
instruct mask_all_evexI(kReg dst, rRegI src, rRegL tmp) %{
93199349
match(Set dst (MaskAll src));
9320-
effect(TEMP tmp);
9350+
effect(TEMP_DEF dst, TEMP tmp);
93219351
format %{ "mask_all_evexI $dst, $src \t! using $tmp as TEMP" %}
93229352
ins_encode %{
93239353
int vec_len = Matcher::vector_length(this);

‎src/hotspot/share/classfile/vmIntrinsics.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1080,7 +1080,7 @@ class methodHandle;
10801080
"I" \
10811081
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
10821082
"Ljdk/internal/vm/vector/VectorSupport$VectorMaskOp;)" \
1083-
"I") \
1083+
"J") \
10841084
do_name(vector_mask_oper_name, "maskReductionCoerced") \
10851085
\
10861086
/* (2) Bytecode intrinsics */ \

‎src/hotspot/share/opto/classes.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,7 @@ macro(VectorMaskOp)
425425
macro(VectorMaskTrueCount)
426426
macro(VectorMaskFirstTrue)
427427
macro(VectorMaskLastTrue)
428+
macro(VectorMaskToLong)
428429
macro(Pack)
429430
macro(PackB)
430431
macro(PackS)

‎src/hotspot/share/opto/vectorIntrinsics.cpp

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -653,7 +653,7 @@ bool LibraryCallKit::inline_vector_shuffle_iota() {
653653
}
654654

655655
// <E, M>
656-
// int maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
656+
// long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
657657
// int length, M m, VectorMaskOp<M> defaultImpl)
658658
bool LibraryCallKit::inline_vector_mask_operation() {
659659
const TypeInt* oper = gvn().type(argument(0))->isa_int();
@@ -698,8 +698,14 @@ bool LibraryCallKit::inline_vector_mask_operation() {
698698
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
699699
const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
700700
Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
701-
Node* store_mask = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
702-
Node* maskoper = gvn().transform(VectorMaskOpNode::make(store_mask, TypeInt::INT, mopc));
701+
if (mask_vec->bottom_type()->isa_vectmask() == NULL) {
702+
mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
703+
}
704+
const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
705+
Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
706+
if (mopc != Op_VectorMaskToLong) {
707+
maskoper = ConvI2L(maskoper);
708+
}
703709
set_result(maskoper);
704710

705711
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));

‎src/hotspot/share/opto/vectornode.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1425,13 +1425,14 @@ Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) {
14251425
return new VectorMaskLastTrueNode(mask, ty);
14261426
case Op_VectorMaskFirstTrue:
14271427
return new VectorMaskFirstTrueNode(mask, ty);
1428+
case Op_VectorMaskToLong:
1429+
return new VectorMaskToLongNode(mask, ty);
14281430
default:
14291431
assert(false, "Unhandled operation");
14301432
}
14311433
return NULL;
14321434
}
14331435

1434-
14351436
#ifndef PRODUCT
14361437
void VectorBoxAllocateNode::dump_spec(outputStream *st) const {
14371438
CallStaticJavaNode::dump_spec(st);

‎src/hotspot/share/opto/vectornode.hpp

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -922,7 +922,7 @@ class VectorMaskOpNode : public TypeNode {
922922
public:
923923
VectorMaskOpNode(Node* mask, const Type* ty, int mopc):
924924
TypeNode(ty, 2), _mopc(mopc) {
925-
assert(mask->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN, "");
925+
assert(Matcher::has_predicated_vectors() || mask->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN, "");
926926
init_req(1, mask);
927927
}
928928

@@ -957,6 +957,14 @@ class VectorMaskLastTrueNode : public VectorMaskOpNode {
957957
virtual int Opcode() const;
958958
};
959959

960+
class VectorMaskToLongNode : public VectorMaskOpNode {
961+
public:
962+
VectorMaskToLongNode(Node* mask, const Type* ty):
963+
VectorMaskOpNode(mask, ty, Op_VectorMaskToLong) {}
964+
virtual int Opcode() const;
965+
virtual uint ideal_reg() const { return Op_RegL; }
966+
};
967+
960968
//-------------------------- Vector mask broadcast -----------------------------------
961969
class MaskAllNode : public VectorNode {
962970
public:

‎src/hotspot/share/prims/vectorSupport.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,18 @@ int VectorSupport::vop2ideal(jint id, BasicType bt) {
430430
}
431431
break;
432432
}
433+
case VECTOR_OP_MASK_TOLONG: {
434+
switch (bt) {
435+
case T_BYTE: // fall-through
436+
case T_SHORT: // fall-through
437+
case T_INT: // fall-through
438+
case T_LONG: // fall-through
439+
case T_FLOAT: // fall-through
440+
case T_DOUBLE: return Op_VectorMaskToLong;
441+
default: fatal("MASK_TOLONG: %s", type2name(bt));
442+
}
443+
break;
444+
}
433445
case VECTOR_OP_TAN:
434446
case VECTOR_OP_TANH:
435447
case VECTOR_OP_SIN:

‎src/hotspot/share/prims/vectorSupport.hpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,11 @@ class VectorSupport : AllStatic {
8282
VECTOR_OP_MASK_TRUECOUNT = 19,
8383
VECTOR_OP_MASK_FIRSTTRUE = 20,
8484
VECTOR_OP_MASK_LASTTRUE = 21,
85+
VECTOR_OP_MASK_TOLONG = 22,
8586

8687
// Rotate operations
87-
VECTOR_OP_LROTATE = 22,
88-
VECTOR_OP_RROTATE = 23,
88+
VECTOR_OP_LROTATE = 23,
89+
VECTOR_OP_RROTATE = 24,
8990

9091
// Vector Math Library
9192
VECTOR_OP_TAN = 101,

‎src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,11 @@ public class VectorSupport {
6969
public static final int VECTOR_OP_MASK_TRUECOUNT = 19;
7070
public static final int VECTOR_OP_MASK_FIRSTTRUE = 20;
7171
public static final int VECTOR_OP_MASK_LASTTRUE = 21;
72+
public static final int VECTOR_OP_MASK_TOLONG = 22;
7273

7374
// Rotate operations
74-
public static final int VECTOR_OP_LROTATE = 22;
75-
public static final int VECTOR_OP_RROTATE = 23;
75+
public static final int VECTOR_OP_LROTATE = 23;
76+
public static final int VECTOR_OP_RROTATE = 24;
7677

7778
// Math routines
7879
public static final int VECTOR_OP_TAN = 101;
@@ -632,18 +633,18 @@ VP maybeRebox(VP v) {
632633

633634
/* ============================================================================ */
634635
public interface VectorMaskOp<M extends VectorMask<?>> {
635-
int apply(M m);
636+
long apply(M m);
636637
}
637638

638639
@IntrinsicCandidate
639640
public static
640641
<M extends VectorMask<E>,
641642
E>
642-
int maskReductionCoerced(int oper,
643-
Class<? extends M> mClass, Class<?> eClass,
644-
int length,
645-
M m,
646-
VectorMaskOp<M> defaultImpl) {
643+
long maskReductionCoerced(int oper,
644+
Class<? extends M> mClass, Class<?> eClass,
645+
int length,
646+
M m,
647+
VectorMaskOp<M> defaultImpl) {
647648
assert isNonCapturingLambda(defaultImpl) : defaultImpl;
648649
return defaultImpl.apply(m);
649650
}

‎src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
*/
2525
package jdk.incubator.vector;
2626

27+
import java.util.Objects;
28+
2729
import jdk.internal.vm.annotation.ForceInline;
2830

2931
import static jdk.incubator.vector.VectorOperators.*;
@@ -63,23 +65,13 @@ public final VectorSpecies<E> vectorSpecies() {
6365

6466
@Override
6567
public boolean laneIsSet(int i) {
66-
return getBits()[i];
67-
}
68-
69-
@Override
70-
public long toLong() {
71-
// FIXME: This should be an intrinsic.
72-
if (length() > Long.SIZE) {
73-
throw new UnsupportedOperationException("too many lanes for one long");
74-
}
75-
long res = 0;
76-
long set = 1;
77-
boolean[] bits = getBits();
78-
for (int i = 0; i < bits.length; i++) {
79-
res = bits[i] ? res | set : res;
80-
set = set << 1;
68+
int length = length();
69+
Objects.checkIndex(i, length);
70+
if (length <= Long.SIZE) {
71+
return ((toLong() >>> i) & 1L) == 1;
72+
} else {
73+
return getBits()[i];
8174
}
82-
return res;
8375
}
8476

8577
@Override
@@ -180,6 +172,17 @@ static int lastTrueHelper(boolean[] bits) {
180172
return -1;
181173
}
182174

175+
/*package-private*/
176+
static long toLongHelper(boolean[] bits) {
177+
long res = 0;
178+
long set = 1;
179+
for (int i = 0; i < bits.length; i++) {
180+
res = bits[i] ? res | set : res;
181+
set = set << 1;
182+
}
183+
return res;
184+
}
185+
183186
@Override
184187
@ForceInline
185188
public VectorMask<E> indexInRange(int offset, int limit) {

0 commit comments

Comments
 (0)