Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
8278889: AArch64: [vectorapi] VectorMaskLoadStoreTest.testMaskCast() …
…test fail

Reviewed-by: njian, kvn
  • Loading branch information
theRealELiu authored and Vladimir Kozlov committed Dec 24, 2021
1 parent 04ad668 commit 6588bedc19ab42cec9e5bb6f13be14fb4dc5a655
Showing 37 changed files with 409 additions and 91 deletions.
@@ -411,21 +411,22 @@ instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, r
ins_pipe(pipe_slow);
%}

// maskAll
// maskAll (full or partial predicate size)

instruct vmaskAll_immI(pRegGov dst, immI src) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
ins_cost(SVE_COST);
format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
ins_encode %{
int con = (int)$src$$constant;
if (con == 0) {
__ sve_pfalse(as_PRegister($dst$$reg));
} else {
assert(con == -1, "invalid constant value for mask");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
}
%}
ins_pipe(pipe_slow);
@@ -435,14 +436,22 @@ instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
effect(TEMP tmp, KILL cr);
ins_cost(2 * SVE_COST);
ins_cost(3 * SVE_COST);
format %{ "sve_dup $tmp, $src\n\t"
"sve_cmpne $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
"sve_ptrue_lanecnt $dst\n\t"
"sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
if (length_in_bytes < MaxVectorSize) {
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
} else {
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
}
%}
ins_pipe(pipe_slow);
%}
@@ -451,15 +460,16 @@ instruct vmaskAll_immL(pRegGov dst, immL src) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
ins_cost(SVE_COST);
format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) (D)" %}
format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (D)" %}
ins_encode %{
long con = (long)$src$$constant;
if (con == 0) {
__ sve_pfalse(as_PRegister($dst$$reg));
} else {
assert(con == -1, "invalid constant value for mask");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
}
%}
ins_pipe(pipe_slow);
@@ -469,14 +479,22 @@ instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
effect(TEMP tmp, KILL cr);
ins_cost(2 * SVE_COST);
ins_cost(3 * SVE_COST);
format %{ "sve_dup $tmp, $src\n\t"
"sve_cmpne $dst, $tmp, 0\t# mask all (sve) (D)" %}
"sve_ptrue_lanecnt $dst\n\t"
"sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (D)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
if (length_in_bytes < MaxVectorSize) {
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
} else {
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
}
%}
ins_pipe(pipe_slow);
%}
@@ -3084,6 +3102,7 @@ instruct reduce_maxF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
match(Set dst (MaxReductionV (Binary src1 src2) pg));
ins_cost(SVE_COST);
effect(TEMP_DEF dst);
format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated (sve)" %}
ins_encode %{
__ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
@@ -3098,6 +3117,7 @@ instruct reduce_maxD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
match(Set dst (MaxReductionV (Binary src1 src2) pg));
ins_cost(SVE_COST);
effect(TEMP_DEF dst);
format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated (sve)" %}
ins_encode %{
__ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
@@ -3380,6 +3400,7 @@ instruct reduce_minF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
match(Set dst (MinReductionV (Binary src1 src2) pg));
ins_cost(SVE_COST);
effect(TEMP_DEF dst);
format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated (sve)" %}
ins_encode %{
__ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
@@ -3394,6 +3415,7 @@ instruct reduce_minD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
match(Set dst (MinReductionV (Binary src1 src2) pg));
ins_cost(SVE_COST);
effect(TEMP_DEF dst);
format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated (sve)" %}
ins_encode %{
__ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
@@ -356,15 +356,16 @@ instruct vmaskAll_imm$1(pRegGov dst, imm$1 src) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
ins_cost(SVE_COST);
format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) ($2)" %}
format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) ($2)" %}
ins_encode %{
ifelse($1, `I', int, long) con = (ifelse($1, `I', int, long))$src$$constant;
if (con == 0) {
__ sve_pfalse(as_PRegister($dst$$reg));
} else {
assert(con == -1, "invalid constant value for mask");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
}
%}
ins_pipe(pipe_slow);
@@ -377,19 +378,27 @@ instruct vmaskAll$1(pRegGov dst, ifelse($1, `I', iRegIorL2I, iRegL) src, vReg tm
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
effect(TEMP tmp, KILL cr);
ins_cost(2 * SVE_COST);
ins_cost(3 * SVE_COST);
format %{ "sve_dup $tmp, $src\n\t"
"sve_cmpne $dst, $tmp, 0\t# mask all (sve) ($2)" %}
"sve_ptrue_lanecnt $dst\n\t"
"sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) ($2)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
if (length_in_bytes < MaxVectorSize) {
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
} else {
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// maskAll
// maskAll (full or partial predicate size)
MASKALL_IMM(I, B/H/S)
MASKALL(I, B/H/S)
MASKALL_IMM(L, D)
@@ -1807,6 +1816,7 @@ instruct reduce_$1$2_masked($5 dst, $5 src1, vReg src2, pRegGov pg) %{
n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
ins_cost(SVE_COST);
effect(TEMP_DEF dst);
format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated (sve)" %}
ins_encode %{
__ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
@@ -1232,3 +1232,39 @@ void C2_MacroAssembler::sve_reduce_integral(int opc, Register dst, BasicType bt,
}
}
}

// Set elements of the dst predicate to true if the element number is
// in the range of [0, lane_cnt), or to false otherwise.
void C2_MacroAssembler::sve_ptrue_lanecnt(PRegister dst, SIMD_RegVariant size, int lane_cnt) {
assert(size != Q, "invalid size");
switch(lane_cnt) {
case 1: /* VL1 */
case 2: /* VL2 */
case 3: /* VL3 */
case 4: /* VL4 */
case 5: /* VL5 */
case 6: /* VL6 */
case 7: /* VL7 */
case 8: /* VL8 */
sve_ptrue(dst, size, lane_cnt);
break;
case 16:
sve_ptrue(dst, size, /* VL16 */ 0b01001);
break;
case 32:
sve_ptrue(dst, size, /* VL32 */ 0b01010);
break;
case 64:
sve_ptrue(dst, size, /* VL64 */ 0b01011);
break;
case 128:
sve_ptrue(dst, size, /* VL128 */ 0b01100);
break;
case 256:
sve_ptrue(dst, size, /* VL256 */ 0b01101);
break;
default:
assert(false, "unsupported");
ShouldNotReachHere();
}
}
@@ -88,6 +88,10 @@
void sve_reduce_integral(int opc, Register dst, BasicType bt, Register src1,
FloatRegister src2, PRegister pg, FloatRegister tmp);

// Set elements of the dst predicate to true if the element number is
// in the range of [0, lane_cnt), or to false otherwise.
void sve_ptrue_lanecnt(PRegister dst, SIMD_RegVariant size, int lane_cnt);

// Generate predicate through whilelo, by comparing ZR with an unsigned
// immediate. rscratch1 will be clobbered.
inline void sve_whilelo_zr_imm(PRegister pd, SIMD_RegVariant size, uint imm) {
@@ -1723,9 +1723,22 @@ def generate(kind, names):
["bic", "__ sve_bic(p10, p7, p9, p11);", "bic\tp10.b, p7/z, p9.b, p11.b"],
["ptest", "__ sve_ptest(p7, p1);", "ptest\tp7, p1.b"],
["ptrue", "__ sve_ptrue(p1, __ B);", "ptrue\tp1.b"],
["ptrue", "__ sve_ptrue(p1, __ B, 0b00001);", "ptrue\tp1.b, vl1"],
["ptrue", "__ sve_ptrue(p1, __ B, 0b00101);", "ptrue\tp1.b, vl5"],
["ptrue", "__ sve_ptrue(p1, __ B, 0b01001);", "ptrue\tp1.b, vl16"],
["ptrue", "__ sve_ptrue(p1, __ B, 0b01101);", "ptrue\tp1.b, vl256"],
["ptrue", "__ sve_ptrue(p2, __ H);", "ptrue\tp2.h"],
["ptrue", "__ sve_ptrue(p2, __ H, 0b00010);", "ptrue\tp2.h, vl2"],
["ptrue", "__ sve_ptrue(p2, __ H, 0b00110);", "ptrue\tp2.h, vl6"],
["ptrue", "__ sve_ptrue(p2, __ H, 0b01010);", "ptrue\tp2.h, vl32"],
["ptrue", "__ sve_ptrue(p3, __ S);", "ptrue\tp3.s"],
["ptrue", "__ sve_ptrue(p3, __ S, 0b00011);", "ptrue\tp3.s, vl3"],
["ptrue", "__ sve_ptrue(p3, __ S, 0b00111);", "ptrue\tp3.s, vl7"],
["ptrue", "__ sve_ptrue(p3, __ S, 0b01011);", "ptrue\tp3.s, vl64"],
["ptrue", "__ sve_ptrue(p4, __ D);", "ptrue\tp4.d"],
["ptrue", "__ sve_ptrue(p4, __ D, 0b00100);", "ptrue\tp4.d, vl4"],
["ptrue", "__ sve_ptrue(p4, __ D, 0b01000);", "ptrue\tp4.d, vl8"],
["ptrue", "__ sve_ptrue(p4, __ D, 0b01100);", "ptrue\tp4.d, vl128"],
["pfalse", "__ sve_pfalse(p7);", "pfalse\tp7.b"],
["uzp1", "__ sve_uzp1(p0, __ B, p0, p1);", "uzp1\tp0.b, p0.b, p1.b"],
["uzp1", "__ sve_uzp1(p0, __ H, p0, p1);", "uzp1\tp0.h, p0.h, p1.h"],

1 comment on commit 6588bed

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on 6588bed Dec 24, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.