Skip to content
This repository has been archived by the owner. It is now read-only.

8278889: AArch64: [vectorapi] VectorMaskLoadStoreTest.testMaskCast() test fail #49

Closed
wants to merge 2 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -411,21 +411,22 @@ instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, r
ins_pipe(pipe_slow);
%}

// maskAll
// maskAll (full or partial predicate size)

instruct vmaskAll_immI(pRegGov dst, immI src) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
ins_cost(SVE_COST);
format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
ins_encode %{
int con = (int)$src$$constant;
if (con == 0) {
__ sve_pfalse(as_PRegister($dst$$reg));
} else {
assert(con == -1, "invalid constant value for mask");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
}
%}
ins_pipe(pipe_slow);
@@ -435,14 +436,22 @@ instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
effect(TEMP tmp, KILL cr);
ins_cost(2 * SVE_COST);
ins_cost(3 * SVE_COST);
format %{ "sve_dup $tmp, $src\n\t"
"sve_cmpne $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
"sve_ptrue_lanecnt $dst\n\t"
"sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
if (length_in_bytes < MaxVectorSize) {
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
} else {
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
}
%}
ins_pipe(pipe_slow);
%}
@@ -451,15 +460,16 @@ instruct vmaskAll_immL(pRegGov dst, immL src) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
ins_cost(SVE_COST);
format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) (D)" %}
format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (D)" %}
ins_encode %{
long con = (long)$src$$constant;
if (con == 0) {
__ sve_pfalse(as_PRegister($dst$$reg));
} else {
assert(con == -1, "invalid constant value for mask");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
}
%}
ins_pipe(pipe_slow);
@@ -469,14 +479,22 @@ instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
effect(TEMP tmp, KILL cr);
ins_cost(2 * SVE_COST);
ins_cost(3 * SVE_COST);
format %{ "sve_dup $tmp, $src\n\t"
"sve_cmpne $dst, $tmp, 0\t# mask all (sve) (D)" %}
"sve_ptrue_lanecnt $dst\n\t"
"sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (D)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
if (length_in_bytes < MaxVectorSize) {
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
} else {
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
}
%}
ins_pipe(pipe_slow);
%}
@@ -3084,6 +3102,7 @@ instruct reduce_maxF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
match(Set dst (MaxReductionV (Binary src1 src2) pg));
ins_cost(SVE_COST);
effect(TEMP_DEF dst);
format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated (sve)" %}
ins_encode %{
__ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
@@ -3098,6 +3117,7 @@ instruct reduce_maxD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
match(Set dst (MaxReductionV (Binary src1 src2) pg));
ins_cost(SVE_COST);
effect(TEMP_DEF dst);
format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated (sve)" %}
ins_encode %{
__ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
@@ -3380,6 +3400,7 @@ instruct reduce_minF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
match(Set dst (MinReductionV (Binary src1 src2) pg));
ins_cost(SVE_COST);
effect(TEMP_DEF dst);
format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated (sve)" %}
ins_encode %{
__ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
@@ -3394,6 +3415,7 @@ instruct reduce_minD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
match(Set dst (MinReductionV (Binary src1 src2) pg));
ins_cost(SVE_COST);
effect(TEMP_DEF dst);
format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated (sve)" %}
ins_encode %{
__ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
@@ -356,15 +356,16 @@ instruct vmaskAll_imm$1(pRegGov dst, imm$1 src) %{
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
ins_cost(SVE_COST);
format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) ($2)" %}
format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) ($2)" %}
ins_encode %{
ifelse($1, `I', int, long) con = (ifelse($1, `I', int, long))$src$$constant;
if (con == 0) {
__ sve_pfalse(as_PRegister($dst$$reg));
} else {
assert(con == -1, "invalid constant value for mask");
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
}
%}
ins_pipe(pipe_slow);
@@ -377,19 +378,27 @@ instruct vmaskAll$1(pRegGov dst, ifelse($1, `I', iRegIorL2I, iRegL) src, vReg tm
predicate(UseSVE > 0);
match(Set dst (MaskAll src));
effect(TEMP tmp, KILL cr);
ins_cost(2 * SVE_COST);
ins_cost(3 * SVE_COST);
format %{ "sve_dup $tmp, $src\n\t"
"sve_cmpne $dst, $tmp, 0\t# mask all (sve) ($2)" %}
"sve_ptrue_lanecnt $dst\n\t"
"sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) ($2)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
if (length_in_bytes < MaxVectorSize) {
__ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
} else {
__ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// maskAll
// maskAll (full or partial predicate size)
MASKALL_IMM(I, B/H/S)
MASKALL(I, B/H/S)
MASKALL_IMM(L, D)
@@ -1807,6 +1816,7 @@ instruct reduce_$1$2_masked($5 dst, $5 src1, vReg src2, pRegGov pg) %{
n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
ins_cost(SVE_COST);
effect(TEMP_DEF dst);
format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated (sve)" %}
ins_encode %{
__ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
@@ -1232,3 +1232,39 @@ void C2_MacroAssembler::sve_reduce_integral(int opc, Register dst, BasicType bt,
}
}
}

// Set elements of the dst predicate to true if the element number is
// in the range of [0, lane_cnt), or to false otherwise.
void C2_MacroAssembler::sve_ptrue_lanecnt(PRegister dst, SIMD_RegVariant size, int lane_cnt) {
assert(size != Q, "invalid size");
switch(lane_cnt) {
case 1: /* VL1 */
case 2: /* VL2 */
case 3: /* VL3 */
case 4: /* VL4 */
case 5: /* VL5 */
case 6: /* VL6 */
case 7: /* VL7 */
case 8: /* VL8 */
sve_ptrue(dst, size, lane_cnt);
break;
case 16:
sve_ptrue(dst, size, /* VL16 */ 0b01001);
break;
case 32:
sve_ptrue(dst, size, /* VL32 */ 0b01010);
break;
case 64:
sve_ptrue(dst, size, /* VL64 */ 0b01011);
break;
case 128:
sve_ptrue(dst, size, /* VL128 */ 0b01100);
break;
case 256:
sve_ptrue(dst, size, /* VL256 */ 0b01101);
break;
default:
assert(false, "unsupported");
ShouldNotReachHere();
}
}
@@ -88,6 +88,10 @@
void sve_reduce_integral(int opc, Register dst, BasicType bt, Register src1,
FloatRegister src2, PRegister pg, FloatRegister tmp);

// Set elements of the dst predicate to true if the element number is
// in the range of [0, lane_cnt), or to false otherwise.
void sve_ptrue_lanecnt(PRegister dst, SIMD_RegVariant size, int lane_cnt);

// Generate predicate through whilelo, by comparing ZR with an unsigned
// immediate. rscratch1 will be clobbered.
inline void sve_whilelo_zr_imm(PRegister pd, SIMD_RegVariant size, uint imm) {
@@ -1723,9 +1723,22 @@ def generate(kind, names):
["bic", "__ sve_bic(p10, p7, p9, p11);", "bic\tp10.b, p7/z, p9.b, p11.b"],
["ptest", "__ sve_ptest(p7, p1);", "ptest\tp7, p1.b"],
["ptrue", "__ sve_ptrue(p1, __ B);", "ptrue\tp1.b"],
["ptrue", "__ sve_ptrue(p1, __ B, 0b00001);", "ptrue\tp1.b, vl1"],
["ptrue", "__ sve_ptrue(p1, __ B, 0b00101);", "ptrue\tp1.b, vl5"],
["ptrue", "__ sve_ptrue(p1, __ B, 0b01001);", "ptrue\tp1.b, vl16"],
["ptrue", "__ sve_ptrue(p1, __ B, 0b01101);", "ptrue\tp1.b, vl256"],
["ptrue", "__ sve_ptrue(p2, __ H);", "ptrue\tp2.h"],
["ptrue", "__ sve_ptrue(p2, __ H, 0b00010);", "ptrue\tp2.h, vl2"],
["ptrue", "__ sve_ptrue(p2, __ H, 0b00110);", "ptrue\tp2.h, vl6"],
["ptrue", "__ sve_ptrue(p2, __ H, 0b01010);", "ptrue\tp2.h, vl32"],
["ptrue", "__ sve_ptrue(p3, __ S);", "ptrue\tp3.s"],
["ptrue", "__ sve_ptrue(p3, __ S, 0b00011);", "ptrue\tp3.s, vl3"],
["ptrue", "__ sve_ptrue(p3, __ S, 0b00111);", "ptrue\tp3.s, vl7"],
["ptrue", "__ sve_ptrue(p3, __ S, 0b01011);", "ptrue\tp3.s, vl64"],
["ptrue", "__ sve_ptrue(p4, __ D);", "ptrue\tp4.d"],
["ptrue", "__ sve_ptrue(p4, __ D, 0b00100);", "ptrue\tp4.d, vl4"],
["ptrue", "__ sve_ptrue(p4, __ D, 0b01000);", "ptrue\tp4.d, vl8"],
["ptrue", "__ sve_ptrue(p4, __ D, 0b01100);", "ptrue\tp4.d, vl128"],
["pfalse", "__ sve_pfalse(p7);", "pfalse\tp7.b"],
["uzp1", "__ sve_uzp1(p0, __ B, p0, p1);", "uzp1\tp0.b, p0.b, p1.b"],
["uzp1", "__ sve_uzp1(p0, __ H, p0, p1);", "uzp1\tp0.h, p0.h, p1.h"],