Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
8264469: Add Insert float nodes implementation for Arm SVE 
Co-authored-by: Wang Huang <whuang@openjdk.org>
Co-authored-by: Ai Jiaming <aijiaming1@huawei.com>
Co-authored-by: He Xuejin <hexuejin2@huawei.com>
Reviewed-by: njian, xgong
  • Loading branch information
3 people committed Apr 7, 2021
1 parent cc3f2ea commit b6a5e16
Show file tree
Hide file tree
Showing 3 changed files with 284 additions and 58 deletions.
252 changes: 207 additions & 45 deletions src/hotspot/cpu/aarch64/aarch64_sve.ad
View file
Expand Up @@ -3171,25 +3171,193 @@ instruct vtest_anytrue_partial(iRegINoSp dst, vReg src1, vReg src2, pRegGov pTmp

// ------------------------------ Vector insert ---------------------------------

instruct insertB(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp, vReg tmp2, pRegGov pTmp, rFlagsReg cr)
instruct insertB_small(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp, pRegGov pTmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP pTmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $tmp, B, -16, 1\n\t"
"sve_cmpeq $pTmp, $tmp, ($idx-#16) // shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (B)" %}
ins_encode %{
__ sve_index(as_FloatRegister($tmp$$reg), __ B, -16, 1);
__ sve_cmpeq(as_PRegister($pTmp$$reg), __ B, ptrue,
as_FloatRegister($tmp$$reg), (int)($idx$$constant) - 16);
// If src and dst are the same reg, this move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg),
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), __ B,
as_PRegister($pTmp$$reg), as_Register($val$$reg));

%}
ins_pipe(pipe_slow);
%}

instruct insertS_small(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp, pRegGov pTmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP pTmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $tmp, H, -16, 1\n\t"
"sve_cmpeq $pTmp, $tmp, ($idx-#16) // shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (S)" %}
ins_encode %{
__ sve_index(as_FloatRegister($tmp$$reg), __ H, -16, 1);
__ sve_cmpeq(as_PRegister($pTmp$$reg), __ H, ptrue,
as_FloatRegister($tmp$$reg), (int)($idx$$constant) - 16);
// If src and dst are the same reg, this move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg),
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), __ H,
as_PRegister($pTmp$$reg), as_Register($val$$reg));

%}
ins_pipe(pipe_slow);
%}

instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp, pRegGov pTmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
n->bottom_type()->is_vect()->element_basic_type() == T_INT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP pTmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $tmp, S, -16, 1\n\t"
"sve_cmpeq $pTmp, $tmp, ($idx-#16) // shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (I)" %}
ins_encode %{
__ sve_index(as_FloatRegister($tmp$$reg), __ S, -16, 1);
__ sve_cmpeq(as_PRegister($pTmp$$reg), __ S, ptrue,
as_FloatRegister($tmp$$reg), (int)($idx$$constant) - 16);
// If src and dst are the same reg, this move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg),
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), __ S,
as_PRegister($pTmp$$reg), as_Register($val$$reg));

%}
ins_pipe(pipe_slow);
%}

instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, vReg tmp, pRegGov pTmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP pTmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $tmp, S, -16, 1\n\t"
"sve_cmpeq $pTmp, $tmp, ($idx-#16) // shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (F)" %}
ins_encode %{
__ sve_index(as_FloatRegister($tmp$$reg), __ S, -16, 1);
__ sve_cmpeq(as_PRegister($pTmp$$reg), __ S, ptrue,
as_FloatRegister($tmp$$reg), (int)($idx$$constant) - 16);
// If src and dst are the same reg, this move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg),
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), __ S,
as_PRegister($pTmp$$reg), as_FloatRegister($val$$reg));

%}
ins_pipe(pipe_slow);
%}


instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg tmp, pRegGov pTmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 &&
n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP pTmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $tmp, D, -16, 1\n\t"
"sve_cmpeq $pTmp, $tmp, ($idx-#16) // shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (L)" %}
ins_encode %{
__ sve_index(as_FloatRegister($tmp$$reg), __ D, -16, 1);
__ sve_cmpeq(as_PRegister($pTmp$$reg), __ D, ptrue,
as_FloatRegister($tmp$$reg), (int)($idx$$constant) - 16);
// If src and dst are the same reg, this move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg),
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), __ D,
as_PRegister($pTmp$$reg), as_Register($val$$reg));

%}
ins_pipe(pipe_slow);
%}

instruct insertD(vReg dst, vReg src, vRegD val, immI idx, vReg tmp, pRegGov pTmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 &&
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP pTmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $tmp, D, -16, 1\n\t"
"sve_cmpeq $pTmp, $tmp, ($idx-#16) // shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (D)" %}
ins_encode %{
__ sve_index(as_FloatRegister($tmp$$reg), __ D, -16, 1);
__ sve_cmpeq(as_PRegister($pTmp$$reg), __ D, ptrue,
as_FloatRegister($tmp$$reg), (int)($idx$$constant) - 16);
// If src and dst are the same reg, this move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg),
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), __ D,
as_PRegister($pTmp$$reg), as_FloatRegister($val$$reg));

%}
ins_pipe(pipe_slow);
%}


instruct insertB(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, vReg tmp2, pRegGov pTmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP tmp2, TEMP pTmp, KILL cr);
effect(TEMP tmp1, TEMP tmp2, TEMP pTmp, KILL cr);
ins_cost(5 * SVE_COST);
format %{ "sve_index $tmp, B, 0, 1\n\t"
format %{ "sve_index $tmp1, B, 0, 1\n\t"
"sve_dup $tmp2, B, $idx\n\t"
"sve_cmpeq $pTmp, $tmp, $tmp2\n\t"
"sve_cmpeq $pTmp, $tmp1, $tmp2\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (B)" %}
ins_encode %{
Assembler::SIMD_RegVariant size =
elemType_to_regVariant(vector_element_basic_type(this));
__ sve_index(as_FloatRegister($tmp$$reg), __ B, 0, 1);
__ sve_index(as_FloatRegister($tmp1$$reg), __ B, 0, 1);
__ sve_dup(as_FloatRegister($tmp2$$reg), __ B, (int)($idx$$constant));
__ sve_cmpeq(as_PRegister($pTmp$$reg), size, ptrue,
as_FloatRegister($tmp$$reg), as_FloatRegister($tmp2$$reg));
__ sve_cmpeq(as_PRegister($pTmp$$reg), __ B, ptrue,
as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg));
// If src and dst are the same reg, this move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg),
Expand All @@ -3203,25 +3371,23 @@ instruct insertB(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp, vReg tm
ins_pipe(pipe_slow);
%}

instruct insertS(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp, vReg tmp2, pRegGov pTmp, rFlagsReg cr)
instruct insertS(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, vReg tmp2, pRegGov pTmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 &&
predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP tmp2, TEMP pTmp, KILL cr);
effect(TEMP tmp1, TEMP tmp2, TEMP pTmp, KILL cr);
ins_cost(5 * SVE_COST);
format %{ "sve_index $tmp, H, 0, 1\n\t"
format %{ "sve_index $tmp1, H, 0, 1\n\t"
"sve_dup $tmp2, H, $idx\n\t"
"sve_cmpeq $pTmp, $tmp, $tmp2\n\t"
"sve_cmpeq $pTmp, $tmp1, $tmp2\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (S)" %}
ins_encode %{
Assembler::SIMD_RegVariant size =
elemType_to_regVariant(vector_element_basic_type(this));
__ sve_index(as_FloatRegister($tmp$$reg), __ H, 0, 1);
__ sve_index(as_FloatRegister($tmp1$$reg), __ H, 0, 1);
__ sve_dup(as_FloatRegister($tmp2$$reg), __ H, (int)($idx$$constant));
__ sve_cmpeq(as_PRegister($pTmp$$reg), size, ptrue,
as_FloatRegister($tmp$$reg), as_FloatRegister($tmp2$$reg));
__ sve_cmpeq(as_PRegister($pTmp$$reg), __ H, ptrue,
as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg));
// If src and dst are the same reg, this move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg),
Expand All @@ -3235,25 +3401,23 @@ instruct insertS(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp, vReg tm
ins_pipe(pipe_slow);
%}

instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp, vReg tmp2, pRegGov pTmp, rFlagsReg cr)
instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, vReg tmp2, pRegGov pTmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 &&
predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
n->bottom_type()->is_vect()->element_basic_type() == T_INT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP tmp2, TEMP pTmp, KILL cr);
effect(TEMP tmp1, TEMP tmp2, TEMP pTmp, KILL cr);
ins_cost(5 * SVE_COST);
format %{ "sve_index $tmp, S, 0, 1\n\t"
format %{ "sve_index $tmp1, S, 0, 1\n\t"
"sve_dup $tmp2, S, $idx\n\t"
"sve_cmpeq $pTmp, $tmp, $tmp2\n\t"
"sve_cmpeq $pTmp, $tmp1, $tmp2\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (I)" %}
ins_encode %{
Assembler::SIMD_RegVariant size =
elemType_to_regVariant(vector_element_basic_type(this));
__ sve_index(as_FloatRegister($tmp$$reg), __ S, 0, 1);
__ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
__ sve_dup(as_FloatRegister($tmp2$$reg), __ S, (int)($idx$$constant));
__ sve_cmpeq(as_PRegister($pTmp$$reg), size, ptrue,
as_FloatRegister($tmp$$reg), as_FloatRegister($tmp2$$reg));
__ sve_cmpeq(as_PRegister($pTmp$$reg), __ S, ptrue,
as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg));
// If src and dst are the same reg, this move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg),
Expand All @@ -3267,33 +3431,31 @@ instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp, vReg tm
ins_pipe(pipe_slow);
%}

instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg tmp, vReg tmp2, pRegGov pTmp, rFlagsReg cr)
instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, vReg tmp2, pRegGov pTmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 &&
n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP tmp, TEMP tmp2, TEMP pTmp, KILL cr);
effect(TEMP tmp1, TEMP tmp2, TEMP pTmp, KILL cr);
ins_cost(5 * SVE_COST);
format %{ "sve_index $tmp, D, 0, 1\n\t"
"sve_dup $tmp2, D, $idx\n\t"
"sve_cmpeq $pTmp, $tmp, $tmp2\n\t"
format %{ "sve_index $tmp1, S, 0, 1\n\t"
"sve_dup $tmp2, S, $idx\n\t"
"sve_cmpeq $pTmp, $tmp1, $tmp2\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (L)" %}
"sve_cpy $dst, $pTmp, $val\n\t# insert into vector (F)" %}
ins_encode %{
Assembler::SIMD_RegVariant size =
elemType_to_regVariant(vector_element_basic_type(this));
__ sve_index(as_FloatRegister($tmp$$reg), __ D, 0, 1);
__ sve_dup(as_FloatRegister($tmp2$$reg), __ D, (int)($idx$$constant));
__ sve_cmpeq(as_PRegister($pTmp$$reg), size, ptrue,
as_FloatRegister($tmp$$reg), as_FloatRegister($tmp2$$reg));
__ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
__ sve_dup(as_FloatRegister($tmp2$$reg), __ S, (int)($idx$$constant));
__ sve_cmpeq(as_PRegister($pTmp$$reg), __ S, ptrue,
as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg));
// If src and dst are the same reg, this move is not needed.
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg),
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), __ D,
as_PRegister($pTmp$$reg), as_Register($val$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ S,
as_PRegister($pTmp$$reg), as_FloatRegister($val$$reg));

%}
ins_pipe(pipe_slow);
Expand Down

0 comments on commit b6a5e16

Please sign in to comment.