Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
8284960: Integration of JEP 426: Vector API (Fourth Incubator)
Co-authored-by: Jatin Bhateja <jbhateja@openjdk.org>
Co-authored-by: Paul Sandoz <psandoz@openjdk.org>
Co-authored-by: Sandhya Viswanathan <sviswanathan@openjdk.org>
Co-authored-by: Smita Kamath <svkamath@openjdk.org>
Co-authored-by: Joshua Zhu <jzhu@openjdk.org>
Co-authored-by: Xiaohong Gong <xgong@openjdk.org>
Co-authored-by: John R Rose <jrose@openjdk.org>
Co-authored-by: Eric Liu <eliu@openjdk.org>
Co-authored-by: Ningsheng Jian <njian@openjdk.org>
Reviewed-by: ngasson, vlivanov, mcimadamore, jlahoda, kvn
  • Loading branch information
9 people committed May 31, 2022
1 parent 171a7cd commit 6f6486e
Show file tree
Hide file tree
Showing 227 changed files with 20,942 additions and 21,214 deletions.
7 changes: 3 additions & 4 deletions src/hotspot/cpu/aarch64/aarch64.ad
Expand Up @@ -2468,6 +2468,9 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
break;
case Op_LoadVectorGather:
case Op_StoreVectorScatter:
case Op_CompressV:
case Op_CompressM:
case Op_ExpandV:
return false;
default:
break;
Expand Down Expand Up @@ -8658,7 +8661,6 @@ instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
//

instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
predicate(UsePopCountInstruction);
match(Set dst (PopCountI src));
effect(TEMP tmp);
ins_cost(INSN_COST * 13);
Expand All @@ -8680,7 +8682,6 @@ instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
%}

instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
predicate(UsePopCountInstruction);
match(Set dst (PopCountI (LoadI mem)));
effect(TEMP tmp);
ins_cost(INSN_COST * 13);
Expand All @@ -8703,7 +8704,6 @@ instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{

// Note: Long.bitCount(long) returns an int.
instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
predicate(UsePopCountInstruction);
match(Set dst (PopCountL src));
effect(TEMP tmp);
ins_cost(INSN_COST * 13);
Expand All @@ -8723,7 +8723,6 @@ instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
%}

instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
predicate(UsePopCountInstruction);
match(Set dst (PopCountL (LoadL mem)));
effect(TEMP tmp);
ins_cost(INSN_COST * 13);
Expand Down
210 changes: 193 additions & 17 deletions src/hotspot/cpu/aarch64/aarch64_neon.ad
Expand Up @@ -5683,39 +5683,87 @@ instruct vround2D_reg(vecX dst, vecX src, immI rmode) %{
ins_pipe(vdop_fp128);
%}

instruct vpopcount4I(vecX dst, vecX src) %{
predicate(UsePopCountInstruction && n->as_Vector()->length() == 4);
instruct vpopcountID(vecD dst, vecD src) %{
predicate(n->as_Vector()->length_in_bytes() < 16);
match(Set dst (PopCountVI src));
format %{
"cnt $dst, $src\t# vector (16B)\n\t"
"uaddlp $dst, $dst\t# vector (16B)\n\t"
"uaddlp $dst, $dst\t# vector (8H)"
ins_cost(3 * INSN_COST);
format %{ "vpopcountI $dst, $src\t# vector (8B/4H/2S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ cnt(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg));
if (bt == T_SHORT || bt == T_INT) {
__ uaddlp(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($dst$$reg));
if (bt == T_INT) {
__ uaddlp(as_FloatRegister($dst$$reg), __ T4H,
as_FloatRegister($dst$$reg));
}
}
%}
ins_pipe(pipe_class_default);
%}

instruct vpopcountIX(vecX dst, vecX src) %{
predicate(n->as_Vector()->length_in_bytes() == 16);
match(Set dst (PopCountVI src));
ins_cost(3 * INSN_COST);
format %{ "vpopcountI $dst, $src\t# vector (16B/8H/4S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ cnt(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg));
if (bt == T_SHORT || bt == T_INT) {
__ uaddlp(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($dst$$reg));
if (bt == T_INT) {
__ uaddlp(as_FloatRegister($dst$$reg), __ T8H,
as_FloatRegister($dst$$reg));
}
}
%}
ins_pipe(pipe_class_default);
%}

// If the PopCountVL is generated by auto-vectorization, the dst basic
// type is T_INT. And once we have unified the type definition for
// Vector API and auto-vectorization, this rule can be merged with
// "vpopcountLX" rule.
instruct vpopcountLD(vecD dst, vecX src) %{
predicate(n->as_Vector()->length_in_bytes() < 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_INT);
match(Set dst (PopCountVL src));
ins_cost(5 * INSN_COST);
format %{ "vpopcountL $dst, $src\t# vector (2S)" %}
ins_encode %{
__ cnt(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg));
__ uaddlp(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($dst$$reg));
__ uaddlp(as_FloatRegister($dst$$reg), __ T8H,
as_FloatRegister($dst$$reg));
__ uaddlp(as_FloatRegister($dst$$reg), __ T4S,
as_FloatRegister($dst$$reg));
__ xtn(as_FloatRegister($dst$$reg), __ T2S,
as_FloatRegister($dst$$reg), __ T2D);
%}
ins_pipe(pipe_class_default);
%}

instruct vpopcount2I(vecD dst, vecD src) %{
predicate(UsePopCountInstruction && n->as_Vector()->length() == 2);
match(Set dst (PopCountVI src));
format %{
"cnt $dst, $src\t# vector (8B)\n\t"
"uaddlp $dst, $dst\t# vector (8B)\n\t"
"uaddlp $dst, $dst\t# vector (4H)"
%}
instruct vpopcountLX(vecX dst, vecX src) %{
predicate(n->as_Vector()->length_in_bytes() == 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
match(Set dst (PopCountVL src));
ins_cost(4 * INSN_COST);
format %{ "vpopcountL $dst, $src\t# vector (2D)" %}
ins_encode %{
__ cnt(as_FloatRegister($dst$$reg), __ T8B,
__ cnt(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg));
__ uaddlp(as_FloatRegister($dst$$reg), __ T8B,
__ uaddlp(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($dst$$reg));
__ uaddlp(as_FloatRegister($dst$$reg), __ T8H,
as_FloatRegister($dst$$reg));
__ uaddlp(as_FloatRegister($dst$$reg), __ T4H,
__ uaddlp(as_FloatRegister($dst$$reg), __ T4S,
as_FloatRegister($dst$$reg));
%}
ins_pipe(pipe_class_default);
Expand Down Expand Up @@ -5921,3 +5969,131 @@ instruct vmask_tolong16B(iRegLNoSp dst, vecX src) %{
%}
ins_pipe(pipe_slow);
%}

//------------------------- CountLeadingZerosV -----------------------------

instruct countLeadingZerosVD(vecD dst, vecD src) %{
predicate(n->as_Vector()->length_in_bytes() == 8);
match(Set dst (CountLeadingZerosV src));
ins_cost(INSN_COST);
format %{ "countLeadingZerosV $dst, $src\t# vector (8B/4H/2S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_Arrangement size = __ esize2arrangement((unsigned)type2aelembytes(bt), false);
__ clz(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct countLeadingZerosVX(vecX dst, vecX src) %{
predicate(n->as_Vector()->length_in_bytes() == 16);
match(Set dst (CountLeadingZerosV src));
ins_cost(INSN_COST);
format %{ "countLeadingZerosV $dst, $src\t# vector (16B/8H/4S/2D)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_Arrangement size = __ esize2arrangement((unsigned)type2aelembytes(bt), true);
if (bt != T_LONG) {
__ clz(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg));
} else {
__ umov(rscratch1, as_FloatRegister($src$$reg), __ D, 0);
__ clz(rscratch1, rscratch1);
__ mov(as_FloatRegister($dst$$reg), __ D, 0, rscratch1);
__ umov(rscratch1, as_FloatRegister($src$$reg), __ D, 1);
__ clz(rscratch1, rscratch1);
__ mov(as_FloatRegister($dst$$reg), __ D, 1, rscratch1);
}
%}
ins_pipe(pipe_slow);
%}

//------------------------- CountTrailingZerosV ----------------------------

instruct countTrailingZerosVD(vecD dst, vecD src) %{
predicate(n->as_Vector()->length_in_bytes() == 8);
match(Set dst (CountTrailingZerosV src));
ins_cost(3 * INSN_COST);
format %{ "countTrailingZerosV $dst, $src\t# vector (8B/4H/2S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_Arrangement size = __ esize2arrangement((unsigned)type2aelembytes(bt), false);
__ neon_reverse_bits(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, false);
__ clz(as_FloatRegister($dst$$reg), size, as_FloatRegister($dst$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct countTrailingZerosVX(vecX dst, vecX src) %{
predicate(n->as_Vector()->length_in_bytes() == 16);
match(Set dst (CountTrailingZerosV src));
ins_cost(3 * INSN_COST);
format %{ "countTrailingZerosV $dst, $src\t# vector (16B/8H/4S/2D)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_Arrangement size = __ esize2arrangement((unsigned)type2aelembytes(bt), true);
__ neon_reverse_bits(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, true);
if (bt != T_LONG) {
__ clz(as_FloatRegister($dst$$reg), size, as_FloatRegister($dst$$reg));
} else {
__ umov(rscratch1, as_FloatRegister($dst$$reg), __ D, 0);
__ clz(rscratch1, rscratch1);
__ mov(as_FloatRegister($dst$$reg), __ D, 0, rscratch1);
__ umov(rscratch1, as_FloatRegister($dst$$reg), __ D, 1);
__ clz(rscratch1, rscratch1);
__ mov(as_FloatRegister($dst$$reg), __ D, 1, rscratch1);
}
%}
ins_pipe(pipe_slow);
%}

//------------------------------ ReverseV -----------------------------------

instruct vreverseD(vecD dst, vecD src) %{
predicate(n->as_Vector()->length_in_bytes() == 8);
match(Set dst (ReverseV src));
ins_cost(2 * INSN_COST);
format %{ "ReverseV $dst, $src\t# vector (D)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ neon_reverse_bits(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, false);
%}
ins_pipe(pipe_slow);
%}

instruct vreverseX(vecX dst, vecX src) %{
predicate(n->as_Vector()->length_in_bytes() == 16);
match(Set dst (ReverseV src));
ins_cost(2 * INSN_COST);
format %{ "ReverseV $dst, $src\t# vector (X)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ neon_reverse_bits(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, true);
%}
ins_pipe(pipe_slow);
%}

//---------------------------- ReverseBytesV --------------------------------

instruct vreverseBytesD(vecD dst, vecD src) %{
predicate(n->as_Vector()->length_in_bytes() == 8);
match(Set dst (ReverseBytesV src));
ins_cost(INSN_COST);
format %{ "ReverseBytesV $dst, $src\t# vector (D)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ neon_reverse_bytes(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, false);
%}
ins_pipe(pipe_slow);
%}

instruct vreverseBytesX(vecX dst, vecX src) %{
predicate(n->as_Vector()->length_in_bytes() == 16);
match(Set dst (ReverseBytesV src));
ins_cost(INSN_COST);
format %{ "ReverseBytesV $dst, $src\t# vector (X)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ neon_reverse_bytes(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), bt, true);
%}
ins_pipe(pipe_slow);
%}

1 comment on commit 6f6486e

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.