Skip to content
This repository has been archived by the owner on Sep 19, 2023. It is now read-only.
/ jdk19 Public archive

Commit

Permalink
8288397: AArch64: Fix register issues in SVE backend match rules
Browse files Browse the repository at this point in the history
Reviewed-by: njian, ngasson
  • Loading branch information
Xiaohong Gong committed Jun 20, 2022
1 parent f12d044 commit ae030bc
Show file tree
Hide file tree
Showing 4 changed files with 334 additions and 60 deletions.
68 changes: 36 additions & 32 deletions src/hotspot/cpu/aarch64/aarch64_sve.ad
Original file line number Diff line number Diff line change
Expand Up @@ -1163,30 +1163,30 @@ instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{

// vector not - predicated

instruct vnotI_masked(vReg dst, vReg src, immI_M1 m1, pRegGov pg) %{
instruct vnotI_masked(vReg dst_src, immI_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst (XorV (Binary src (ReplicateB m1)) pg));
match(Set dst (XorV (Binary src (ReplicateS m1)) pg));
match(Set dst (XorV (Binary src (ReplicateI m1)) pg));
match(Set dst_src (XorV (Binary dst_src (ReplicateB m1)) pg));
match(Set dst_src (XorV (Binary dst_src (ReplicateS m1)) pg));
match(Set dst_src (XorV (Binary dst_src (ReplicateI m1)) pg));
ins_cost(SVE_COST);
format %{ "sve_not $dst, $pg, $src\t# vector (sve) B/H/S" %}
format %{ "sve_not $dst_src, $pg, $dst_src\t# vector (sve) B/H/S" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
__ sve_not(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct vnotL_masked(vReg dst, vReg src, immL_M1 m1, pRegGov pg) %{
instruct vnotL_masked(vReg dst_src, immL_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst (XorV (Binary src (ReplicateL m1)) pg));
match(Set dst_src (XorV (Binary dst_src (ReplicateL m1)) pg));
ins_cost(SVE_COST);
format %{ "sve_not $dst, $pg, $src\t# vector (sve) D" %}
format %{ "sve_not $dst_src, $pg, $dst_src\t# vector (sve) D" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
__ sve_not(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand Down Expand Up @@ -5254,18 +5254,19 @@ instruct gatherI(vReg dst, indirect mem, vReg idx) %{
ins_pipe(pipe_slow);
%}

instruct gatherL(vReg dst, indirect mem, vReg idx) %{
instruct gatherL(vReg dst, indirect mem, vReg idx, vReg tmp) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set dst (LoadVectorGather mem idx));
effect(TEMP tmp);
ins_cost(2 * SVE_COST);
format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
ins_encode %{
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
as_FloatRegister($idx$$reg));
as_FloatRegister($tmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand All @@ -5289,20 +5290,20 @@ instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsR
ins_pipe(pipe_slow);
%}

instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
instruct gatherL_partial(vReg dst, indirect mem, vReg idx, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set dst (LoadVectorGather mem idx));
effect(TEMP ptmp, KILL cr);
effect(TEMP vtmp, TEMP ptmp, KILL cr);
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
ins_encode %{
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($vtmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
as_Register($mem$$base), as_FloatRegister($vtmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand All @@ -5323,17 +5324,18 @@ instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
ins_pipe(pipe_slow);
%}

instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg, vReg tmp) %{
predicate(UseSVE > 0 &&
(n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
effect(TEMP tmp);
ins_cost(2 * SVE_COST);
format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
ins_encode %{
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
as_Register($mem$$base), as_FloatRegister($tmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand All @@ -5355,18 +5357,19 @@ instruct scatterI(indirect mem, vReg src, vReg idx) %{
ins_pipe(pipe_slow);
%}

instruct scatterL(indirect mem, vReg src, vReg idx) %{
instruct scatterL(indirect mem, vReg src, vReg idx, vReg tmp) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
effect(TEMP tmp);
ins_cost(2 * SVE_COST);
format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
ins_encode %{
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
as_Register($mem$$base), as_FloatRegister($idx$$reg));
as_Register($mem$$base), as_FloatRegister($tmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand All @@ -5390,20 +5393,20 @@ instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlags
ins_pipe(pipe_slow);
%}

instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
instruct scatterL_partial(indirect mem, vReg src, vReg idx, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
effect(TEMP ptmp, KILL cr);
effect(TEMP vtmp, TEMP ptmp, KILL cr);
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
ins_encode %{
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($vtmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
as_Register($mem$$base), as_FloatRegister($vtmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand All @@ -5424,17 +5427,18 @@ instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
ins_pipe(pipe_slow);
%}

instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg, vReg tmp) %{
predicate(UseSVE > 0 &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
effect(TEMP tmp);
ins_cost(2 * SVE_COST);
format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
ins_encode %{
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
as_Register($mem$$base), as_FloatRegister($tmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand Down
60 changes: 32 additions & 28 deletions src/hotspot/cpu/aarch64/aarch64_sve_ad.m4
Original file line number Diff line number Diff line change
Expand Up @@ -657,21 +657,21 @@ dnl
// vector not - predicated
dnl
define(`MATCH_RULE', `ifelse($1, I,
`match(Set dst (XorV (Binary src (ReplicateB m1)) pg));
match(Set dst (XorV (Binary src (ReplicateS m1)) pg));
match(Set dst (XorV (Binary src (ReplicateI m1)) pg));',
`match(Set dst (XorV (Binary src (ReplicateL m1)) pg));')')dnl
`match(Set dst_src (XorV (Binary dst_src (ReplicateB m1)) pg));
match(Set dst_src (XorV (Binary dst_src (ReplicateS m1)) pg));
match(Set dst_src (XorV (Binary dst_src (ReplicateI m1)) pg));',
`match(Set dst_src (XorV (Binary dst_src (ReplicateL m1)) pg));')')dnl
dnl
define(`VECTOR_NOT_PREDICATE', `
instruct vnot$1_masked`'(vReg dst, vReg src, imm$1_M1 m1, pRegGov pg) %{
instruct vnot$1_masked`'(vReg dst_src, imm$1_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
MATCH_RULE($1)
ins_cost(SVE_COST);
format %{ "sve_not $dst, $pg, $src\t# vector (sve) $2" %}
format %{ "sve_not $dst_src, $pg, $dst_src\t# vector (sve) $2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
__ sve_not(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
%}
ins_pipe(pipe_slow);
%}')dnl
Expand Down Expand Up @@ -2824,18 +2824,19 @@ instruct gatherI(vReg dst, indirect mem, vReg idx) %{
ins_pipe(pipe_slow);
%}

instruct gatherL(vReg dst, indirect mem, vReg idx) %{
instruct gatherL(vReg dst, indirect mem, vReg idx, vReg tmp) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set dst (LoadVectorGather mem idx));
effect(TEMP tmp);
ins_cost(2 * SVE_COST);
format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
ins_encode %{
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
as_FloatRegister($idx$$reg));
as_FloatRegister($tmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand All @@ -2859,20 +2860,20 @@ instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsR
ins_pipe(pipe_slow);
%}

instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
instruct gatherL_partial(vReg dst, indirect mem, vReg idx, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set dst (LoadVectorGather mem idx));
effect(TEMP ptmp, KILL cr);
effect(TEMP vtmp, TEMP ptmp, KILL cr);
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
ins_encode %{
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($vtmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
as_Register($mem$$base), as_FloatRegister($vtmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand All @@ -2893,17 +2894,18 @@ instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
ins_pipe(pipe_slow);
%}

instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg, vReg tmp) %{
predicate(UseSVE > 0 &&
(n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
effect(TEMP tmp);
ins_cost(2 * SVE_COST);
format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
ins_encode %{
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
as_Register($mem$$base), as_FloatRegister($tmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand All @@ -2925,18 +2927,19 @@ instruct scatterI(indirect mem, vReg src, vReg idx) %{
ins_pipe(pipe_slow);
%}

instruct scatterL(indirect mem, vReg src, vReg idx) %{
instruct scatterL(indirect mem, vReg src, vReg idx, vReg tmp) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
effect(TEMP tmp);
ins_cost(2 * SVE_COST);
format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
ins_encode %{
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
as_Register($mem$$base), as_FloatRegister($idx$$reg));
as_Register($mem$$base), as_FloatRegister($tmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand All @@ -2960,20 +2963,20 @@ instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlags
ins_pipe(pipe_slow);
%}

instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
instruct scatterL_partial(indirect mem, vReg src, vReg idx, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
effect(TEMP ptmp, KILL cr);
effect(TEMP vtmp, TEMP ptmp, KILL cr);
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
ins_encode %{
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($vtmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
as_Register($mem$$base), as_FloatRegister($vtmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand All @@ -2994,17 +2997,18 @@ instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
ins_pipe(pipe_slow);
%}

instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg, vReg tmp) %{
predicate(UseSVE > 0 &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
effect(TEMP tmp);
ins_cost(2 * SVE_COST);
format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
ins_encode %{
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
as_Register($mem$$base), as_FloatRegister($tmp$$reg));
%}
ins_pipe(pipe_slow);
%}
Expand Down

1 comment on commit ae030bc

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.