Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8266286: Add LoadVectorGather and StoreVectorScatter's partial version #88

Closed
Closed
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -228,10 +228,6 @@ source %{
case Op_ExtractUB:
return false;
// Vector API specific
case Op_LoadVectorGather:
case Op_StoreVectorScatter:
// Partial size of gather/scatter are not supported for now.
return length_in_bytes == MaxVectorSize;
case Op_VectorLoadShuffle:
case Op_VectorRearrange:
if (vlen < 4) {
@@ -4111,6 +4107,7 @@ instruct rearrangeL(vReg dst, vReg src, vReg shuffle)

instruct gatherI(vReg dst, indirect mem, vReg idx) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
match(Set dst (LoadVectorGather mem idx));
@@ -4124,6 +4121,7 @@ instruct gatherI(vReg dst, indirect mem, vReg idx) %{

instruct gatherL(vReg dst, indirect mem, vReg idx) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set dst (LoadVectorGather mem idx));
@@ -4141,6 +4139,7 @@ instruct gatherL(vReg dst, indirect mem, vReg idx) %{

instruct scatterI(indirect mem, vReg src, vReg idx) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
@@ -4154,6 +4153,7 @@ instruct scatterI(indirect mem, vReg src, vReg idx) %{

instruct scatterL(indirect mem, vReg src, vReg idx) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
@@ -4167,6 +4167,90 @@ instruct scatterL(indirect mem, vReg src, vReg idx) %{
ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Load Gather Partial-------------------------------
instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
Wanghuang-Huawei marked this conversation as resolved.
Show resolved Hide resolved
match(Set dst (LoadVectorGather mem idx));
effect(TEMP pTmp, KILL cr);
ins_cost(2 * SVE_COST + INSN_COST);
format %{ "mov rscratch1, vector_length\n\t"
"sve_whilelo $pTmp, zr, rscratch1\n\t"
"load_vector_gather $dst, $pTmp, $mem, $idx\t# vector load gather partial (I/F)" %}
ins_encode %{
__ mov(rscratch1, vector_length(this));
__ sve_whilelo(as_PRegister($pTmp$$reg), __ S, zr, rscratch1);
__ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pTmp$$reg), as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set dst (LoadVectorGather mem idx));
effect(TEMP pTmp, KILL cr);
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "mov rscratch1, vector_length\n\t"
"sve_whilelo $pTmp, zr, rscratch1\n\t"
"sve_uunpklo $idx, $idx\n\t"
"load_vector_gather $dst, $pTmp, $mem, $idx\t# vector load gather partial (L/D)" %}
ins_encode %{
__ mov(rscratch1, vector_length(this));
__ sve_whilelo(as_PRegister($pTmp$$reg), __ D, zr, rscratch1);
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pTmp$$reg), as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Store Scatter Partial-------------------------------

instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
effect(TEMP pTmp, KILL cr);
ins_cost(2 * SVE_COST + INSN_COST);
format %{ "mov rscratch1, vector_length\n\t"
"sve_whilelo $pTmp, zr, rscratch1\n\t"
"store_vector_scatter $mem, $pTmp, $idx, $src\t# vector store scatter partial (I/F)" %}
ins_encode %{
__ mov(rscratch1, vector_length(this, $src));
__ sve_whilelo(as_PRegister($pTmp$$reg), __ S, zr, rscratch1);
__ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pTmp$$reg), as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
effect(TEMP pTmp, KILL cr);
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "mov rscratch1, vector_length\n\t"
"sve_whilelo $pTmp, zr, rscratch1\n\t"
"sve_uunpklo $idx, $idx\n\t"
"store_vector_scatter $mem, $pTmp, $idx, $src\t# vector store scatter partial (L/D)" %}
ins_encode %{
__ mov(rscratch1, vector_length(this, $src));
__ sve_whilelo(as_PRegister($pTmp$$reg), __ D, zr, rscratch1);
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pTmp$$reg), as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
ins_pipe(pipe_slow);
%}


// ------------------------------ Vector Load Const -------------------------------

instruct loadconB(vReg dst, immI0 src) %{
@@ -223,10 +223,6 @@ source %{
case Op_ExtractUB:
return false;
// Vector API specific
case Op_LoadVectorGather:
case Op_StoreVectorScatter:
// Partial size of gather/scatter are not supported for now.
return length_in_bytes == MaxVectorSize;
case Op_VectorLoadShuffle:
case Op_VectorRearrange:
if (vlen < 4) {
@@ -2427,6 +2423,7 @@ VECTOR_REARRANGE(L, 8, D)

instruct gatherI(vReg dst, indirect mem, vReg idx) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
match(Set dst (LoadVectorGather mem idx));
@@ -2440,6 +2437,7 @@ instruct gatherI(vReg dst, indirect mem, vReg idx) %{

instruct gatherL(vReg dst, indirect mem, vReg idx) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set dst (LoadVectorGather mem idx));
@@ -2457,6 +2455,7 @@ instruct gatherL(vReg dst, indirect mem, vReg idx) %{

instruct scatterI(indirect mem, vReg src, vReg idx) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
@@ -2470,6 +2469,7 @@ instruct scatterI(indirect mem, vReg src, vReg idx) %{

instruct scatterL(indirect mem, vReg src, vReg idx) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
@@ -2483,6 +2483,90 @@ instruct scatterL(indirect mem, vReg src, vReg idx) %{
ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Load Gather Partial-------------------------------
instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
match(Set dst (LoadVectorGather mem idx));
effect(TEMP pTmp, KILL cr);
ins_cost(2 * SVE_COST + INSN_COST);
format %{ "mov rscratch1, vector_length\n\t"
"sve_whilelo $pTmp, zr, rscratch1\n\t"
"load_vector_gather $dst, $pTmp, $mem, $idx\t# vector load gather partial (I/F)" %}
ins_encode %{
__ mov(rscratch1, vector_length(this));
__ sve_whilelo(as_PRegister($pTmp$$reg), __ S, zr, rscratch1);
__ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pTmp$$reg), as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
(n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set dst (LoadVectorGather mem idx));
effect(TEMP pTmp, KILL cr);
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "mov rscratch1, vector_length\n\t"
"sve_whilelo $pTmp, zr, rscratch1\n\t"
"sve_uunpklo $idx, $idx\n\t"
"load_vector_gather $dst, $pTmp, $mem, $idx\t# vector load gather partial (L/D)" %}
ins_encode %{
__ mov(rscratch1, vector_length(this));
__ sve_whilelo(as_PRegister($pTmp$$reg), __ D, zr, rscratch1);
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pTmp$$reg), as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Store Scatter Partial-------------------------------

instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
effect(TEMP pTmp, KILL cr);
ins_cost(2 * SVE_COST + INSN_COST);
format %{ "mov rscratch1, vector_length\n\t"
"sve_whilelo $pTmp, zr, rscratch1\n\t"
"store_vector_scatter $mem, $pTmp, $idx, $src\t# vector store scatter partial (I/F)" %}
ins_encode %{
__ mov(rscratch1, vector_length(this, $src));
__ sve_whilelo(as_PRegister($pTmp$$reg), __ S, zr, rscratch1);
__ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pTmp$$reg), as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
(n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
match(Set mem (StoreVectorScatter mem (Binary src idx)));
effect(TEMP pTmp, KILL cr);
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "mov rscratch1, vector_length\n\t"
"sve_whilelo $pTmp, zr, rscratch1\n\t"
"sve_uunpklo $idx, $idx\n\t"
"store_vector_scatter $mem, $pTmp, $idx, $src\t# vector store scatter partial (L/D)" %}
ins_encode %{
__ mov(rscratch1, vector_length(this, $src));
__ sve_whilelo(as_PRegister($pTmp$$reg), __ D, zr, rscratch1);
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pTmp$$reg), as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
ins_pipe(pipe_slow);
%}


// ------------------------------ Vector Load Const -------------------------------

instruct loadconB(vReg dst, immI0 src) %{