Skip to content

Commit 0dca573

Browse files
changpeng1997Xiaohong Gong
changpeng1997
authored and
Xiaohong Gong
committed
8301739: AArch64: Add optimized rules for vector compare with immediate for SVE
Reviewed-by: aph, eliu
1 parent 3d3eaed commit 0dca573

File tree

10 files changed

+997
-357
lines changed

10 files changed

+997
-357
lines changed

‎src/hotspot/cpu/aarch64/aarch64.ad

+56-1
Original file line numberDiff line numberDiff line change
@@ -4318,7 +4318,18 @@ operand immI_positive()
43184318
// BoolTest condition for signed compare
43194319
operand immI_cmp_cond()
43204320
%{
4321-
predicate(n->get_int() < (int)(BoolTest::unsigned_compare));
4321+
predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4322+
match(ConI);
4323+
4324+
op_cost(0);
4325+
format %{ %}
4326+
interface(CONST_INTER);
4327+
%}
4328+
4329+
// BoolTest condition for unsigned compare
4330+
operand immI_cmpU_cond()
4331+
%{
4332+
predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
43224333
match(ConI);
43234334

43244335
op_cost(0);
@@ -4425,6 +4436,28 @@ operand immI19()
44254436
interface(CONST_INTER);
44264437
%}
44274438

4439+
// 5 bit signed integer
4440+
operand immI5()
4441+
%{
4442+
predicate(Assembler::is_simm(n->get_int(), 5));
4443+
match(ConI);
4444+
4445+
op_cost(0);
4446+
format %{ %}
4447+
interface(CONST_INTER);
4448+
%}
4449+
4450+
// 7 bit unsigned integer
4451+
operand immIU7()
4452+
%{
4453+
predicate(Assembler::is_uimm(n->get_int(), 7));
4454+
match(ConI);
4455+
4456+
op_cost(0);
4457+
format %{ %}
4458+
interface(CONST_INTER);
4459+
%}
4460+
44284461
// 12 bit unsigned offset -- for base plus immediate loads
44294462
operand immIU12()
44304463
%{
@@ -4567,6 +4600,28 @@ operand immLoffset16()
45674600
interface(CONST_INTER);
45684601
%}
45694602

4603+
// 5 bit signed long integer
4604+
operand immL5()
4605+
%{
4606+
predicate(Assembler::is_simm(n->get_long(), 5));
4607+
match(ConL);
4608+
4609+
op_cost(0);
4610+
format %{ %}
4611+
interface(CONST_INTER);
4612+
%}
4613+
4614+
// 7 bit unsigned long integer
4615+
operand immLU7()
4616+
%{
4617+
predicate(Assembler::is_uimm(n->get_long(), 7));
4618+
match(ConL);
4619+
4620+
op_cost(0);
4621+
format %{ %}
4622+
interface(CONST_INTER);
4623+
%}
4624+
45704625
// 8 bit signed value.
45714626
operand immI8()
45724627
%{

‎src/hotspot/cpu/aarch64/aarch64_vector.ad

+112
Original file line numberDiff line numberDiff line change
@@ -5220,6 +5220,118 @@ instruct vmaskcmp_sve(pReg dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %
52205220
ins_pipe(pipe_slow);
52215221
%}
52225222

5223+
instruct vmaskcmp_immB_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
5224+
predicate(UseSVE > 0);
5225+
match(Set dst (VectorMaskCmp (Binary src (ReplicateB imm)) cond));
5226+
effect(KILL cr);
5227+
format %{ "vmaskcmp_immB_sve $dst, $src, $imm, $cond\t# KILL cr" %}
5228+
ins_encode %{
5229+
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
5230+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5231+
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
5232+
__ sve_cmp(condition, $dst$$PRegister, __ B, ptrue, $src$$FloatRegister, (int)$imm$$constant);
5233+
%}
5234+
ins_pipe(pipe_slow);
5235+
%}
5236+
5237+
instruct vmaskcmpU_immB_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
5238+
predicate(UseSVE > 0);
5239+
match(Set dst (VectorMaskCmp (Binary src (ReplicateB imm)) cond));
5240+
effect(KILL cr);
5241+
format %{ "vmaskcmpU_immB_sve $dst, $src, $imm, $cond\t# KILL cr" %}
5242+
ins_encode %{
5243+
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
5244+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5245+
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
5246+
__ sve_cmp(condition, $dst$$PRegister, __ B, ptrue, $src$$FloatRegister, (int)$imm$$constant);
5247+
%}
5248+
ins_pipe(pipe_slow);
5249+
%}
5250+
5251+
instruct vmaskcmp_immS_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
5252+
predicate(UseSVE > 0);
5253+
match(Set dst (VectorMaskCmp (Binary src (ReplicateS imm)) cond));
5254+
effect(KILL cr);
5255+
format %{ "vmaskcmp_immS_sve $dst, $src, $imm, $cond\t# KILL cr" %}
5256+
ins_encode %{
5257+
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
5258+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5259+
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
5260+
__ sve_cmp(condition, $dst$$PRegister, __ H, ptrue, $src$$FloatRegister, (int)$imm$$constant);
5261+
%}
5262+
ins_pipe(pipe_slow);
5263+
%}
5264+
5265+
instruct vmaskcmpU_immS_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
5266+
predicate(UseSVE > 0);
5267+
match(Set dst (VectorMaskCmp (Binary src (ReplicateS imm)) cond));
5268+
effect(KILL cr);
5269+
format %{ "vmaskcmpU_immS_sve $dst, $src, $imm, $cond\t# KILL cr" %}
5270+
ins_encode %{
5271+
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
5272+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5273+
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
5274+
__ sve_cmp(condition, $dst$$PRegister, __ H, ptrue, $src$$FloatRegister, (int)$imm$$constant);
5275+
%}
5276+
ins_pipe(pipe_slow);
5277+
%}
5278+
5279+
instruct vmaskcmp_immI_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
5280+
predicate(UseSVE > 0);
5281+
match(Set dst (VectorMaskCmp (Binary src (ReplicateI imm)) cond));
5282+
effect(KILL cr);
5283+
format %{ "vmaskcmp_immI_sve $dst, $src, $imm, $cond\t# KILL cr" %}
5284+
ins_encode %{
5285+
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
5286+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5287+
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
5288+
__ sve_cmp(condition, $dst$$PRegister, __ S, ptrue, $src$$FloatRegister, (int)$imm$$constant);
5289+
%}
5290+
ins_pipe(pipe_slow);
5291+
%}
5292+
5293+
instruct vmaskcmpU_immI_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
5294+
predicate(UseSVE > 0);
5295+
match(Set dst (VectorMaskCmp (Binary src (ReplicateI imm)) cond));
5296+
effect(KILL cr);
5297+
format %{ "vmaskcmpU_immI_sve $dst, $src, $imm, $cond\t# KILL cr" %}
5298+
ins_encode %{
5299+
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
5300+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5301+
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
5302+
__ sve_cmp(condition, $dst$$PRegister, __ S, ptrue, $src$$FloatRegister, (int)$imm$$constant);
5303+
%}
5304+
ins_pipe(pipe_slow);
5305+
%}
5306+
5307+
instruct vmaskcmp_immL_sve(pReg dst, vReg src, immL5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
5308+
predicate(UseSVE > 0);
5309+
match(Set dst (VectorMaskCmp (Binary src (ReplicateL imm)) cond));
5310+
effect(KILL cr);
5311+
format %{ "vmaskcmp_immL_sve $dst, $src, $imm, $cond\t# KILL cr" %}
5312+
ins_encode %{
5313+
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
5314+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5315+
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
5316+
__ sve_cmp(condition, $dst$$PRegister, __ D, ptrue, $src$$FloatRegister, (int)$imm$$constant);
5317+
%}
5318+
ins_pipe(pipe_slow);
5319+
%}
5320+
5321+
instruct vmaskcmpU_immL_sve(pReg dst, vReg src, immLU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
5322+
predicate(UseSVE > 0);
5323+
match(Set dst (VectorMaskCmp (Binary src (ReplicateL imm)) cond));
5324+
effect(KILL cr);
5325+
format %{ "vmaskcmpU_immL_sve $dst, $src, $imm, $cond\t# KILL cr" %}
5326+
ins_encode %{
5327+
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
5328+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5329+
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
5330+
__ sve_cmp(condition, $dst$$PRegister, __ D, ptrue, $src$$FloatRegister, (int)$imm$$constant);
5331+
%}
5332+
ins_pipe(pipe_slow);
5333+
%}
5334+
52235335
instruct vmaskcmp_masked(pReg dst, vReg src1, vReg src2, immI cond,
52245336
pRegGov pg, rFlagsReg cr) %{
52255337
predicate(UseSVE > 0);

‎src/hotspot/cpu/aarch64/aarch64_vector_ad.m4

+25
Original file line numberDiff line numberDiff line change
@@ -3616,6 +3616,31 @@ instruct vmaskcmp_sve(pReg dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %
36163616
%}
36173617
ins_pipe(pipe_slow);
36183618
%}
3619+
dnl
3620+
dnl VMASKCMP_SVE_IMM($1 , $2 , $3 , $4 )
3621+
dnl VMASKCMP_SVE_IMM(element_size, element_type, type_imm, type_condition)
3622+
define(`VMASKCMP_SVE_IMM', `
3623+
instruct vmask$4_imm$2_sve(pReg dst, vReg src, $3 imm, immI_$4_cond cond, rFlagsReg cr) %{
3624+
predicate(UseSVE > 0);
3625+
match(Set dst (VectorMaskCmp (Binary src (Replicate$2 imm)) cond));
3626+
effect(KILL cr);
3627+
format %{ "vmask$4_imm$2_sve $dst, $src, $imm, $cond\t# KILL cr" %}
3628+
ins_encode %{
3629+
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
3630+
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
3631+
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
3632+
__ sve_cmp(condition, $dst$$PRegister, __ $1, ptrue, $src$$FloatRegister, (int)$imm$$constant);
3633+
%}
3634+
ins_pipe(pipe_slow);
3635+
%}')dnl
3636+
VMASKCMP_SVE_IMM(B, B, immI5, cmp)
3637+
VMASKCMP_SVE_IMM(B, B, immIU7, cmpU)
3638+
VMASKCMP_SVE_IMM(H, S, immI5, cmp)
3639+
VMASKCMP_SVE_IMM(H, S, immIU7, cmpU)
3640+
VMASKCMP_SVE_IMM(S, I, immI5, cmp)
3641+
VMASKCMP_SVE_IMM(S, I, immIU7, cmpU)
3642+
VMASKCMP_SVE_IMM(D, L, immL5, cmp)
3643+
VMASKCMP_SVE_IMM(D, L, immLU7, cmpU)
36193644

36203645
instruct vmaskcmp_masked(pReg dst, vReg src1, vReg src2, immI cond,
36213646
pRegGov pg, rFlagsReg cr) %{

‎src/hotspot/cpu/aarch64/assembler_aarch64.hpp

+69-41
Original file line numberDiff line numberDiff line change
@@ -3786,50 +3786,78 @@ template<typename R, typename... Rx>
37863786
INSN(sve_fac, 0b01100101, 0b11, 1); // Floating-point absolute compare vectors
37873787
#undef INSN
37883788

3789-
// SVE Integer Compare - Signed Immediate
3790-
void sve_cmp(Condition cond, PRegister Pd, SIMD_RegVariant T,
3791-
PRegister Pg, FloatRegister Zn, int imm5) {
3792-
starti;
3793-
assert(T != Q, "invalid size");
3794-
guarantee(-16 <= imm5 && imm5 <= 15, "invalid immediate");
3795-
int cond_op;
3796-
switch(cond) {
3797-
case EQ: cond_op = 0b1000; break;
3798-
case NE: cond_op = 0b1001; break;
3799-
case GE: cond_op = 0b0000; break;
3800-
case GT: cond_op = 0b0001; break;
3801-
case LE: cond_op = 0b0011; break;
3802-
case LT: cond_op = 0b0010; break;
3803-
default:
3804-
ShouldNotReachHere();
3789+
private:
3790+
// Convert Assembler::Condition to op encoding - used by sve integer compare encoding
3791+
static int assembler_cond_to_sve_op(Condition cond, bool &is_unsigned) {
3792+
if (cond == HI || cond == HS || cond == LO || cond == LS) {
3793+
is_unsigned = true;
3794+
} else {
3795+
is_unsigned = false;
3796+
}
3797+
3798+
switch (cond) {
3799+
case HI:
3800+
case GT:
3801+
return 0b0001;
3802+
case HS:
3803+
case GE:
3804+
return 0b0000;
3805+
case LO:
3806+
case LT:
3807+
return 0b0010;
3808+
case LS:
3809+
case LE:
3810+
return 0b0011;
3811+
case EQ:
3812+
return 0b1000;
3813+
case NE:
3814+
return 0b1001;
3815+
default:
3816+
ShouldNotReachHere();
3817+
return -1;
3818+
}
38053819
}
3806-
f(0b00100101, 31, 24), f(T, 23, 22), f(0b0, 21), sf(imm5, 20, 16),
3807-
f((cond_op >> 1) & 0x7, 15, 13), pgrf(Pg, 10), rf(Zn, 5);
3808-
f(cond_op & 0x1, 4), prf(Pd, 0);
3809-
}
38103820

3811-
// SVE Floating-point compare vector with zero
3812-
void sve_fcm(Condition cond, PRegister Pd, SIMD_RegVariant T,
3813-
PRegister Pg, FloatRegister Zn, double d) {
3814-
starti;
3815-
assert(T != Q, "invalid size");
3816-
guarantee(d == 0.0, "invalid immediate");
3817-
int cond_op;
3818-
switch(cond) {
3819-
case EQ: cond_op = 0b100; break;
3820-
case GT: cond_op = 0b001; break;
3821-
case GE: cond_op = 0b000; break;
3822-
case LT: cond_op = 0b010; break;
3823-
case LE: cond_op = 0b011; break;
3824-
case NE: cond_op = 0b110; break;
3825-
default:
3826-
ShouldNotReachHere();
3821+
public:
3822+
// SVE Integer Compare - 5 bits signed imm and 7 bits unsigned imm
3823+
void sve_cmp(Condition cond, PRegister Pd, SIMD_RegVariant T,
3824+
PRegister Pg, FloatRegister Zn, int imm) {
3825+
starti;
3826+
assert(T != Q, "invalid size");
3827+
bool is_unsigned = false;
3828+
int cond_op = assembler_cond_to_sve_op(cond, is_unsigned);
3829+
f(is_unsigned ? 0b00100100 : 0b00100101, 31, 24), f(T, 23, 22);
3830+
f(is_unsigned ? 0b1 : 0b0, 21);
3831+
if (is_unsigned) {
3832+
f(imm, 20, 14), f((cond_op >> 1) & 0x1, 13);
3833+
} else {
3834+
sf(imm, 20, 16), f((cond_op >> 1) & 0x7, 15, 13);
3835+
}
3836+
pgrf(Pg, 10), rf(Zn, 5), f(cond_op & 0x1, 4), prf(Pd, 0);
3837+
}
3838+
3839+
// SVE Floating-point compare vector with zero
3840+
void sve_fcm(Condition cond, PRegister Pd, SIMD_RegVariant T,
3841+
PRegister Pg, FloatRegister Zn, double d) {
3842+
starti;
3843+
assert(T != Q, "invalid size");
3844+
guarantee(d == 0.0, "invalid immediate");
3845+
int cond_op;
3846+
switch(cond) {
3847+
case EQ: cond_op = 0b100; break;
3848+
case GT: cond_op = 0b001; break;
3849+
case GE: cond_op = 0b000; break;
3850+
case LT: cond_op = 0b010; break;
3851+
case LE: cond_op = 0b011; break;
3852+
case NE: cond_op = 0b110; break;
3853+
default:
3854+
ShouldNotReachHere();
3855+
}
3856+
f(0b01100101, 31, 24), f(T, 23, 22), f(0b0100, 21, 18),
3857+
f((cond_op >> 1) & 0x3, 17, 16), f(0b001, 15, 13),
3858+
pgrf(Pg, 10), rf(Zn, 5);
3859+
f(cond_op & 0x1, 4), prf(Pd, 0);
38273860
}
3828-
f(0b01100101, 31, 24), f(T, 23, 22), f(0b0100, 21, 18),
3829-
f((cond_op >> 1) & 0x3, 17, 16), f(0b001, 15, 13),
3830-
pgrf(Pg, 10), rf(Zn, 5);
3831-
f(cond_op & 0x1, 4), prf(Pd, 0);
3832-
}
38333861

38343862
// SVE unpack vector elements
38353863
#define INSN(NAME, op) \

0 commit comments

Comments
 (0)