Skip to content
Permalink
Browse files
8285973: x86_64: Improve fp comparison and cmove for eq/ne
Reviewed-by: kvn, sviswanathan
  • Loading branch information
merykitty authored and Sandhya Viswanathan committed May 24, 2022
1 parent 782ae38 commit c1db70d827f7ac81aa6c6646e2431f672c71c8dc
Showing 4 changed files with 634 additions and 84 deletions.
@@ -4268,30 +4268,33 @@ operand cmpOpU()
equal(0x4, "e");
not_equal(0x5, "ne");
less(0x2, "b");
greater_equal(0x3, "nb");
greater_equal(0x3, "ae");
less_equal(0x6, "be");
greater(0x7, "nbe");
greater(0x7, "a");
overflow(0x0, "o");
no_overflow(0x1, "no");
%}
%}


// Floating comparisons that don't require any fixup for the unordered case
// Floating comparisons that don't require any fixup for the unordered case,
// If both inputs of the comparison are the same, ZF is always set so we
// don't need to use cmpOpUCF2 for eq/ne
operand cmpOpUCF() %{
match(Bool);
predicate(n->as_Bool()->_test._test == BoolTest::lt ||
n->as_Bool()->_test._test == BoolTest::ge ||
n->as_Bool()->_test._test == BoolTest::le ||
n->as_Bool()->_test._test == BoolTest::gt);
n->as_Bool()->_test._test == BoolTest::gt ||
n->in(1)->in(1) == n->in(1)->in(2));
format %{ "" %}
interface(COND_INTER) %{
equal(0x4, "e");
not_equal(0x5, "ne");
equal(0xb, "np");
not_equal(0xa, "p");
less(0x2, "b");
greater_equal(0x3, "nb");
greater_equal(0x3, "ae");
less_equal(0x6, "be");
greater(0x7, "nbe");
greater(0x7, "a");
overflow(0x0, "o");
no_overflow(0x1, "no");
%}
@@ -4301,16 +4304,17 @@ operand cmpOpUCF() %{
// Floating comparisons that can be fixed up with extra conditional jumps
operand cmpOpUCF2() %{
match(Bool);
predicate(n->as_Bool()->_test._test == BoolTest::ne ||
n->as_Bool()->_test._test == BoolTest::eq);
predicate((n->as_Bool()->_test._test == BoolTest::ne ||
n->as_Bool()->_test._test == BoolTest::eq) &&
n->in(1)->in(1) != n->in(1)->in(2));
format %{ "" %}
interface(COND_INTER) %{
equal(0x4, "e");
not_equal(0x5, "ne");
less(0x2, "b");
greater_equal(0x3, "nb");
greater_equal(0x3, "ae");
less_equal(0x6, "be");
greater(0x7, "nbe");
greater(0x7, "a");
overflow(0x0, "o");
no_overflow(0x1, "no");
%}
@@ -7037,6 +7041,36 @@ instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
%}
%}

instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));

ins_cost(200); // XXX
format %{ "cmovpl $dst, $src\n\t"
"cmovnel $dst, $src" %}
ins_encode %{
__ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
__ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
%}
ins_pipe(pipe_cmov_reg);
%}

// Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
// inputs of the CMove
instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));

ins_cost(200); // XXX
format %{ "cmovpl $dst, $src\n\t"
"cmovnel $dst, $src" %}
ins_encode %{
__ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
__ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
%}
ins_pipe(pipe_cmov_reg);
%}

// Conditional move
instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
@@ -7104,6 +7138,36 @@ instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
%}
%}

instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));

ins_cost(200); // XXX
format %{ "cmovpl $dst, $src\n\t"
"cmovnel $dst, $src" %}
ins_encode %{
__ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
__ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
%}
ins_pipe(pipe_cmov_reg);
%}

// Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
// inputs of the CMove
instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));

ins_cost(200); // XXX
format %{ "cmovpl $dst, $src\n\t"
"cmovnel $dst, $src" %}
ins_encode %{
__ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
__ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
%}
ins_pipe(pipe_cmov_reg);
%}

// Conditional move
instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
%{
@@ -7138,6 +7202,36 @@ instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
%}
%}

instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));

ins_cost(200); // XXX
format %{ "cmovpq $dst, $src\n\t"
"cmovneq $dst, $src" %}
ins_encode %{
__ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
__ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
%}
ins_pipe(pipe_cmov_reg);
%}

// Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
// inputs of the CMove
instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));

ins_cost(200); // XXX
format %{ "cmovpq $dst, $src\n\t"
"cmovneq $dst, $src" %}
ins_encode %{
__ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
__ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
%}
ins_pipe(pipe_cmov_reg);
%}

// DISABLED: Requires the ADLC to emit a bottom_type call that
// correctly meets the two pointer arguments; one is an incoming
// register but the other is a memory operand. ALSO appears to
@@ -7209,6 +7303,36 @@ instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
%}
%}

instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));

ins_cost(200); // XXX
format %{ "cmovpq $dst, $src\n\t"
"cmovneq $dst, $src" %}
ins_encode %{
__ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
__ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
%}
ins_pipe(pipe_cmov_reg);
%}

// Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
// inputs of the CMove
instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));

ins_cost(200); // XXX
format %{ "cmovpq $dst, $src\n\t"
"cmovneq $dst, $src" %}
ins_encode %{
__ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
__ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
%}
ins_pipe(pipe_cmov_reg);
%}

instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
%{
match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
@@ -10479,11 +10603,12 @@ instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)

//---------- FP Instructions------------------------------------------------

// Really expensive, avoid
instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
%{
match(Set cr (CmpF src1 src2));

ins_cost(145);
ins_cost(500);
format %{ "ucomiss $src1, $src2\n\t"
"jnp,s exit\n\t"
"pushfq\t# saw NaN, set CF\n\t"
@@ -10508,24 +10633,6 @@ instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
ins_pipe(pipe_slow);
%}

instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
%{
match(Set cr (CmpF src1 (LoadF src2)));

ins_cost(145);
format %{ "ucomiss $src1, $src2\n\t"
"jnp,s exit\n\t"
"pushfq\t# saw NaN, set CF\n\t"
"andq [rsp], #0xffffff2b\n\t"
"popfq\n"
"exit:" %}
ins_encode %{
__ ucomiss($src1$$XMMRegister, $src2$$Address);
emit_cmpfp_fixup(_masm);
%}
ins_pipe(pipe_slow);
%}

instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
match(Set cr (CmpF src1 (LoadF src2)));

@@ -10537,23 +10644,6 @@ instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
ins_pipe(pipe_slow);
%}

instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
match(Set cr (CmpF src con));

ins_cost(145);
format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
"jnp,s exit\n\t"
"pushfq\t# saw NaN, set CF\n\t"
"andq [rsp], #0xffffff2b\n\t"
"popfq\n"
"exit:" %}
ins_encode %{
__ ucomiss($src$$XMMRegister, $constantaddress($con));
emit_cmpfp_fixup(_masm);
%}
ins_pipe(pipe_slow);
%}

instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
match(Set cr (CmpF src con));
ins_cost(100);
@@ -10564,11 +10654,12 @@ instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
ins_pipe(pipe_slow);
%}

// Really expensive, avoid
instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
%{
match(Set cr (CmpD src1 src2));

ins_cost(145);
ins_cost(500);
format %{ "ucomisd $src1, $src2\n\t"
"jnp,s exit\n\t"
"pushfq\t# saw NaN, set CF\n\t"
@@ -10593,24 +10684,6 @@ instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
ins_pipe(pipe_slow);
%}

instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
%{
match(Set cr (CmpD src1 (LoadD src2)));

ins_cost(145);
format %{ "ucomisd $src1, $src2\n\t"
"jnp,s exit\n\t"
"pushfq\t# saw NaN, set CF\n\t"
"andq [rsp], #0xffffff2b\n\t"
"popfq\n"
"exit:" %}
ins_encode %{
__ ucomisd($src1$$XMMRegister, $src2$$Address);
emit_cmpfp_fixup(_masm);
%}
ins_pipe(pipe_slow);
%}

instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
match(Set cr (CmpD src1 (LoadD src2)));

@@ -10622,23 +10695,6 @@ instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
ins_pipe(pipe_slow);
%}

instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
match(Set cr (CmpD src con));

ins_cost(145);
format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
"jnp,s exit\n\t"
"pushfq\t# saw NaN, set CF\n\t"
"andq [rsp], #0xffffff2b\n\t"
"popfq\n"
"exit:" %}
ins_encode %{
__ ucomisd($src$$XMMRegister, $constantaddress($con));
emit_cmpfp_fixup(_masm);
%}
ins_pipe(pipe_slow);
%}

instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
match(Set cr (CmpD src con));
ins_cost(100);

1 comment on commit c1db70d

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on c1db70d May 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.