Skip to content

/ loom

Permalink
Browse files

8238690: C2: Handle vector shifts by constant and non-constant scalar… 

… uniformly

Reviewed-by: thartmann, sviswanathan
  • Loading branch information
Vladimir Ivanov
Vladimir Ivanov committed Feb 11, 2020
1 parent 0d84fe9 commit 74e68b40924d68ffe5a7cc5855b7919039798a3c
42 src/hotspot/cpu/aarch64/aarch64.ad
View file
@@ -17312,7 +17312,7 @@ instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 4 ||
n->as_Vector()->length() == 8);
match(Set dst (LShiftVB src shift));
match(Set dst (LShiftVB src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (8B)" %}
ins_encode %{
@@ -17331,7 +17331,7 @@ instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{

instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 16);
match(Set dst (LShiftVB src shift));
match(Set dst (LShiftVB src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (16B)" %}
ins_encode %{
@@ -17351,7 +17351,7 @@ instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 4 ||
n->as_Vector()->length() == 8);
match(Set dst (RShiftVB src shift));
match(Set dst (RShiftVB src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (8B)" %}
ins_encode %{
@@ -17365,7 +17365,7 @@ instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{

instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 16);
match(Set dst (RShiftVB src shift));
match(Set dst (RShiftVB src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (16B)" %}
ins_encode %{
@@ -17380,7 +17380,7 @@ instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 4 ||
n->as_Vector()->length() == 8);
match(Set dst (URShiftVB src shift));
match(Set dst (URShiftVB src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (8B)" %}
ins_encode %{
@@ -17399,7 +17399,7 @@ instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{

instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 16);
match(Set dst (URShiftVB src shift));
match(Set dst (URShiftVB src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (16B)" %}
ins_encode %{
@@ -17516,7 +17516,7 @@ instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2 ||
n->as_Vector()->length() == 4);
match(Set dst (LShiftVS src shift));
match(Set dst (LShiftVS src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (4H)" %}
ins_encode %{
@@ -17535,7 +17535,7 @@ instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{

instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (LShiftVS src shift));
match(Set dst (LShiftVS src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (8H)" %}
ins_encode %{
@@ -17555,7 +17555,7 @@ instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2 ||
n->as_Vector()->length() == 4);
match(Set dst (RShiftVS src shift));
match(Set dst (RShiftVS src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (4H)" %}
ins_encode %{
@@ -17569,7 +17569,7 @@ instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{

instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (RShiftVS src shift));
match(Set dst (RShiftVS src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (8H)" %}
ins_encode %{
@@ -17584,7 +17584,7 @@ instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2 ||
n->as_Vector()->length() == 4);
match(Set dst (URShiftVS src shift));
match(Set dst (URShiftVS src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (4H)" %}
ins_encode %{
@@ -17603,7 +17603,7 @@ instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{

instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (URShiftVS src shift));
match(Set dst (URShiftVS src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (8H)" %}
ins_encode %{
@@ -17716,7 +17716,7 @@ instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{

instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (LShiftVI src shift));
match(Set dst (LShiftVI src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (2S)" %}
ins_encode %{
@@ -17729,7 +17729,7 @@ instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{

instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (LShiftVI src shift));
match(Set dst (LShiftVI src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (4S)" %}
ins_encode %{
@@ -17742,7 +17742,7 @@ instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{

instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (RShiftVI src shift));
match(Set dst (RShiftVI src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (2S)" %}
ins_encode %{
@@ -17755,7 +17755,7 @@ instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{

instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (RShiftVI src shift));
match(Set dst (RShiftVI src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (4S)" %}
ins_encode %{
@@ -17768,7 +17768,7 @@ instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{

instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (URShiftVI src shift));
match(Set dst (URShiftVI src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (2S)" %}
ins_encode %{
@@ -17781,7 +17781,7 @@ instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{

instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (URShiftVI src shift));
match(Set dst (URShiftVI src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (4S)" %}
ins_encode %{
@@ -17841,7 +17841,7 @@ instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{

instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (LShiftVL src shift));
match(Set dst (LShiftVL src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (2D)" %}
ins_encode %{
@@ -17854,7 +17854,7 @@ instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{

instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (RShiftVL src shift));
match(Set dst (RShiftVL src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (2D)" %}
ins_encode %{
@@ -17867,7 +17867,7 @@ instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{

instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (URShiftVL src shift));
match(Set dst (URShiftVL src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (2D)" %}
ins_encode %{
22 src/hotspot/cpu/arm/arm.ad
View file
@@ -10618,7 +10618,7 @@ instruct vsl16B_reg(vecX dst, vecX src, vecX shift) %{

instruct vsl8B_immI(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (LShiftVB src shift));
match(Set dst (LShiftVB src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@@ -10634,7 +10634,7 @@ instruct vsl8B_immI(vecD dst, vecD src, immI shift) %{

instruct vsl16B_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 16);
match(Set dst (LShiftVB src shift));
match(Set dst (LShiftVB src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@@ -10673,7 +10673,7 @@ instruct vsl8S_reg(vecX dst, vecX src, vecX shift) %{

instruct vsl4S_immI(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (LShiftVS src shift));
match(Set dst (LShiftVS src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@@ -10728,7 +10728,7 @@ instruct vsl4I_reg(vecX dst, vecX src, vecX shift) %{

instruct vsl2I_immI(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
match(Set dst (LShiftVI src shift));
match(Set dst (LShiftVI src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@@ -10744,7 +10744,7 @@ instruct vsl2I_immI(vecD dst, vecD src, immI shift) %{

instruct vsl4I_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
match(Set dst (LShiftVI src shift));
match(Set dst (LShiftVI src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@@ -10772,7 +10772,7 @@ instruct vsl2L_reg(vecX dst, vecX src, vecX shift) %{

instruct vsl2L_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (LShiftVL src shift));
match(Set dst (LShiftVL src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@@ -10795,7 +10795,7 @@ instruct vsl2L_immI(vecX dst, vecX src, immI shift) %{
// Chars vector logical right shift
instruct vsrl4S_immI(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (URShiftVS src shift));
match(Set dst (URShiftVS src (RShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@@ -10811,7 +10811,7 @@ instruct vsrl4S_immI(vecD dst, vecD src, immI shift) %{

instruct vsrl8S_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (URShiftVS src shift));
match(Set dst (URShiftVS src (RShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@@ -10828,7 +10828,7 @@ instruct vsrl8S_immI(vecX dst, vecX src, immI shift) %{
// Integers vector logical right shift
instruct vsrl2I_immI(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
match(Set dst (URShiftVI src shift));
match(Set dst (URShiftVI src (RShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@@ -10844,7 +10844,7 @@ instruct vsrl2I_immI(vecD dst, vecD src, immI shift) %{

instruct vsrl4I_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
match(Set dst (URShiftVI src shift));
match(Set dst (URShiftVI src (RShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@@ -10861,7 +10861,7 @@ instruct vsrl4I_immI(vecX dst, vecX src, immI shift) %{
// Longs vector logical right shift
instruct vsrl2L_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (URShiftVL src shift));
match(Set dst (URShiftVL src (RShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
12 src/hotspot/cpu/x86/x86.ad
View file
@@ -5254,18 +5254,6 @@ instruct vshiftcnt(vec dst, rRegI cnt) %{
ins_pipe( pipe_slow );
%}

instruct vshiftcntimm(vec dst, immI8 cnt, rRegI tmp) %{
match(Set dst cnt);
effect(TEMP tmp);
format %{ "movl $tmp,$cnt\t"
"movdl $dst,$tmp\t! load shift count" %}
ins_encode %{
__ movl($tmp$$Register, $cnt$$constant);
__ movdl($dst$$XMMRegister, $tmp$$Register);
%}
ins_pipe( pipe_slow );
%}

// Byte vector shift
instruct vshiftB(vec dst, vec src, vec shift, vec tmp, rRegI scratch) %{
predicate(n->as_Vector()->length() <= 8);
37 src/hotspot/share/opto/matcher.cpp
View file
@@ -2010,6 +2010,15 @@ bool Matcher::is_bmi_pattern(Node *n, Node *m) {
}
#endif // X86

bool Matcher::is_vshift_con_pattern(Node *n, Node *m) {
if (n != NULL && m != NULL) {
return VectorNode::is_vector_shift(n) &&
VectorNode::is_vector_shift_count(m) && m->in(1)->is_Con();
}
return false;
}


bool Matcher::clone_base_plus_offset_address(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
Node *off = m->in(AddPNode::Offset);
if (off->is_Con()) {
@@ -2090,6 +2099,10 @@ void Matcher::find_shared( Node *n ) {
continue;
}
#endif
if (is_vshift_con_pattern(n, m)) {
mstack.push(m, Visit);
continue;
}

// Clone addressing expressions as they are "free" in memory access instructions
if (mem_op && i == mem_addr_idx && mop == Op_AddP &&
@@ -2525,22 +2538,16 @@ void Matcher::do_postselect_cleanup() {
//----------------------------------------------------------------------

// Convert (leg)Vec to (leg)Vec[SDXYZ].
MachOper* Matcher::specialize_vector_operand_helper(MachNode* m, uint opnd_idx, const Type* t) {
MachOper* Matcher::specialize_vector_operand_helper(MachNode* m, uint opnd_idx, const TypeVect* vt) {
MachOper* original_opnd = m->_opnds[opnd_idx];
uint ideal_reg = t->ideal_reg();
uint ideal_reg = vt->ideal_reg();
// Handle special cases.
if (t->isa_vect()) {
// LShiftCntV/RShiftCntV report wide vector type, but Matcher::vector_shift_count_ideal_reg() as ideal register (see vectornode.hpp).
// Look for shift count use sites as well (at vector shift nodes).
int opc = m->ideal_Opcode();
if ((VectorNode::is_shift_count(opc) && opnd_idx == 0) || // DEF operand of LShiftCntV/RShiftCntV
(VectorNode::is_vector_shift(opc) && opnd_idx == 2)) { // shift operand of a vector shift node
ideal_reg = Matcher::vector_shift_count_ideal_reg(t->is_vect()->length_in_bytes());
}
} else {
// Chain instructions which convert scalar to vector (e.g., vshiftcntimm on x86) don't have vector type.
int size_in_bytes = 4 * type2size[t->basic_type()];
ideal_reg = Matcher::vector_ideal_reg(size_in_bytes);
// LShiftCntV/RShiftCntV report wide vector type, but Matcher::vector_shift_count_ideal_reg() as ideal register (see vectornode.hpp).
// Look for shift count use sites as well (at vector shift nodes).
int opc = m->ideal_Opcode();
if ((VectorNode::is_vector_shift_count(opc) && opnd_idx == 0) || // DEF operand of LShiftCntV/RShiftCntV
(VectorNode::is_vector_shift(opc) && opnd_idx == 2)) { // shift operand of a vector shift node
ideal_reg = Matcher::vector_shift_count_ideal_reg(vt->length_in_bytes());
}
return Matcher::specialize_generic_vector_operand(original_opnd, ideal_reg, false);
}
@@ -2575,7 +2582,7 @@ MachOper* Matcher::specialize_vector_operand(MachNode* m, uint opnd_idx) {
}
}
}
return specialize_vector_operand_helper(m, opnd_idx, def->bottom_type());
return specialize_vector_operand_helper(m, opnd_idx, def->bottom_type()->is_vect());
}

void Matcher::specialize_mach_node(MachNode* m) {
4 src/hotspot/share/opto/matcher.hpp
View file
@@ -125,6 +125,8 @@ class Matcher : public PhaseTransform {
bool is_bmi_pattern(Node *n, Node *m);
#endif

bool is_vshift_con_pattern(Node *n, Node *m);

// Debug and profile information for nodes in old space:
GrowableArray<Node_Notes*>* _old_node_note_array;

@@ -517,7 +519,7 @@ class Matcher : public PhaseTransform {
void specialize_mach_node(MachNode* m);
void specialize_temp_node(MachTempNode* tmp, MachNode* use, uint idx);
MachOper* specialize_vector_operand(MachNode* m, uint opnd_idx);
MachOper* specialize_vector_operand_helper(MachNode* m, uint opnd_idx, const Type* t);
MachOper* specialize_vector_operand_helper(MachNode* m, uint opnd_idx, const TypeVect* vt);

static MachOper* specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp);

0 comments on commit 74e68b4

Please sign in to comment.