Skip to content

Commit 6b2a86a

Browse files
rwestrelrwestrel
committed
8300257: C2: vectorization fails on some simple Memory Segment loops
Reviewed-by: kvn, thartmann
1 parent dc81603 commit 6b2a86a

File tree

3 files changed

+250
-62
lines changed

3 files changed

+250
-62
lines changed

‎src/hotspot/share/opto/superword.cpp‎

Lines changed: 102 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
#include "opto/mulnode.hpp"
3737
#include "opto/opcodes.hpp"
3838
#include "opto/opaquenode.hpp"
39+
#include "opto/rootnode.hpp"
3940
#include "opto/superword.hpp"
4041
#include "opto/vectornode.hpp"
4142
#include "opto/movenode.hpp"
@@ -201,7 +202,6 @@ bool SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
201202
//------------------------------early unrolling analysis------------------------------
202203
void SuperWord::unrolling_analysis(int &local_loop_unroll_factor) {
203204
bool is_slp = true;
204-
ResourceMark rm;
205205
size_t ignored_size = lpt()->_body.size();
206206
int *ignored_loop_nodes = NEW_RESOURCE_ARRAY(int, ignored_size);
207207
Node_Stack nstack((int)ignored_size);
@@ -4248,19 +4248,7 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
42484248
invar = new ConvL2INode(invar);
42494249
_igvn.register_new_node_with_optimizer(invar);
42504250
}
4251-
Node* invar_scale = align_to_ref_p.invar_scale();
4252-
if (invar_scale != nullptr) {
4253-
invar = new LShiftINode(invar, invar_scale);
4254-
_igvn.register_new_node_with_optimizer(invar);
4255-
}
4256-
Node* aref = new URShiftINode(invar, log2_elt);
4257-
_igvn.register_new_node_with_optimizer(aref);
4258-
_phase->set_ctrl(aref, pre_ctrl);
4259-
if (align_to_ref_p.negate_invar()) {
4260-
e = new SubINode(e, aref);
4261-
} else {
4262-
e = new AddINode(e, aref);
4263-
}
4251+
e = new URShiftINode(invar, log2_elt);
42644252
_igvn.register_new_node_with_optimizer(e);
42654253
_phase->set_ctrl(e, pre_ctrl);
42664254
}
@@ -4440,9 +4428,11 @@ int SWPointer::Tracer::_depth = 0;
44404428
#endif
44414429
//----------------------------SWPointer------------------------
44424430
SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool analyze_only) :
4443-
_mem(mem), _slp(slp), _base(nullptr), _adr(nullptr),
4444-
_scale(0), _offset(0), _invar(nullptr), _negate_invar(false),
4445-
_invar_scale(nullptr),
4431+
_mem(mem), _slp(slp), _base(nullptr), _adr(nullptr),
4432+
_scale(0), _offset(0), _invar(nullptr),
4433+
#ifdef ASSERT
4434+
_debug_invar(nullptr), _debug_negate_invar(false), _debug_invar_scale(nullptr),
4435+
#endif
44464436
_nstack(nstack), _analyze_only(analyze_only),
44474437
_stack_idx(0)
44484438
#ifndef PRODUCT
@@ -4473,7 +4463,7 @@ SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool anal
44734463
NOT_PRODUCT(_tracer.ctor_2(adr);)
44744464

44754465
int i;
4476-
for (i = 0; i < 3; i++) {
4466+
for (i = 0; ; i++) {
44774467
NOT_PRODUCT(_tracer.ctor_3(adr, i);)
44784468

44794469
if (!scaled_iv_plus_offset(adr->in(AddPNode::Offset))) {
@@ -4509,9 +4499,11 @@ SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool anal
45094499
// Following is used to create a temporary object during
45104500
// the pattern match of an address expression.
45114501
SWPointer::SWPointer(SWPointer* p) :
4512-
_mem(p->_mem), _slp(p->_slp), _base(nullptr), _adr(nullptr),
4513-
_scale(0), _offset(0), _invar(nullptr), _negate_invar(false),
4514-
_invar_scale(nullptr),
4502+
_mem(p->_mem), _slp(p->_slp), _base(nullptr), _adr(nullptr),
4503+
_scale(0), _offset(0), _invar(nullptr),
4504+
#ifdef ASSERT
4505+
_debug_invar(nullptr), _debug_negate_invar(false), _debug_invar_scale(nullptr),
4506+
#endif
45154507
_nstack(p->_nstack), _analyze_only(p->_analyze_only),
45164508
_stack_idx(p->_stack_idx)
45174509
#ifndef PRODUCT
@@ -4625,7 +4617,7 @@ bool SWPointer::scaled_iv(Node* n) {
46254617
return true;
46264618
}
46274619
} else if (opc == Op_LShiftL && n->in(2)->is_Con()) {
4628-
if (!has_iv() && _invar == nullptr) {
4620+
if (!has_iv()) {
46294621
// Need to preserve the current _offset value, so
46304622
// create a temporary object for this expression subtree.
46314623
// Hacky, so should re-engineer the address pattern match.
@@ -4637,12 +4629,15 @@ bool SWPointer::scaled_iv(Node* n) {
46374629
int scale = n->in(2)->get_int();
46384630
_scale = tmp._scale << scale;
46394631
_offset += tmp._offset << scale;
4640-
_invar = tmp._invar;
4641-
if (_invar != nullptr) {
4642-
_negate_invar = tmp._negate_invar;
4643-
_invar_scale = n->in(2);
4632+
if (tmp._invar != nullptr) {
4633+
BasicType bt = tmp._invar->bottom_type()->basic_type();
4634+
assert(bt == T_INT || bt == T_LONG, "");
4635+
maybe_add_to_invar(register_if_new(LShiftNode::make(tmp._invar, n->in(2), bt)), false);
4636+
#ifdef ASSERT
4637+
_debug_invar_scale = n->in(2);
4638+
#endif
46444639
}
4645-
NOT_PRODUCT(_tracer.scaled_iv_9(n, _scale, _offset, _invar, _negate_invar);)
4640+
NOT_PRODUCT(_tracer.scaled_iv_9(n, _scale, _offset, _invar);)
46464641
return true;
46474642
}
46484643
}
@@ -4676,41 +4671,34 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
46764671
NOT_PRODUCT(_tracer.offset_plus_k_4(n);)
46774672
return false;
46784673
}
4679-
if (_invar != nullptr) { // already has an invariant
4680-
NOT_PRODUCT(_tracer.offset_plus_k_5(n, _invar);)
4681-
return false;
4682-
}
4674+
assert((_debug_invar == nullptr) == (_invar == nullptr), "");
46834675

46844676
if (_analyze_only && is_loop_member(n)) {
46854677
_nstack->push(n, _stack_idx++);
46864678
}
46874679
if (opc == Op_AddI) {
46884680
if (n->in(2)->is_Con() && invariant(n->in(1))) {
4689-
_negate_invar = negate;
4690-
_invar = n->in(1);
4681+
maybe_add_to_invar(n->in(1), negate);
46914682
_offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
4692-
NOT_PRODUCT(_tracer.offset_plus_k_6(n, _invar, _negate_invar, _offset);)
4683+
NOT_PRODUCT(_tracer.offset_plus_k_6(n, _invar, negate, _offset);)
46934684
return true;
46944685
} else if (n->in(1)->is_Con() && invariant(n->in(2))) {
46954686
_offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int();
4696-
_negate_invar = negate;
4697-
_invar = n->in(2);
4698-
NOT_PRODUCT(_tracer.offset_plus_k_7(n, _invar, _negate_invar, _offset);)
4687+
maybe_add_to_invar(n->in(2), negate);
4688+
NOT_PRODUCT(_tracer.offset_plus_k_7(n, _invar, negate, _offset);)
46994689
return true;
47004690
}
47014691
}
47024692
if (opc == Op_SubI) {
47034693
if (n->in(2)->is_Con() && invariant(n->in(1))) {
4704-
_negate_invar = negate;
4705-
_invar = n->in(1);
4694+
maybe_add_to_invar(n->in(1), negate);
47064695
_offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
4707-
NOT_PRODUCT(_tracer.offset_plus_k_8(n, _invar, _negate_invar, _offset);)
4696+
NOT_PRODUCT(_tracer.offset_plus_k_8(n, _invar, negate, _offset);)
47084697
return true;
47094698
} else if (n->in(1)->is_Con() && invariant(n->in(2))) {
47104699
_offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int();
4711-
_negate_invar = !negate;
4712-
_invar = n->in(2);
4713-
NOT_PRODUCT(_tracer.offset_plus_k_9(n, _invar, _negate_invar, _offset);)
4700+
maybe_add_to_invar(n->in(2), !negate);
4701+
NOT_PRODUCT(_tracer.offset_plus_k_9(n, _invar, !negate, _offset);)
47144702
return true;
47154703
}
47164704
}
@@ -4727,9 +4715,8 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
47274715
}
47284716
// Check if 'n' can really be used as invariant (not in main loop and dominating the pre loop).
47294717
if (invariant(n)) {
4730-
_negate_invar = negate;
4731-
_invar = n;
4732-
NOT_PRODUCT(_tracer.offset_plus_k_10(n, _invar, _negate_invar, _offset);)
4718+
maybe_add_to_invar(n, negate);
4719+
NOT_PRODUCT(_tracer.offset_plus_k_10(n, _invar, negate, _offset);)
47334720
return true;
47344721
}
47354722
}
@@ -4738,6 +4725,67 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
47384725
return false;
47394726
}
47404727

4728+
Node* SWPointer::maybe_negate_invar(bool negate, Node* invar) {
4729+
#ifdef ASSERT
4730+
_debug_negate_invar = negate;
4731+
#endif
4732+
if (negate) {
4733+
BasicType bt = invar->bottom_type()->basic_type();
4734+
assert(bt == T_INT || bt == T_LONG, "");
4735+
PhaseIterGVN& igvn = phase()->igvn();
4736+
Node* zero = igvn.zerocon(bt);
4737+
phase()->set_ctrl(zero, phase()->C->root());
4738+
Node* sub = SubNode::make(zero, invar, bt);
4739+
invar = register_if_new(sub);
4740+
}
4741+
return invar;
4742+
}
4743+
4744+
Node* SWPointer::register_if_new(Node* n) const {
4745+
PhaseIterGVN& igvn = phase()->igvn();
4746+
Node* prev = igvn.hash_find_insert(n);
4747+
if (prev != nullptr) {
4748+
n->destruct(&igvn);
4749+
n = prev;
4750+
} else {
4751+
Node* c = phase()->get_early_ctrl(n);
4752+
phase()->register_new_node(n, c);
4753+
}
4754+
return n;
4755+
}
4756+
4757+
void SWPointer::maybe_add_to_invar(Node* new_invar, bool negate) {
4758+
new_invar = maybe_negate_invar(negate, new_invar);
4759+
if (_invar == nullptr) {
4760+
_invar = new_invar;
4761+
#ifdef ASSERT
4762+
_debug_invar = new_invar;
4763+
#endif
4764+
return;
4765+
}
4766+
#ifdef ASSERT
4767+
_debug_invar = NodeSentinel;
4768+
#endif
4769+
BasicType new_invar_bt = new_invar->bottom_type()->basic_type();
4770+
assert(new_invar_bt == T_INT || new_invar_bt == T_LONG, "");
4771+
BasicType invar_bt = _invar->bottom_type()->basic_type();
4772+
assert(invar_bt == T_INT || invar_bt == T_LONG, "");
4773+
4774+
BasicType bt = (new_invar_bt == T_LONG || invar_bt == T_LONG) ? T_LONG : T_INT;
4775+
Node* current_invar = _invar;
4776+
if (invar_bt != bt) {
4777+
assert(bt == T_LONG && invar_bt == T_INT, "");
4778+
assert(new_invar_bt == bt, "");
4779+
current_invar = register_if_new(new ConvI2LNode(current_invar));
4780+
} else if (new_invar_bt != bt) {
4781+
assert(bt == T_LONG && new_invar_bt == T_INT, "");
4782+
assert(invar_bt == bt, "");
4783+
new_invar = register_if_new(new ConvI2LNode(new_invar));
4784+
}
4785+
Node* add = AddNode::make(current_invar, new_invar, bt);
4786+
_invar = register_if_new(add);
4787+
}
4788+
47414789
//-----------------has_potential_dependence-----------------
47424790
// Check potential data dependence among all memory accesses.
47434791
// We require every two accesses (with at least one store) of
@@ -4774,7 +4822,7 @@ void SWPointer::print() {
47744822
_adr != nullptr ? _adr->_idx : 0,
47754823
_scale, _offset);
47764824
if (_invar != nullptr) {
4777-
tty->print(" invar: %c[%d] << [%d]", _negate_invar?'-':'+', _invar->_idx, _invar_scale->_idx);
4825+
tty->print(" invar: [%d]", _invar->_idx);
47784826
}
47794827
tty->cr();
47804828
#endif
@@ -4964,13 +5012,13 @@ void SWPointer::Tracer::scaled_iv_8(Node* n, SWPointer* tmp) {
49645012
}
49655013
}
49665014

4967-
void SWPointer::Tracer::scaled_iv_9(Node* n, int scale, int offset, Node* invar, bool negate_invar) {
5015+
void SWPointer::Tracer::scaled_iv_9(Node* n, int scale, int offset, Node* invar) {
49685016
if(_slp->is_trace_alignment()) {
49695017
print_depth(); tty->print_cr(" %d SWPointer::scaled_iv: Op_LShiftL PASSED, setting _scale = %d, _offset = %d", n->_idx, scale, offset);
49705018
print_depth(); tty->print_cr(" \\ SWPointer::scaled_iv: in(1) [%d] is scaled_iv_plus_offset, in(2) [%d] used to scale: _scale = %d, _offset = %d",
49715019
n->in(1)->_idx, n->in(2)->_idx, scale, offset);
49725020
if (invar != nullptr) {
4973-
print_depth(); tty->print_cr(" \\ SWPointer::scaled_iv: scaled invariant: %c[%d]", (negate_invar?'-':'+'), invar->_idx);
5021+
print_depth(); tty->print_cr(" \\ SWPointer::scaled_iv: scaled invariant: [%d]", invar->_idx);
49745022
}
49755023
inc_depth(); inc_depth();
49765024
print_depth(); n->in(1)->dump();
@@ -5022,7 +5070,7 @@ void SWPointer::Tracer::offset_plus_k_5(Node* n, Node* _invar) {
50225070

50235071
void SWPointer::Tracer::offset_plus_k_6(Node* n, Node* _invar, bool _negate_invar, int _offset) {
50245072
if(_slp->is_trace_alignment()) {
5025-
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_AddI PASSED, setting _negate_invar = %d, _invar = %d, _offset = %d",
5073+
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_AddI PASSED, setting _debug_negate_invar = %d, _invar = %d, _offset = %d",
50265074
n->_idx, _negate_invar, _invar->_idx, _offset);
50275075
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(2) is Con: ", n->in(2)->_idx); n->in(2)->dump();
50285076
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(1) is invariant: ", _invar->_idx); _invar->dump();
@@ -5031,7 +5079,7 @@ void SWPointer::Tracer::offset_plus_k_6(Node* n, Node* _invar, bool _negate_inva
50315079

50325080
void SWPointer::Tracer::offset_plus_k_7(Node* n, Node* _invar, bool _negate_invar, int _offset) {
50335081
if(_slp->is_trace_alignment()) {
5034-
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_AddI PASSED, setting _negate_invar = %d, _invar = %d, _offset = %d",
5082+
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_AddI PASSED, setting _debug_negate_invar = %d, _invar = %d, _offset = %d",
50355083
n->_idx, _negate_invar, _invar->_idx, _offset);
50365084
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(1) is Con: ", n->in(1)->_idx); n->in(1)->dump();
50375085
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(2) is invariant: ", _invar->_idx); _invar->dump();
@@ -5040,7 +5088,7 @@ void SWPointer::Tracer::offset_plus_k_7(Node* n, Node* _invar, bool _negate_inva
50405088

50415089
void SWPointer::Tracer::offset_plus_k_8(Node* n, Node* _invar, bool _negate_invar, int _offset) {
50425090
if(_slp->is_trace_alignment()) {
5043-
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_SubI is PASSED, setting _negate_invar = %d, _invar = %d, _offset = %d",
5091+
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_SubI is PASSED, setting _debug_negate_invar = %d, _invar = %d, _offset = %d",
50445092
n->_idx, _negate_invar, _invar->_idx, _offset);
50455093
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(2) is Con: ", n->in(2)->_idx); n->in(2)->dump();
50465094
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(1) is invariant: ", _invar->_idx); _invar->dump();
@@ -5049,15 +5097,15 @@ void SWPointer::Tracer::offset_plus_k_8(Node* n, Node* _invar, bool _negate_inva
50495097

50505098
void SWPointer::Tracer::offset_plus_k_9(Node* n, Node* _invar, bool _negate_invar, int _offset) {
50515099
if(_slp->is_trace_alignment()) {
5052-
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_SubI PASSED, setting _negate_invar = %d, _invar = %d, _offset = %d", n->_idx, _negate_invar, _invar->_idx, _offset);
5100+
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_SubI PASSED, setting _debug_negate_invar = %d, _invar = %d, _offset = %d", n->_idx, _negate_invar, _invar->_idx, _offset);
50535101
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(1) is Con: ", n->in(1)->_idx); n->in(1)->dump();
50545102
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(2) is invariant: ", _invar->_idx); _invar->dump();
50555103
}
50565104
}
50575105

50585106
void SWPointer::Tracer::offset_plus_k_10(Node* n, Node* _invar, bool _negate_invar, int _offset) {
50595107
if(_slp->is_trace_alignment()) {
5060-
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: PASSED, setting _negate_invar = %d, _invar = %d, _offset = %d", n->_idx, _negate_invar, _invar->_idx, _offset);
5108+
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: PASSED, setting _debug_negate_invar = %d, _invar = %d, _offset = %d", n->_idx, _negate_invar, _invar->_idx, _offset);
50615109
print_depth(); tty->print_cr(" \\ %d SWPointer::offset_plus_k: is invariant", n->_idx);
50625110
}
50635111
}

‎src/hotspot/share/opto/superword.hpp‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -655,8 +655,11 @@ class SWPointer : public ArenaObj {
655655
int _offset; // constant offset (in bytes)
656656

657657
Node* _invar; // invariant offset (in bytes), null if none
658-
bool _negate_invar; // if true then use: (0 - _invar)
659-
Node* _invar_scale; // multiplier for invariant
658+
#ifdef ASSERT
659+
Node* _debug_invar;
660+
bool _debug_negate_invar; // if true then use: (0 - _invar)
661+
Node* _debug_invar_scale; // multiplier for invariant
662+
#endif
660663

661664
Node_Stack* _nstack; // stack used to record a swpointer trace of variants
662665
bool _analyze_only; // Used in loop unrolling only for swpointer trace
@@ -698,17 +701,17 @@ class SWPointer : public ArenaObj {
698701
MemNode* mem() { return _mem; }
699702
int scale_in_bytes() { return _scale; }
700703
Node* invar() { return _invar; }
701-
bool negate_invar() { return _negate_invar; }
702-
Node* invar_scale() { return _invar_scale; }
703704
int offset_in_bytes() { return _offset; }
704705
int memory_size() { return _mem->memory_size(); }
705706
Node_Stack* node_stack() { return _nstack; }
706707

707708
// Comparable?
708709
bool invar_equals(SWPointer& q) {
709-
return (_invar == q._invar &&
710-
_invar_scale == q._invar_scale &&
711-
_negate_invar == q._negate_invar);
710+
assert(_debug_invar == NodeSentinel || q._debug_invar == NodeSentinel ||
711+
(_invar == q._invar) == (_debug_invar == q._debug_invar &&
712+
_debug_invar_scale == q._debug_invar_scale &&
713+
_debug_negate_invar == q._debug_negate_invar), "");
714+
return _invar == q._invar;
712715
}
713716

714717
int cmp(SWPointer& q) {
@@ -786,7 +789,7 @@ class SWPointer : public ArenaObj {
786789
void scaled_iv_6(Node* n, int scale);
787790
void scaled_iv_7(Node* n);
788791
void scaled_iv_8(Node* n, SWPointer* tmp);
789-
void scaled_iv_9(Node* n, int _scale, int _offset, Node* _invar, bool _negate_invar);
792+
void scaled_iv_9(Node* n, int _scale, int _offset, Node* _invar);
790793
void scaled_iv_10(Node* n);
791794

792795
void offset_plus_k_1(Node* n);
@@ -803,6 +806,12 @@ class SWPointer : public ArenaObj {
803806

804807
} _tracer;//TRacer;
805808
#endif
809+
810+
Node* maybe_negate_invar(bool negate, Node* invar);
811+
812+
void maybe_add_to_invar(Node* new_invar, bool negate);
813+
814+
Node* register_if_new(Node* n) const;
806815
};
807816

808817
#endif // SHARE_OPTO_SUPERWORD_HPP

0 commit comments

Comments
 (0)