Skip to content

8300257: C2: vectorization fails on some simple Memory Segment loops #12942

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
rwestrel wants to merge 9 commits into from
Closed

8300257: C2: vectorization fails on some simple Memory Segment loops #12942

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
dbf08c3
fix & test
rwestrel Feb 6, 2023
fcd4208
Merge branch 'master' into JDK-8300257
rwestrel Mar 14, 2023
cdcc181
NULL -> nullptr
rwestrel Mar 14, 2023
dc34591
Update src/hotspot/share/opto/superword.hpp
rwestrel Mar 17, 2023
4892f14
Update test/hotspot/jtreg/compiler/c2/irTests/TestVectorizationMultiI…
rwestrel Mar 17, 2023
d4d0765
Update test/hotspot/jtreg/compiler/c2/irTests/TestVectorizationMultiI…
rwestrel Mar 17, 2023
e071d65
Merge branch 'master' into JDK-8300257
rwestrel Mar 17, 2023
35b9498
review
rwestrel Mar 17, 2023
8a08447
Merge branch 'master' into JDK-8300257
rwestrel Apr 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 102 additions & 54 deletions src/hotspot/share/opto/superword.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#include "opto/mulnode.hpp"
#include "opto/opcodes.hpp"
#include "opto/opaquenode.hpp"
#include "opto/rootnode.hpp"
#include "opto/superword.hpp"
#include "opto/vectornode.hpp"
#include "opto/movenode.hpp"
Expand Down Expand Up @@ -201,7 +202,6 @@ bool SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
//------------------------------early unrolling analysis------------------------------
void SuperWord::unrolling_analysis(int &local_loop_unroll_factor) {
bool is_slp = true;
ResourceMark rm;
size_t ignored_size = lpt()->_body.size();
int *ignored_loop_nodes = NEW_RESOURCE_ARRAY(int, ignored_size);
Node_Stack nstack((int)ignored_size);
Expand Down Expand Up @@ -4248,19 +4248,7 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
invar = new ConvL2INode(invar);
_igvn.register_new_node_with_optimizer(invar);
}
Node* invar_scale = align_to_ref_p.invar_scale();
if (invar_scale != nullptr) {
invar = new LShiftINode(invar, invar_scale);
_igvn.register_new_node_with_optimizer(invar);
}
Node* aref = new URShiftINode(invar, log2_elt);
_igvn.register_new_node_with_optimizer(aref);
_phase->set_ctrl(aref, pre_ctrl);
if (align_to_ref_p.negate_invar()) {
e = new SubINode(e, aref);
} else {
e = new AddINode(e, aref);
}
e = new URShiftINode(invar, log2_elt);
_igvn.register_new_node_with_optimizer(e);
_phase->set_ctrl(e, pre_ctrl);
}
Expand Down Expand Up @@ -4440,9 +4428,11 @@ int SWPointer::Tracer::_depth = 0;
#endif
//----------------------------SWPointer------------------------
SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool analyze_only) :
_mem(mem), _slp(slp), _base(nullptr), _adr(nullptr),
_scale(0), _offset(0), _invar(nullptr), _negate_invar(false),
_invar_scale(nullptr),
_mem(mem), _slp(slp), _base(nullptr), _adr(nullptr),
_scale(0), _offset(0), _invar(nullptr),
#ifdef ASSERT
_debug_invar(nullptr), _debug_negate_invar(false), _debug_invar_scale(nullptr),
#endif
_nstack(nstack), _analyze_only(analyze_only),
_stack_idx(0)
#ifndef PRODUCT
Expand Down Expand Up @@ -4473,7 +4463,7 @@ SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool anal
NOT_PRODUCT(_tracer.ctor_2(adr);)

int i;
for (i = 0; i < 3; i++) {
for (i = 0; ; i++) {
NOT_PRODUCT(_tracer.ctor_3(adr, i);)

if (!scaled_iv_plus_offset(adr->in(AddPNode::Offset))) {
Expand Down Expand Up @@ -4509,9 +4499,11 @@ SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool anal
// Following is used to create a temporary object during
// the pattern match of an address expression.
SWPointer::SWPointer(SWPointer* p) :
_mem(p->_mem), _slp(p->_slp), _base(nullptr), _adr(nullptr),
_scale(0), _offset(0), _invar(nullptr), _negate_invar(false),
_invar_scale(nullptr),
_mem(p->_mem), _slp(p->_slp), _base(nullptr), _adr(nullptr),
_scale(0), _offset(0), _invar(nullptr),
#ifdef ASSERT
_debug_invar(nullptr), _debug_negate_invar(false), _debug_invar_scale(nullptr),
#endif
_nstack(p->_nstack), _analyze_only(p->_analyze_only),
_stack_idx(p->_stack_idx)
#ifndef PRODUCT
Expand Down Expand Up @@ -4625,7 +4617,7 @@ bool SWPointer::scaled_iv(Node* n) {
return true;
}
} else if (opc == Op_LShiftL && n->in(2)->is_Con()) {
if (!has_iv() && _invar == nullptr) {
if (!has_iv()) {
// Need to preserve the current _offset value, so
// create a temporary object for this expression subtree.
// Hacky, so should re-engineer the address pattern match.
Expand All @@ -4637,12 +4629,15 @@ bool SWPointer::scaled_iv(Node* n) {
int scale = n->in(2)->get_int();
_scale = tmp._scale << scale;
_offset += tmp._offset << scale;
_invar = tmp._invar;
if (_invar != nullptr) {
_negate_invar = tmp._negate_invar;
_invar_scale = n->in(2);
if (tmp._invar != nullptr) {
BasicType bt = tmp._invar->bottom_type()->basic_type();
assert(bt == T_INT || bt == T_LONG, "");
maybe_add_to_invar(register_if_new(LShiftNode::make(tmp._invar, n->in(2), bt)), false);
#ifdef ASSERT
_debug_invar_scale = n->in(2);
#endif
}
NOT_PRODUCT(_tracer.scaled_iv_9(n, _scale, _offset, _invar, _negate_invar);)
NOT_PRODUCT(_tracer.scaled_iv_9(n, _scale, _offset, _invar);)
return true;
}
}
Expand Down Expand Up @@ -4676,41 +4671,34 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
NOT_PRODUCT(_tracer.offset_plus_k_4(n);)
return false;
}
if (_invar != nullptr) { // already has an invariant
NOT_PRODUCT(_tracer.offset_plus_k_5(n, _invar);)
return false;
}
assert((_debug_invar == nullptr) == (_invar == nullptr), "");

if (_analyze_only && is_loop_member(n)) {
_nstack->push(n, _stack_idx++);
}
if (opc == Op_AddI) {
if (n->in(2)->is_Con() && invariant(n->in(1))) {
_negate_invar = negate;
_invar = n->in(1);
maybe_add_to_invar(n->in(1), negate);
_offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
NOT_PRODUCT(_tracer.offset_plus_k_6(n, _invar, _negate_invar, _offset);)
NOT_PRODUCT(_tracer.offset_plus_k_6(n, _invar, negate, _offset);)
return true;
} else if (n->in(1)->is_Con() && invariant(n->in(2))) {
_offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int();
_negate_invar = negate;
_invar = n->in(2);
NOT_PRODUCT(_tracer.offset_plus_k_7(n, _invar, _negate_invar, _offset);)
maybe_add_to_invar(n->in(2), negate);
NOT_PRODUCT(_tracer.offset_plus_k_7(n, _invar, negate, _offset);)
return true;
}
}
if (opc == Op_SubI) {
if (n->in(2)->is_Con() && invariant(n->in(1))) {
_negate_invar = negate;
_invar = n->in(1);
maybe_add_to_invar(n->in(1), negate);
_offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
NOT_PRODUCT(_tracer.offset_plus_k_8(n, _invar, _negate_invar, _offset);)
NOT_PRODUCT(_tracer.offset_plus_k_8(n, _invar, negate, _offset);)
return true;
} else if (n->in(1)->is_Con() && invariant(n->in(2))) {
_offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int();
_negate_invar = !negate;
_invar = n->in(2);
NOT_PRODUCT(_tracer.offset_plus_k_9(n, _invar, _negate_invar, _offset);)
maybe_add_to_invar(n->in(2), !negate);
NOT_PRODUCT(_tracer.offset_plus_k_9(n, _invar, !negate, _offset);)
return true;
}
}
Expand All @@ -4727,9 +4715,8 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
}
// Check if 'n' can really be used as invariant (not in main loop and dominating the pre loop).
if (invariant(n)) {
_negate_invar = negate;
_invar = n;
NOT_PRODUCT(_tracer.offset_plus_k_10(n, _invar, _negate_invar, _offset);)
maybe_add_to_invar(n, negate);
NOT_PRODUCT(_tracer.offset_plus_k_10(n, _invar, negate, _offset);)
return true;
}
}
Expand All @@ -4738,6 +4725,67 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
return false;
}

Node* SWPointer::maybe_negate_invar(bool negate, Node* invar) {
#ifdef ASSERT
_debug_negate_invar = negate;
#endif
if (negate) {
BasicType bt = invar->bottom_type()->basic_type();
assert(bt == T_INT || bt == T_LONG, "");
PhaseIterGVN& igvn = phase()->igvn();
Node* zero = igvn.zerocon(bt);
phase()->set_ctrl(zero, phase()->C->root());
Node* sub = SubNode::make(zero, invar, bt);
invar = register_if_new(sub);
}
return invar;
}

Node* SWPointer::register_if_new(Node* n) const {
PhaseIterGVN& igvn = phase()->igvn();
Node* prev = igvn.hash_find_insert(n);
if (prev != nullptr) {
n->destruct(&igvn);
n = prev;
} else {
Node* c = phase()->get_early_ctrl(n);
phase()->register_new_node(n, c);
}
return n;
}

void SWPointer::maybe_add_to_invar(Node* new_invar, bool negate) {
new_invar = maybe_negate_invar(negate, new_invar);
if (_invar == nullptr) {
_invar = new_invar;
#ifdef ASSERT
_debug_invar = new_invar;
#endif
return;
}
#ifdef ASSERT
_debug_invar = NodeSentinel;
#endif
BasicType new_invar_bt = new_invar->bottom_type()->basic_type();
assert(new_invar_bt == T_INT || new_invar_bt == T_LONG, "");
BasicType invar_bt = _invar->bottom_type()->basic_type();
assert(invar_bt == T_INT || invar_bt == T_LONG, "");

BasicType bt = (new_invar_bt == T_LONG || invar_bt == T_LONG) ? T_LONG : T_INT;
Node* current_invar = _invar;
if (invar_bt != bt) {
assert(bt == T_LONG && invar_bt == T_INT, "");
assert(new_invar_bt == bt, "");
current_invar = register_if_new(new ConvI2LNode(current_invar));
} else if (new_invar_bt != bt) {
assert(bt == T_LONG && new_invar_bt == T_INT, "");
assert(invar_bt == bt, "");
new_invar = register_if_new(new ConvI2LNode(new_invar));
}
Node* add = AddNode::make(current_invar, new_invar, bt);
_invar = register_if_new(add);
}

//-----------------has_potential_dependence-----------------
// Check potential data dependence among all memory accesses.
// We require every two accesses (with at least one store) of
Expand Down Expand Up @@ -4774,7 +4822,7 @@ void SWPointer::print() {
_adr != nullptr ? _adr->_idx : 0,
_scale, _offset);
if (_invar != nullptr) {
tty->print(" invar: %c[%d] << [%d]", _negate_invar?'-':'+', _invar->_idx, _invar_scale->_idx);
tty->print(" invar: [%d]", _invar->_idx);
}
tty->cr();
#endif
Expand Down Expand Up @@ -4964,13 +5012,13 @@ void SWPointer::Tracer::scaled_iv_8(Node* n, SWPointer* tmp) {
}
}

void SWPointer::Tracer::scaled_iv_9(Node* n, int scale, int offset, Node* invar, bool negate_invar) {
void SWPointer::Tracer::scaled_iv_9(Node* n, int scale, int offset, Node* invar) {
if(_slp->is_trace_alignment()) {
print_depth(); tty->print_cr(" %d SWPointer::scaled_iv: Op_LShiftL PASSED, setting _scale = %d, _offset = %d", n->_idx, scale, offset);
print_depth(); tty->print_cr(" \\ SWPointer::scaled_iv: in(1) [%d] is scaled_iv_plus_offset, in(2) [%d] used to scale: _scale = %d, _offset = %d",
n->in(1)->_idx, n->in(2)->_idx, scale, offset);
if (invar != nullptr) {
print_depth(); tty->print_cr(" \\ SWPointer::scaled_iv: scaled invariant: %c[%d]", (negate_invar?'-':'+'), invar->_idx);
print_depth(); tty->print_cr(" \\ SWPointer::scaled_iv: scaled invariant: [%d]", invar->_idx);
}
inc_depth(); inc_depth();
print_depth(); n->in(1)->dump();
Expand Down Expand Up @@ -5022,7 +5070,7 @@ void SWPointer::Tracer::offset_plus_k_5(Node* n, Node* _invar) {

void SWPointer::Tracer::offset_plus_k_6(Node* n, Node* _invar, bool _negate_invar, int _offset) {
if(_slp->is_trace_alignment()) {
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_AddI PASSED, setting _negate_invar = %d, _invar = %d, _offset = %d",
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_AddI PASSED, setting _debug_negate_invar = %d, _invar = %d, _offset = %d",
n->_idx, _negate_invar, _invar->_idx, _offset);
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(2) is Con: ", n->in(2)->_idx); n->in(2)->dump();
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(1) is invariant: ", _invar->_idx); _invar->dump();
Expand All @@ -5031,7 +5079,7 @@ void SWPointer::Tracer::offset_plus_k_6(Node* n, Node* _invar, bool _negate_inva

void SWPointer::Tracer::offset_plus_k_7(Node* n, Node* _invar, bool _negate_invar, int _offset) {
if(_slp->is_trace_alignment()) {
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_AddI PASSED, setting _negate_invar = %d, _invar = %d, _offset = %d",
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_AddI PASSED, setting _debug_negate_invar = %d, _invar = %d, _offset = %d",
n->_idx, _negate_invar, _invar->_idx, _offset);
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(1) is Con: ", n->in(1)->_idx); n->in(1)->dump();
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(2) is invariant: ", _invar->_idx); _invar->dump();
Expand All @@ -5040,7 +5088,7 @@ void SWPointer::Tracer::offset_plus_k_7(Node* n, Node* _invar, bool _negate_inva

void SWPointer::Tracer::offset_plus_k_8(Node* n, Node* _invar, bool _negate_invar, int _offset) {
if(_slp->is_trace_alignment()) {
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_SubI is PASSED, setting _negate_invar = %d, _invar = %d, _offset = %d",
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_SubI is PASSED, setting _debug_negate_invar = %d, _invar = %d, _offset = %d",
n->_idx, _negate_invar, _invar->_idx, _offset);
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(2) is Con: ", n->in(2)->_idx); n->in(2)->dump();
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(1) is invariant: ", _invar->_idx); _invar->dump();
Expand All @@ -5049,15 +5097,15 @@ void SWPointer::Tracer::offset_plus_k_8(Node* n, Node* _invar, bool _negate_inva

void SWPointer::Tracer::offset_plus_k_9(Node* n, Node* _invar, bool _negate_invar, int _offset) {
if(_slp->is_trace_alignment()) {
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_SubI PASSED, setting _negate_invar = %d, _invar = %d, _offset = %d", n->_idx, _negate_invar, _invar->_idx, _offset);
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: Op_SubI PASSED, setting _debug_negate_invar = %d, _invar = %d, _offset = %d", n->_idx, _negate_invar, _invar->_idx, _offset);
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(1) is Con: ", n->in(1)->_idx); n->in(1)->dump();
print_depth(); tty->print(" \\ %d SWPointer::offset_plus_k: in(2) is invariant: ", _invar->_idx); _invar->dump();
}
}

void SWPointer::Tracer::offset_plus_k_10(Node* n, Node* _invar, bool _negate_invar, int _offset) {
if(_slp->is_trace_alignment()) {
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: PASSED, setting _negate_invar = %d, _invar = %d, _offset = %d", n->_idx, _negate_invar, _invar->_idx, _offset);
print_depth(); tty->print_cr(" %d SWPointer::offset_plus_k: PASSED, setting _debug_negate_invar = %d, _invar = %d, _offset = %d", n->_idx, _negate_invar, _invar->_idx, _offset);
print_depth(); tty->print_cr(" \\ %d SWPointer::offset_plus_k: is invariant", n->_idx);
}
}
Expand Down
25 changes: 17 additions & 8 deletions src/hotspot/share/opto/superword.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -655,8 +655,11 @@ class SWPointer : public ArenaObj {
int _offset; // constant offset (in bytes)

Node* _invar; // invariant offset (in bytes), null if none
bool _negate_invar; // if true then use: (0 - _invar)
Node* _invar_scale; // multiplier for invariant
#ifdef ASSERT
Node* _debug_invar;
bool _debug_negate_invar; // if true then use: (0 - _invar)
Node* _debug_invar_scale; // multiplier for invariant
#endif

Node_Stack* _nstack; // stack used to record a swpointer trace of variants
bool _analyze_only; // Used in loop unrolling only for swpointer trace
Expand Down Expand Up @@ -698,17 +701,17 @@ class SWPointer : public ArenaObj {
MemNode* mem() { return _mem; }
int scale_in_bytes() { return _scale; }
Node* invar() { return _invar; }
bool negate_invar() { return _negate_invar; }
Node* invar_scale() { return _invar_scale; }
int offset_in_bytes() { return _offset; }
int memory_size() { return _mem->memory_size(); }
Node_Stack* node_stack() { return _nstack; }

// Comparable?
bool invar_equals(SWPointer& q) {
return (_invar == q._invar &&
_invar_scale == q._invar_scale &&
_negate_invar == q._negate_invar);
assert(_debug_invar == NodeSentinel || q._debug_invar == NodeSentinel ||
(_invar == q._invar) == (_debug_invar == q._debug_invar &&
_debug_invar_scale == q._debug_invar_scale &&
_debug_negate_invar == q._debug_negate_invar), "");
return _invar == q._invar;
}

int cmp(SWPointer& q) {
Expand Down Expand Up @@ -786,7 +789,7 @@ class SWPointer : public ArenaObj {
void scaled_iv_6(Node* n, int scale);
void scaled_iv_7(Node* n);
void scaled_iv_8(Node* n, SWPointer* tmp);
void scaled_iv_9(Node* n, int _scale, int _offset, Node* _invar, bool _negate_invar);
void scaled_iv_9(Node* n, int _scale, int _offset, Node* _invar);
void scaled_iv_10(Node* n);

void offset_plus_k_1(Node* n);
Expand All @@ -803,6 +806,12 @@ class SWPointer : public ArenaObj {

} _tracer;//TRacer;
#endif

Node* maybe_negate_invar(bool negate, Node* invar);

void maybe_add_to_invar(Node* new_invar, bool negate);

Node* register_if_new(Node* n) const;
};

#endif // SHARE_OPTO_SUPERWORD_HPP
Loading