Skip to content

8302673: [SuperWord] MaxReduction and MinReduction should vectorize for int #13924

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 30 commits into from
Closed

8302673: [SuperWord] MaxReduction and MinReduction should vectorize for int #13924

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
be1888f
Clean up MinI/MaxI Ideal function
robcasloz Apr 12, 2023
9b218df
Add basic test from https://github.com/openjdk/jdk/pull/13260
robcasloz Apr 12, 2023
cd84227
Added more Max test cases with swapped inputs
robcasloz Apr 14, 2023
3973102
Handle all four max-of-max cases without canonicalization
robcasloz Apr 14, 2023
c56efa2
Extract MaxINode::Ideal() and MinINode::Ideal() into MaxNode::IdealI()
robcasloz May 5, 2023
2149d34
Flatten nested if
robcasloz May 5, 2023
7adf075
Re-add comment
robcasloz May 5, 2023
872bcbe
Simplify
robcasloz May 5, 2023
d7c7987
Refactor
robcasloz May 8, 2023
89cebb1
Add some comments
robcasloz May 8, 2023
d53cb53
Complement idealization tests with negative ones
robcasloz May 10, 2023
3c12095
Uncomment tests
robcasloz May 10, 2023
83e7630
Update copyright header
robcasloz May 10, 2023
9fd482b
Refine comments
robcasloz May 10, 2023
80b100e
Merge branch 'master' into JDK-8302673
robcasloz May 22, 2023
6308044
Merge branch 'master' into JDK-8302673
robcasloz May 23, 2023
c736c0a
Randomize array values in min/max test computation
robcasloz May 23, 2023
8f8faf2
Make auxiliary add operand extraction function return a tuple
robcasloz May 28, 2023
3aba9bc
Refactor idealization and extracted Identity transformation for clarity
robcasloz May 28, 2023
a769913
Merge branch 'master' into JDK-8302673
robcasloz May 28, 2023
c3ea6f7
Defer op(x, x) to constant/identity propagation early
robcasloz May 30, 2023
a6db3cc
Merge branch 'master' into JDK-8302673
robcasloz May 30, 2023
29922ea
Extract MinI/MaxI construction; pass around ConstAddOperands instead …
robcasloz May 31, 2023
3adee22
Add tests to exercise the case without inner additions
robcasloz May 31, 2023
efb8ac0
Handle case without inner additions by restoring 'as_add_with_constan…
robcasloz May 31, 2023
3b4d799
Complete test battery with remaining no-add cases
robcasloz Jun 1, 2023
c26f585
Revert extraction of min/max building
robcasloz Jun 1, 2023
5a96171
Abort idealization if any of the adds has a TOP input
robcasloz Jun 1, 2023
f4837ac
Merge branch 'master' into JDK-8302673
robcasloz Jun 2, 2023
cfcc16f
Re-apply extraction of min/max building after JDK-8309295
robcasloz Jun 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
279 changes: 113 additions & 166 deletions src/hotspot/share/opto/addnode.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1030,6 +1030,14 @@ const Type* XorLNode::Value(PhaseGVN* phase) const {
return AddNode::Value(phase);
}

Node* build_min_max_int(Node* a, Node* b, bool is_max) {
if (is_max) {
return new MaxINode(a, b);
} else {
return new MinINode(a, b);
}
}

Node* MaxNode::build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, const Type* t, PhaseGVN& gvn) {
bool is_int = gvn.type(a)->isa_int();
assert(is_int || gvn.type(a)->isa_long(), "int or long inputs");
Expand All @@ -1044,13 +1052,7 @@ Node* MaxNode::build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, co
}
Node* res = nullptr;
if (is_int && !is_unsigned) {
Node* res_new = nullptr;
if (is_max) {
res_new = new MaxINode(a, b);
} else {
res_new = new MinINode(a, b);
}
res = gvn.transform(res_new);
res = gvn.transform(build_min_max_int(a, b, is_max));
assert(gvn.type(res)->is_int()->_lo >= t->is_int()->_lo && gvn.type(res)->is_int()->_hi <= t->is_int()->_hi, "type doesn't match");
} else {
Node* cmp = nullptr;
Expand Down Expand Up @@ -1096,185 +1098,130 @@ Node* MaxNode::build_min_max_diff_with_zero(Node* a, Node* b, bool is_max, const
return res;
}

//=============================================================================
//------------------------------add_ring---------------------------------------
// Supplied function returns the sum of the inputs.
const Type *MaxINode::add_ring( const Type *t0, const Type *t1 ) const {
const TypeInt *r0 = t0->is_int(); // Handy access
const TypeInt *r1 = t1->is_int();

// Otherwise just MAX them bits.
return TypeInt::make( MAX2(r0->_lo,r1->_lo), MAX2(r0->_hi,r1->_hi), MAX2(r0->_widen,r1->_widen) );
}

// Check if addition of an integer with type 't' and a constant 'c' can overflow
// Check if addition of an integer with type 't' and a constant 'c' can overflow.
static bool can_overflow(const TypeInt* t, jint c) {
jint t_lo = t->_lo;
jint t_hi = t->_hi;
return ((c < 0 && (java_add(t_lo, c) > t_lo)) ||
(c > 0 && (java_add(t_hi, c) < t_hi)));
}

// Ideal transformations for MaxINode
Node* MaxINode::Ideal(PhaseGVN* phase, bool can_reshape) {
// Force a right-spline graph
Node* l = in(1);
Node* r = in(2);
// Transform MaxI1(MaxI2(a, b), c) into MaxI1(a, MaxI2(b, c))
// to force a right-spline graph for the rest of MaxINode::Ideal().
if (l->Opcode() == Op_MaxI) {
assert(l != l->in(1), "dead loop in MaxINode::Ideal");
r = phase->transform(new MaxINode(l->in(2), r));
l = l->in(1);
set_req_X(1, l, phase);
set_req_X(2, r, phase);
return this;
}

// Get left input & constant
Node* x = l;
jint x_off = 0;
if (x->Opcode() == Op_AddI && // Check for "x+c0" and collect constant
x->in(2)->is_Con()) {
const Type* t = x->in(2)->bottom_type();
if (t == Type::TOP) return nullptr; // No progress
x_off = t->is_int()->get_con();
x = x->in(1);
}

// Scan a right-spline-tree for MAXs
Node* y = r;
jint y_off = 0;
// Check final part of MAX tree
if (y->Opcode() == Op_AddI && // Check for "y+c1" and collect constant
y->in(2)->is_Con()) {
const Type* t = y->in(2)->bottom_type();
if (t == Type::TOP) return nullptr; // No progress
y_off = t->is_int()->get_con();
y = y->in(1);
}
if (x->_idx > y->_idx && r->Opcode() != Op_MaxI) {
swap_edges(1, 2);
return this;
// Let <x, x_off> = x_operands and <y, y_off> = y_operands.
// If x == y and neither add(x, x_off) nor add(y, y_off) overflow, return
// add(x, op(x_off, y_off)). Otherwise, return nullptr.
Node* MaxNode::extract_add(PhaseGVN* phase, ConstAddOperands x_operands, ConstAddOperands y_operands) {
Node* x = x_operands.first;
Node* y = y_operands.first;
int opcode = Opcode();
assert(opcode == Op_MaxI || opcode == Op_MinI, "Unexpected opcode");
const TypeInt* tx = phase->type(x)->isa_int();
jint x_off = x_operands.second;
jint y_off = y_operands.second;
if (x == y && tx != nullptr &&
!can_overflow(tx, x_off) &&
!can_overflow(tx, y_off)) {
jint c = opcode == Op_MinI ? MIN2(x_off, y_off) : MAX2(x_off, y_off);
return new AddINode(x, phase->intcon(c));
}
return nullptr;
}

const TypeInt* tx = phase->type(x)->isa_int();
// Try to cast n as an integer addition with a constant. Return:
// <x, C>, if n == add(x, C), where 'C' is a non-TOP constant;
// <nullptr, 0>, if n == add(x, C), where 'C' is a TOP constant; or
// <n, 0>, otherwise.
static ConstAddOperands as_add_with_constant(Node* n) {
if (n->Opcode() != Op_AddI) {
return ConstAddOperands(n, 0);
}
Node* x = n->in(1);
Node* c = n->in(2);
if (!c->is_Con()) {
return ConstAddOperands(n, 0);
}
const Type* c_type = c->bottom_type();
if (c_type == Type::TOP) {
return ConstAddOperands(nullptr, 0);
}
return ConstAddOperands(x, c_type->is_int()->get_con());
}

if (r->Opcode() == Op_MaxI) {
assert(r != r->in(2), "dead loop in MaxINode::Ideal");
y = r->in(1);
// Check final part of MAX tree
if (y->Opcode() == Op_AddI &&// Check for "y+c1" and collect constant
y->in(2)->is_Con()) {
const Type* t = y->in(2)->bottom_type();
if (t == Type::TOP) return nullptr; // No progress
y_off = t->is_int()->get_con();
y = y->in(1);
Node* MaxNode::IdealI(PhaseGVN* phase, bool can_reshape) {
int opcode = Opcode();
assert(opcode == Op_MinI || opcode == Op_MaxI, "Unexpected opcode");
// Try to transform the following pattern, in any of its four possible
// permutations induced by op's commutativity:
// op(op(add(inner, inner_off), inner_other), add(outer, outer_off))
// into
// op(add(inner, op(inner_off, outer_off)), inner_other),
// where:
// op is either MinI or MaxI, and
// inner == outer, and
// the additions cannot overflow.
for (uint inner_op_index = 1; inner_op_index <= 2; inner_op_index++) {
if (in(inner_op_index)->Opcode() != opcode) {
continue;
}

if (x->_idx > y->_idx)
return new MaxINode(r->in(1), phase->transform(new MaxINode(l, r->in(2))));

// Transform MAX2(x + c0, MAX2(x + c1, z)) into MAX2(x + MAX2(c0, c1), z)
// if x == y and the additions can't overflow.
if (x == y && tx != nullptr &&
!can_overflow(tx, x_off) &&
!can_overflow(tx, y_off)) {
return new MaxINode(phase->transform(new AddINode(x, phase->intcon(MAX2(x_off, y_off)))), r->in(2));
Node* outer_add = in(inner_op_index == 1 ? 2 : 1);
ConstAddOperands outer_add_operands = as_add_with_constant(outer_add);
if (outer_add_operands.first == nullptr) {
return nullptr; // outer_add has a TOP input, no need to continue.
}
} else {
// Transform MAX2(x + c0, y + c1) into x + MAX2(c0, c1)
// if x == y and the additions can't overflow.
if (x == y && tx != nullptr &&
!can_overflow(tx, x_off) &&
!can_overflow(tx, y_off)) {
return new AddINode(x, phase->intcon(MAX2(x_off, y_off)));
// One operand is a MinI/MaxI and the other is an integer addition with
// constant. Test the operands of the inner MinI/MaxI.
for (uint inner_add_index = 1; inner_add_index <= 2; inner_add_index++) {
Node* inner_op = in(inner_op_index);
Node* inner_add = inner_op->in(inner_add_index);
ConstAddOperands inner_add_operands = as_add_with_constant(inner_add);
if (inner_add_operands.first == nullptr) {
return nullptr; // inner_add has a TOP input, no need to continue.
}
// Try to extract the inner add.
Node* add_extracted = extract_add(phase, inner_add_operands, outer_add_operands);
if (add_extracted == nullptr) {
continue;
}
Node* add_transformed = phase->transform(add_extracted);
Node* inner_other = inner_op->in(inner_add_index == 1 ? 2 : 1);
return build_min_max_int(add_transformed, inner_other, opcode == Op_MaxI);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did something prevent you from directly using MaxNode::build_min_max?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Technically I think that should be possible, but I find that extracting the core logic into a separate function is more readable (makes it straightforward to understand the effect of the call in MaxNode::IdealI()) and efficient (avoids a redundant application of PhaseGVN::transform() to the newly created MinI/MaxI nodes).

}
}
return nullptr;
// Try to transform
// op(add(x, x_off), add(y, y_off))
// into
// add(x, op(x_off, y_off)),
// where:
// op is either MinI or MaxI, and
// inner == outer, and
// the additions cannot overflow.
ConstAddOperands xC = as_add_with_constant(in(1));
ConstAddOperands yC = as_add_with_constant(in(2));
if (xC.first == nullptr || yC.first == nullptr) return nullptr;
return extract_add(phase, xC, yC);
}

// Ideal transformations for MaxINode
Node* MaxINode::Ideal(PhaseGVN* phase, bool can_reshape) {
return IdealI(phase, can_reshape);
}

//=============================================================================
//------------------------------add_ring---------------------------------------
// Supplied function returns the sum of the inputs.
const Type *MaxINode::add_ring( const Type *t0, const Type *t1 ) const {
const TypeInt *r0 = t0->is_int(); // Handy access
const TypeInt *r1 = t1->is_int();

// Otherwise just MAX them bits.
return TypeInt::make( MAX2(r0->_lo,r1->_lo), MAX2(r0->_hi,r1->_hi), MAX2(r0->_widen,r1->_widen) );
}

//=============================================================================
//------------------------------Idealize---------------------------------------
// MINs show up in range-check loop limit calculations. Look for
// "MIN2(x+c0,MIN2(y,x+c1))". Pick the smaller constant: "MIN2(x+c0,y)"
Node *MinINode::Ideal(PhaseGVN *phase, bool can_reshape) {
Node *progress = nullptr;
// Force a right-spline graph
Node *l = in(1);
Node *r = in(2);
// Transform MinI1( MinI2(a,b), c) into MinI1( a, MinI2(b,c) )
// to force a right-spline graph for the rest of MinINode::Ideal().
if( l->Opcode() == Op_MinI ) {
assert( l != l->in(1), "dead loop in MinINode::Ideal" );
r = phase->transform(new MinINode(l->in(2),r));
l = l->in(1);
set_req_X(1, l, phase);
set_req_X(2, r, phase);
return this;
}

// Get left input & constant
Node *x = l;
jint x_off = 0;
if( x->Opcode() == Op_AddI && // Check for "x+c0" and collect constant
x->in(2)->is_Con() ) {
const Type *t = x->in(2)->bottom_type();
if( t == Type::TOP ) return nullptr; // No progress
x_off = t->is_int()->get_con();
x = x->in(1);
}

// Scan a right-spline-tree for MINs
Node *y = r;
jint y_off = 0;
// Check final part of MIN tree
if( y->Opcode() == Op_AddI && // Check for "y+c1" and collect constant
y->in(2)->is_Con() ) {
const Type *t = y->in(2)->bottom_type();
if( t == Type::TOP ) return nullptr; // No progress
y_off = t->is_int()->get_con();
y = y->in(1);
}
if( x->_idx > y->_idx && r->Opcode() != Op_MinI ) {
swap_edges(1, 2);
return this;
}

const TypeInt* tx = phase->type(x)->isa_int();

if( r->Opcode() == Op_MinI ) {
assert( r != r->in(2), "dead loop in MinINode::Ideal" );
y = r->in(1);
// Check final part of MIN tree
if( y->Opcode() == Op_AddI &&// Check for "y+c1" and collect constant
y->in(2)->is_Con() ) {
const Type *t = y->in(2)->bottom_type();
if( t == Type::TOP ) return nullptr; // No progress
y_off = t->is_int()->get_con();
y = y->in(1);
}

if( x->_idx > y->_idx )
return new MinINode(r->in(1),phase->transform(new MinINode(l,r->in(2))));

// Transform MIN2(x + c0, MIN2(x + c1, z)) into MIN2(x + MIN2(c0, c1), z)
// if x == y and the additions can't overflow.
if (x == y && tx != nullptr &&
!can_overflow(tx, x_off) &&
!can_overflow(tx, y_off)) {
return new MinINode(phase->transform(new AddINode(x, phase->intcon(MIN2(x_off, y_off)))), r->in(2));
}
} else {
// Transform MIN2(x + c0, y + c1) into x + MIN2(c0, c1)
// if x == y and the additions can't overflow.
if (x == y && tx != nullptr &&
!can_overflow(tx, x_off) &&
!can_overflow(tx, y_off)) {
return new AddINode(x,phase->intcon(MIN2(x_off,y_off)));
}
}
return nullptr;
Node* MinINode::Ideal(PhaseGVN* phase, bool can_reshape) {
return IdealI(phase, can_reshape);
}

//------------------------------add_ring---------------------------------------
Expand Down
4 changes: 4 additions & 0 deletions src/hotspot/share/opto/addnode.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@
#include "opto/node.hpp"
#include "opto/opcodes.hpp"
#include "opto/type.hpp"
#include "utilities/pair.hpp"

// Portions of code courtesy of Clifford Click

class PhaseTransform;
typedef const Pair<Node*, jint> ConstAddOperands;

//------------------------------AddNode----------------------------------------
// Classic Add functionality. This covers all the usual 'add' behaviors for
Expand Down Expand Up @@ -252,12 +254,14 @@ class MaxNode : public AddNode {
private:
static Node* build_min_max(Node* a, Node* b, bool is_max, bool is_unsigned, const Type* t, PhaseGVN& gvn);
static Node* build_min_max_diff_with_zero(Node* a, Node* b, bool is_max, const Type* t, PhaseGVN& gvn);
Node* extract_add(PhaseGVN* phase, ConstAddOperands x_operands, ConstAddOperands y_operands);

public:
MaxNode( Node *in1, Node *in2 ) : AddNode(in1,in2) {}
virtual int Opcode() const = 0;
virtual int max_opcode() const = 0;
virtual int min_opcode() const = 0;
Node* IdealI(PhaseGVN* phase, bool can_reshape);

static Node* unsigned_max(Node* a, Node* b, const Type* t, PhaseGVN& gvn) {
return build_min_max(a, b, true, true, t, gvn);
Expand Down
Loading