Skip to content

Commit 14c20bc

Browse files
committed
8283187: C2: loop candidate for superword not always unrolled fully if superword fails
Reviewed-by: thartmann, chagedorn
1 parent 1c4f5fc commit 14c20bc

File tree

5 files changed

+129
-45
lines changed

5 files changed

+129
-45
lines changed

src/hotspot/share/opto/loopnode.cpp

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4568,7 +4568,14 @@ void PhaseIdealLoop::build_and_optimize() {
45684568
sw.transform_loop(lpt, true);
45694569
}
45704570
} else if (cl->is_main_loop()) {
4571-
sw.transform_loop(lpt, true);
4571+
if (!sw.transform_loop(lpt, true)) {
4572+
// Instigate more unrolling for optimization when vectorization fails.
4573+
if (cl->has_passed_slp()) {
4574+
C->set_major_progress();
4575+
cl->set_notpassed_slp();
4576+
cl->mark_do_unroll_only();
4577+
}
4578+
}
45724579
}
45734580
}
45744581
}

src/hotspot/share/opto/superword.cpp

Lines changed: 51 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -95,38 +95,48 @@ SuperWord::SuperWord(PhaseIdealLoop* phase) :
9595
static const bool _do_vector_loop_experimental = false; // Experimental vectorization which uses data from loop unrolling.
9696

9797
//------------------------------transform_loop---------------------------
98-
void SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
98+
bool SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
9999
assert(UseSuperWord, "should be");
100100
// SuperWord only works with power of two vector sizes.
101101
int vector_width = Matcher::vector_width_in_bytes(T_BYTE);
102102
if (vector_width < 2 || !is_power_of_2(vector_width)) {
103-
return;
103+
return false;
104104
}
105105

106106
assert(lpt->_head->is_CountedLoop(), "must be");
107107
CountedLoopNode *cl = lpt->_head->as_CountedLoop();
108108

109-
if (!cl->is_valid_counted_loop(T_INT)) return; // skip malformed counted loop
109+
if (!cl->is_valid_counted_loop(T_INT)) {
110+
return false; // skip malformed counted loop
111+
}
110112

111113
bool post_loop_allowed = (PostLoopMultiversioning && Matcher::has_predicated_vectors() && cl->is_post_loop());
112114
if (post_loop_allowed) {
113-
if (cl->is_reduction_loop()) return; // no predication mapping
115+
if (cl->is_reduction_loop()) {
116+
return false; // no predication mapping
117+
}
114118
Node *limit = cl->limit();
115-
if (limit->is_Con()) return; // non constant limits only
119+
if (limit->is_Con()) {
120+
return false; // non constant limits only
121+
}
116122
// Now check the limit for expressions we do not handle
117123
if (limit->is_Add()) {
118124
Node *in2 = limit->in(2);
119125
if (in2->is_Con()) {
120126
int val = in2->get_int();
121127
// should not try to program these cases
122-
if (val < 0) return;
128+
if (val < 0) {
129+
return false;
130+
}
123131
}
124132
}
125133
}
126134

127135
// skip any loop that has not been assigned max unroll by analysis
128136
if (do_optimization) {
129-
if (SuperWordLoopUnrollAnalysis && cl->slp_max_unroll() == 0) return;
137+
if (SuperWordLoopUnrollAnalysis && cl->slp_max_unroll() == 0) {
138+
return false;
139+
}
130140
}
131141

132142
// Check for no control flow in body (other than exit)
@@ -141,28 +151,32 @@ void SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
141151
lpt->dump_head();
142152
}
143153
#endif
144-
return;
154+
return false;
145155
}
146156

147157
// Make sure the are no extra control users of the loop backedge
148158
if (cl->back_control()->outcnt() != 1) {
149-
return;
159+
return false;
150160
}
151161

152162
// Skip any loops already optimized by slp
153-
if (cl->is_vectorized_loop()) return;
163+
if (cl->is_vectorized_loop()) {
164+
return false;
165+
}
154166

155-
if (cl->is_unroll_only()) return;
167+
if (cl->is_unroll_only()) {
168+
return false;
169+
}
156170

157171
if (cl->is_main_loop()) {
158172
// Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit))))
159173
CountedLoopEndNode* pre_end = find_pre_loop_end(cl);
160174
if (pre_end == NULL) {
161-
return;
175+
return false;
162176
}
163177
Node* pre_opaq1 = pre_end->limit();
164178
if (pre_opaq1->Opcode() != Op_Opaque1) {
165-
return;
179+
return false;
166180
}
167181
set_pre_loop_end(pre_end);
168182
}
@@ -175,9 +189,10 @@ void SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
175189
// For now, define one block which is the entire loop body
176190
set_bb(cl);
177191

192+
bool success = true;
178193
if (do_optimization) {
179194
assert(_packset.length() == 0, "packset must be empty");
180-
SLP_extract();
195+
success = SLP_extract();
181196
if (PostLoopMultiversioning && Matcher::has_predicated_vectors()) {
182197
if (cl->is_vectorized_loop() && cl->is_main_loop() && !cl->is_reduction_loop()) {
183198
IdealLoopTree *lpt_next = lpt->_next;
@@ -192,6 +207,7 @@ void SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
192207
}
193208
}
194209
}
210+
return success;
195211
}
196212

197213
//------------------------------early unrolling analysis------------------------------
@@ -451,7 +467,7 @@ void SuperWord::unrolling_analysis(int &local_loop_unroll_factor) {
451467
// inserting scalar promotion, vector creation from multiple scalars, and
452468
// extraction of scalar values from vectors.
453469
//
454-
void SuperWord::SLP_extract() {
470+
bool SuperWord::SLP_extract() {
455471

456472
#ifndef PRODUCT
457473
if (_do_vector_loop && TraceSuperWord) {
@@ -466,7 +482,7 @@ void SuperWord::SLP_extract() {
466482
#endif
467483
// Ready the block
468484
if (!construct_bb()) {
469-
return; // Exit if no interesting nodes or complex graph.
485+
return false; // Exit if no interesting nodes or complex graph.
470486
}
471487

472488
// build _dg, _disjoint_ptrs
@@ -483,7 +499,7 @@ void SuperWord::SLP_extract() {
483499
hoist_loads_in_graph(); // this only rebuild the graph; all basic structs need rebuild explicitly
484500

485501
if (!construct_bb()) {
486-
return; // Exit if no interesting nodes or complex graph.
502+
return false; // Exit if no interesting nodes or complex graph.
487503
}
488504
dependence_graph();
489505
compute_max_depth();
@@ -511,7 +527,7 @@ void SuperWord::SLP_extract() {
511527
find_adjacent_refs();
512528

513529
if (align_to_ref() == NULL) {
514-
return; // Did not find memory reference to align vectors
530+
return false; // Did not find memory reference to align vectors
515531
}
516532

517533
extend_packlist();
@@ -563,15 +579,15 @@ void SuperWord::SLP_extract() {
563579
// map base types for vector usage
564580
compute_vector_element_type();
565581
} else {
566-
return;
582+
return false;
567583
}
568584
} else {
569585
// for some reason we could not map the slp analysis state of the vectorized loop
570-
return;
586+
return false;
571587
}
572588
}
573589

574-
output();
590+
return output();
575591
}
576592

577593
//------------------------------find_adjacent_refs---------------------------
@@ -2385,17 +2401,11 @@ void SuperWord::print_loop(bool whole) {
23852401

23862402
//------------------------------output---------------------------
23872403
// Convert packs into vector node operations
2388-
void SuperWord::output() {
2404+
bool SuperWord::output() {
23892405
CountedLoopNode *cl = lpt()->_head->as_CountedLoop();
23902406
Compile* C = _phase->C;
23912407
if (_packset.length() == 0) {
2392-
if (cl->is_main_loop()) {
2393-
// Instigate more unrolling for optimization when vectorization fails.
2394-
C->set_major_progress();
2395-
cl->set_notpassed_slp();
2396-
cl->mark_do_unroll_only();
2397-
}
2398-
return;
2408+
return false;
23992409
}
24002410

24012411
#ifndef PRODUCT
@@ -2429,7 +2439,7 @@ void SuperWord::output() {
24292439

24302440
if (do_reserve_copy() && !make_reversable.has_reserved()) {
24312441
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: loop was not reserved correctly, exiting SuperWord");})
2432-
return;
2442+
return false;
24332443
}
24342444

24352445
for (int i = 0; i < _block.length(); i++) {
@@ -2474,7 +2484,7 @@ void SuperWord::output() {
24742484
if (val == NULL) {
24752485
if (do_reserve_copy()) {
24762486
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: val should not be NULL, exiting SuperWord");})
2477-
return; //and reverse to backup IG
2487+
return false; //and reverse to backup IG
24782488
}
24792489
ShouldNotReachHere();
24802490
}
@@ -2518,7 +2528,7 @@ void SuperWord::output() {
25182528
if (in1 == NULL) {
25192529
if (do_reserve_copy()) {
25202530
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: in1 should not be NULL, exiting SuperWord");})
2521-
return; //and reverse to backup IG
2531+
return false; //and reverse to backup IG
25222532
}
25232533
ShouldNotReachHere();
25242534
}
@@ -2527,7 +2537,7 @@ void SuperWord::output() {
25272537
if (in2 == NULL) {
25282538
if (do_reserve_copy()) {
25292539
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: in2 should not be NULL, exiting SuperWord");})
2530-
return; //and reverse to backup IG
2540+
return false; //and reverse to backup IG
25312541
}
25322542
ShouldNotReachHere();
25332543
}
@@ -2569,7 +2579,7 @@ void SuperWord::output() {
25692579
} else if (is_cmov_pack(p)) {
25702580
if (can_process_post_loop) {
25712581
// do not refactor of flow in post loop context
2572-
return;
2582+
return false;
25732583
}
25742584
if (!n->is_CMove()) {
25752585
continue;
@@ -2586,7 +2596,7 @@ void SuperWord::output() {
25862596
if (!bol->is_Bool()) {
25872597
if (do_reserve_copy()) {
25882598
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: expected %d bool node, exiting SuperWord", bol->_idx); bol->dump();})
2589-
return; //and reverse to backup IG
2599+
return false; //and reverse to backup IG
25902600
}
25912601
ShouldNotReachHere();
25922602
}
@@ -2602,15 +2612,15 @@ void SuperWord::output() {
26022612
if (src1 == NULL) {
26032613
if (do_reserve_copy()) {
26042614
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: src1 should not be NULL, exiting SuperWord");})
2605-
return; //and reverse to backup IG
2615+
return false; //and reverse to backup IG
26062616
}
26072617
ShouldNotReachHere();
26082618
}
26092619
Node* src2 = vector_opd(p, 3); //3=CMoveNode::IfTrue
26102620
if (src2 == NULL) {
26112621
if (do_reserve_copy()) {
26122622
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: src2 should not be NULL, exiting SuperWord");})
2613-
return; //and reverse to backup IG
2623+
return false; //and reverse to backup IG
26142624
}
26152625
ShouldNotReachHere();
26162626
}
@@ -2634,7 +2644,7 @@ void SuperWord::output() {
26342644
} else {
26352645
if (do_reserve_copy()) {
26362646
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: ShouldNotReachHere, exiting SuperWord");})
2637-
return; //and reverse to backup IG
2647+
return false; //and reverse to backup IG
26382648
}
26392649
ShouldNotReachHere();
26402650
}
@@ -2643,7 +2653,7 @@ void SuperWord::output() {
26432653
if (vn == NULL) {
26442654
if (do_reserve_copy()){
26452655
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: got NULL node, cannot proceed, exiting SuperWord");})
2646-
return; //and reverse to backup IG
2656+
return false; //and reverse to backup IG
26472657
}
26482658
ShouldNotReachHere();
26492659
}
@@ -2661,7 +2671,7 @@ void SuperWord::output() {
26612671
// first check if the vector size if the maximum vector which we can use on the machine,
26622672
// other vector size have reduced values for predicated data mapping.
26632673
if (vlen_in_bytes != (uint)MaxVectorSize) {
2664-
return;
2674+
return false;
26652675
}
26662676
}
26672677

@@ -2734,7 +2744,7 @@ void SuperWord::output() {
27342744
make_reversable.use_new();
27352745
}
27362746
NOT_PRODUCT(if(is_trace_loop_reverse()) {tty->print_cr("\n Final loop after SuperWord"); print_loop(true);})
2737-
return;
2747+
return true;
27382748
}
27392749

27402750
//------------------------------vector_opd---------------------------

src/hotspot/share/opto/superword.hpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ class SuperWord : public ResourceObj {
286286
public:
287287
SuperWord(PhaseIdealLoop* phase);
288288

289-
void transform_loop(IdealLoopTree* lpt, bool do_optimization);
289+
bool transform_loop(IdealLoopTree* lpt, bool do_optimization);
290290

291291
void unrolling_analysis(int &local_loop_unroll_factor);
292292

@@ -422,7 +422,7 @@ class SuperWord : public ResourceObj {
422422
// methods
423423

424424
// Extract the superword level parallelism
425-
void SLP_extract();
425+
bool SLP_extract();
426426
// Find the adjacent memory references and create pack pairs for them.
427427
void find_adjacent_refs();
428428
// Tracing support
@@ -509,7 +509,7 @@ class SuperWord : public ResourceObj {
509509
Node* find_last_mem_state(Node_List* pk, Node* first_mem);
510510

511511
// Convert packs into vector node operations
512-
void output();
512+
bool output();
513513
// Create a vector operand for the nodes in pack p for operand: in(opd_idx)
514514
Node* vector_opd(Node_List* p, int opd_idx);
515515
// Can code be generated for pack p?

0 commit comments

Comments
 (0)