From 2f10a928a16672141e03b7a24315d0001421a2a7 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 24 Oct 2024 08:28:12 +0000 Subject: [PATCH 01/41] JDK-8302459: compiler/vectorapi/VectorLogicalOpIdentityTest.java failed with "IRViolationException: There were one or multiple IR rule failures" --- src/hotspot/share/opto/callGenerator.cpp | 22 ++++++++++++++++++- src/hotspot/share/opto/callGenerator.hpp | 2 ++ src/hotspot/share/opto/compile.cpp | 28 +++++++++++++++++++++++- src/hotspot/share/opto/compile.hpp | 6 +++++ src/hotspot/share/opto/doCall.cpp | 2 +- src/hotspot/share/opto/phasetype.hpp | 1 + src/hotspot/share/opto/vector.cpp | 2 +- src/hotspot/share/opto/vector.hpp | 2 +- test/hotspot/jtreg/ProblemList.txt | 1 - 9 files changed, 60 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 36fca9f61b691..d56b649e2469d 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -731,7 +731,7 @@ void CallGenerator::do_late_inline_helper() { C->env()->notice_inlined_method(inline_cg()->method()); } C->set_inlining_progress(true); - C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup + C->set_do_cleanup(kit.stopped() || is_vector_late_inline()); // path is dead or vector inlining; needs cleanup kit.replace_call(call, result, true, do_asserts); } } @@ -823,6 +823,26 @@ CallGenerator* CallGenerator::for_vector_reboxing_late_inline(ciMethod* method, return new LateInlineVectorReboxingCallGenerator(method, inline_cg); } +class LateInlineVectorCallGenerator : public LateInlineCallGenerator { + + public: + LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* inline_cg) : + LateInlineCallGenerator(method, inline_cg) {} + + virtual bool is_vector_late_inline() const { return true; } + + virtual CallGenerator* with_call_node(CallNode* call) { + LateInlineVectorCallGenerator* cg = new LateInlineVectorCallGenerator(method(), _inline_cg); + cg->set_call_node(call->as_CallStaticJava()); + return cg; + } +}; + +// static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* inline_cg); +CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* method, CallGenerator* inline_cg) { + return new LateInlineVectorCallGenerator(method, inline_cg); +} + //------------------------PredictedCallGenerator------------------------------ // Internal class which handles all out-of-line calls checking receiver type. class PredictedCallGenerator : public CallGenerator { diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index 77182580207ca..4375cce74e166 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -76,6 +76,7 @@ class CallGenerator : public ArenaObj { virtual bool is_mh_late_inline() const { return false; } virtual bool is_string_late_inline() const { return false; } virtual bool is_virtual_late_inline() const { return false; } + virtual bool is_vector_late_inline() const { return false; } // Replace the call with an inline version of the code virtual void do_late_inline() { ShouldNotReachHere(); } @@ -141,6 +142,7 @@ class CallGenerator : public ArenaObj { static CallGenerator* for_string_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_boxing_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg); + static CallGenerator* for_vector_late_inline(ciMethod*m , CallGenerator* inline_cg); static CallGenerator* for_late_inline_virtual(ciMethod* m, int vtable_index, float expected_uses); // How to make a call that optimistically assumes a receiver type: diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index c6b316e527760..9266e62a3c4f2 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -407,6 +407,7 @@ void Compile::remove_useless_node(Node* dead) { remove_useless_late_inlines( &_string_late_inlines, dead); remove_useless_late_inlines( &_boxing_late_inlines, dead); remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead); + remove_useless_late_inlines( &_vector_late_inlines, dead); if (dead->is_CallStaticJava()) { remove_unstable_if_trap(dead->as_CallStaticJava(), false); @@ -465,6 +466,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis remove_useless_late_inlines( &_string_late_inlines, useful); remove_useless_late_inlines( &_boxing_late_inlines, useful); remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful); + remove_useless_late_inlines( &_vector_late_inlines, useful); debug_only(verify_graph_edges(true/*check for no_dead_code*/);) } @@ -668,6 +670,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _string_late_inlines(comp_arena(), 2, 0, nullptr), _boxing_late_inlines(comp_arena(), 2, 0, nullptr), _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr), + _vector_late_inlines(comp_arena(), 2, 0, nullptr), _late_inlines_pos(0), _number_of_mh_late_inlines(0), _oom(false), @@ -2148,6 +2151,11 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { inline_incrementally_cleanup(igvn); } + if (EnableVectorSupport && _vector_late_inlines.length() > 0) { + inline_vector_calls(igvn); + if (failing()) return; + } + set_inlining_incrementally(false); } @@ -2294,7 +2302,7 @@ void Compile::Optimize() { if (EnableVectorSupport && has_vbox_nodes()) { TracePhase tp("", &timers[_t_vector]); PhaseVector pv(igvn); - pv.optimize_vector_boxes(); + pv.optimize_vector_boxes(igvn); if (failing()) return; print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2); } @@ -2546,6 +2554,24 @@ bool Compile::has_vbox_nodes() { return false; } +void Compile::inline_vector_calls(PhaseIterGVN &igvn) { + if (C->_vector_late_inlines.length() > 0) { + _late_inlines_pos = C->_late_inlines.length(); + while (_vector_late_inlines.length() > 0) { + CallGenerator* cg = _vector_late_inlines.pop(); + bool does_dispatch = cg->is_virtual_late_inline() || cg->is_mh_late_inline(); + if (inlining_incrementally() || does_dispatch) { // a call can be either inlined or strength-reduced to a direct call + cg->do_late_inline(); + if (failing()) { + return; + } + print_method(PHASE_INLINE_VECTOR, 3, cg->call_node()); + } + } + _vector_late_inlines.trunc_to(0); + } +} + //---------------------------- Bitwise operation packing optimization --------------------------- static bool is_vector_unary_bitwise_op(Node* n) { diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 6568828125fe2..13367c87f74d5 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -463,6 +463,7 @@ class Compile : public Phase { GrowableArray _boxing_late_inlines; // same but for boxing operations GrowableArray _vector_reboxing_late_inlines; // same but for vector reboxing operations + GrowableArray _vector_late_inlines; // same but for other vector operations int _late_inlines_pos; // Where in the queue should the next late inlining candidate go (emulate depth first inlining) uint _number_of_mh_late_inlines; // number of method handle late inlining still pending @@ -1090,6 +1091,10 @@ class Compile : public Phase { _vector_reboxing_late_inlines.push(cg); } + void add_vector_late_inline(CallGenerator* cg) { + _vector_late_inlines.insert_before(0, cg); + } + template::value)> void remove_useless_nodes(GrowableArray& node_list, Unique_Node_List& useful); @@ -1127,6 +1132,7 @@ class Compile : public Phase { void remove_root_to_sfpts_edges(PhaseIterGVN& igvn); void inline_vector_reboxing_calls(); + void inline_vector_calls(PhaseIterGVN& igvn); bool has_vbox_nodes(); void process_late_inline_calls_no_inline(PhaseIterGVN& igvn); diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index 0a5e27ed5b13f..1967076bf2ded 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -158,7 +158,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool cg_intrinsic = cg; cg = nullptr; } else if (IncrementalInline && should_delay_vector_inlining(callee, jvms)) { - return CallGenerator::for_late_inline(callee, cg); + return CallGenerator::for_vector_late_inline(callee, cg); } else { return cg; } diff --git a/src/hotspot/share/opto/phasetype.hpp b/src/hotspot/share/opto/phasetype.hpp index 7a83f8c7f27d9..f4c07382b0f85 100644 --- a/src/hotspot/share/opto/phasetype.hpp +++ b/src/hotspot/share/opto/phasetype.hpp @@ -44,6 +44,7 @@ flags(EXPAND_VUNBOX, "Expand VectorUnbox") \ flags(SCALARIZE_VBOX, "Scalarize VectorBox") \ flags(INLINE_VECTOR_REBOX, "Inline Vector Rebox Calls") \ + flags(INLINE_VECTOR, "Inline Vector Calls") \ flags(EXPAND_VBOX, "Expand VectorBox") \ flags(ELIMINATE_VBOX_ALLOC, "Eliminate VectorBoxAllocate") \ flags(ITER_GVN_BEFORE_EA, "Iter GVN before EA") \ diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index 9ab62c282d124..b93312d49b2a5 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -41,7 +41,7 @@ static bool is_vector_shuffle(ciKlass* klass) { } -void PhaseVector::optimize_vector_boxes() { +void PhaseVector::optimize_vector_boxes(PhaseIterGVN &igvn) { Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); // Signal GraphKit it's post-parse phase. diff --git a/src/hotspot/share/opto/vector.hpp b/src/hotspot/share/opto/vector.hpp index e76b4df376d88..fda5e89b89ecc 100644 --- a/src/hotspot/share/opto/vector.hpp +++ b/src/hotspot/share/opto/vector.hpp @@ -57,7 +57,7 @@ class PhaseVector : public Phase { public: PhaseVector(PhaseIterGVN& igvn) : Phase(Vector), _igvn(igvn) {} - void optimize_vector_boxes(); + void optimize_vector_boxes(PhaseIterGVN &igvn); }; #endif // SHARE_OPTO_VECTOR_HPP diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index 3ff450dc3ad92..f1efa66c790e2 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -58,7 +58,6 @@ compiler/codecache/jmx/PoolsIndependenceTest.java 8264632 macosx-all compiler/codecache/CheckLargePages.java 8332654 linux-x64 compiler/vectorapi/reshape/TestVectorReinterpret.java 8320897 aix-ppc64,linux-ppc64le -compiler/vectorapi/VectorLogicalOpIdentityTest.java 8302459 linux-x64,windows-x64 compiler/vectorapi/VectorRebracket128Test.java#ZSinglegen 8330538 generic-all compiler/vectorapi/VectorRebracket128Test.java#ZGenerational 8330538 generic-all From 3ba3823b1a2f67b063896ac410b8aa022d44658a Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 24 Oct 2024 10:38:23 +0000 Subject: [PATCH 02/41] JDK-8302459: remove unused vector inline queue --- src/hotspot/share/opto/compile.cpp | 27 +-------------------------- src/hotspot/share/opto/compile.hpp | 5 ----- src/hotspot/share/opto/phasetype.hpp | 1 - src/hotspot/share/opto/vector.cpp | 3 +-- src/hotspot/share/opto/vector.hpp | 2 +- 5 files changed, 3 insertions(+), 35 deletions(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 9266e62a3c4f2..dea244df0d536 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -407,7 +407,6 @@ void Compile::remove_useless_node(Node* dead) { remove_useless_late_inlines( &_string_late_inlines, dead); remove_useless_late_inlines( &_boxing_late_inlines, dead); remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead); - remove_useless_late_inlines( &_vector_late_inlines, dead); if (dead->is_CallStaticJava()) { remove_unstable_if_trap(dead->as_CallStaticJava(), false); @@ -466,7 +465,6 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis remove_useless_late_inlines( &_string_late_inlines, useful); remove_useless_late_inlines( &_boxing_late_inlines, useful); remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful); - remove_useless_late_inlines( &_vector_late_inlines, useful); debug_only(verify_graph_edges(true/*check for no_dead_code*/);) } @@ -670,7 +668,6 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _string_late_inlines(comp_arena(), 2, 0, nullptr), _boxing_late_inlines(comp_arena(), 2, 0, nullptr), _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr), - _vector_late_inlines(comp_arena(), 2, 0, nullptr), _late_inlines_pos(0), _number_of_mh_late_inlines(0), _oom(false), @@ -2151,11 +2148,6 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { inline_incrementally_cleanup(igvn); } - if (EnableVectorSupport && _vector_late_inlines.length() > 0) { - inline_vector_calls(igvn); - if (failing()) return; - } - set_inlining_incrementally(false); } @@ -2302,7 +2294,7 @@ void Compile::Optimize() { if (EnableVectorSupport && has_vbox_nodes()) { TracePhase tp("", &timers[_t_vector]); PhaseVector pv(igvn); - pv.optimize_vector_boxes(igvn); + pv.optimize_vector_boxes(); if (failing()) return; print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2); } @@ -2554,23 +2546,6 @@ bool Compile::has_vbox_nodes() { return false; } -void Compile::inline_vector_calls(PhaseIterGVN &igvn) { - if (C->_vector_late_inlines.length() > 0) { - _late_inlines_pos = C->_late_inlines.length(); - while (_vector_late_inlines.length() > 0) { - CallGenerator* cg = _vector_late_inlines.pop(); - bool does_dispatch = cg->is_virtual_late_inline() || cg->is_mh_late_inline(); - if (inlining_incrementally() || does_dispatch) { // a call can be either inlined or strength-reduced to a direct call - cg->do_late_inline(); - if (failing()) { - return; - } - print_method(PHASE_INLINE_VECTOR, 3, cg->call_node()); - } - } - _vector_late_inlines.trunc_to(0); - } -} //---------------------------- Bitwise operation packing optimization --------------------------- diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 13367c87f74d5..e10329274ee2b 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -463,7 +463,6 @@ class Compile : public Phase { GrowableArray _boxing_late_inlines; // same but for boxing operations GrowableArray _vector_reboxing_late_inlines; // same but for vector reboxing operations - GrowableArray _vector_late_inlines; // same but for other vector operations int _late_inlines_pos; // Where in the queue should the next late inlining candidate go (emulate depth first inlining) uint _number_of_mh_late_inlines; // number of method handle late inlining still pending @@ -1091,10 +1090,6 @@ class Compile : public Phase { _vector_reboxing_late_inlines.push(cg); } - void add_vector_late_inline(CallGenerator* cg) { - _vector_late_inlines.insert_before(0, cg); - } - template::value)> void remove_useless_nodes(GrowableArray& node_list, Unique_Node_List& useful); diff --git a/src/hotspot/share/opto/phasetype.hpp b/src/hotspot/share/opto/phasetype.hpp index f4c07382b0f85..7a83f8c7f27d9 100644 --- a/src/hotspot/share/opto/phasetype.hpp +++ b/src/hotspot/share/opto/phasetype.hpp @@ -44,7 +44,6 @@ flags(EXPAND_VUNBOX, "Expand VectorUnbox") \ flags(SCALARIZE_VBOX, "Scalarize VectorBox") \ flags(INLINE_VECTOR_REBOX, "Inline Vector Rebox Calls") \ - flags(INLINE_VECTOR, "Inline Vector Calls") \ flags(EXPAND_VBOX, "Expand VectorBox") \ flags(ELIMINATE_VBOX_ALLOC, "Eliminate VectorBoxAllocate") \ flags(ITER_GVN_BEFORE_EA, "Iter GVN before EA") \ diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index b93312d49b2a5..6f486e8b8a754 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -40,8 +40,7 @@ static bool is_vector_shuffle(ciKlass* klass) { return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); } - -void PhaseVector::optimize_vector_boxes(PhaseIterGVN &igvn) { +void PhaseVector::optimize_vector_boxes() { Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); // Signal GraphKit it's post-parse phase. diff --git a/src/hotspot/share/opto/vector.hpp b/src/hotspot/share/opto/vector.hpp index fda5e89b89ecc..e76b4df376d88 100644 --- a/src/hotspot/share/opto/vector.hpp +++ b/src/hotspot/share/opto/vector.hpp @@ -57,7 +57,7 @@ class PhaseVector : public Phase { public: PhaseVector(PhaseIterGVN& igvn) : Phase(Vector), _igvn(igvn) {} - void optimize_vector_boxes(PhaseIterGVN &igvn); + void optimize_vector_boxes(); }; #endif // SHARE_OPTO_VECTOR_HPP From 80901d424b7431292d83cde41454617d2c574958 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 24 Oct 2024 10:40:23 +0000 Subject: [PATCH 03/41] JDK-8302459: remove unneeded changes --- src/hotspot/share/opto/compile.cpp | 1 - src/hotspot/share/opto/vector.cpp | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index dea244df0d536..c6b316e527760 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2546,7 +2546,6 @@ bool Compile::has_vbox_nodes() { return false; } - //---------------------------- Bitwise operation packing optimization --------------------------- static bool is_vector_unary_bitwise_op(Node* n) { diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index 6f486e8b8a754..9ab62c282d124 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -40,6 +40,7 @@ static bool is_vector_shuffle(ciKlass* klass) { return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); } + void PhaseVector::optimize_vector_boxes() { Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); From 7ec8069d65d38000568ed8d792454e4f92f4e541 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 24 Oct 2024 10:45:01 +0000 Subject: [PATCH 04/41] JDK-8302459: remove unneeded function declaration --- src/hotspot/share/opto/compile.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index e10329274ee2b..6568828125fe2 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -1127,7 +1127,6 @@ class Compile : public Phase { void remove_root_to_sfpts_edges(PhaseIterGVN& igvn); void inline_vector_reboxing_calls(); - void inline_vector_calls(PhaseIterGVN& igvn); bool has_vbox_nodes(); void process_late_inline_calls_no_inline(PhaseIterGVN& igvn); From 4131271e923d6b8cd49bf324a209c49d414a6127 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 24 Oct 2024 12:29:41 +0000 Subject: [PATCH 05/41] JDK-8302459: add explicit -TieredCompilation to tests --- .../compiler/vectorapi/VectorGatherMaskFoldingTest.java | 6 +++++- .../compiler/vectorapi/VectorLogicalOpIdentityTest.java | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java index 88175baf8e4a6..e5cd538109f74 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java @@ -1398,7 +1398,11 @@ public static void testFloatVectorLoadMaskedStoreVector() { public static void main(String[] args) { TestFramework testFramework = new TestFramework(); testFramework.setDefaultWarmup(10000) - .addFlags("--add-modules=jdk.incubator.vector", "-XX:+IgnoreUnrecognizedVMOptions", "-XX:+IncrementalInlineForceCleanup") + .addFlags("--add-modules=jdk.incubator.vector") + .start(); + testFramework = new TestFramework(); + testFramework.setDefaultWarmup(10000) + .addFlags("--add-modules=jdk.incubator.vector", "-XX:-TieredCompilation") .start(); } } diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java index 426dec6701933..8bd9d573c7544 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java @@ -761,5 +761,6 @@ public static void testMaskXorSame() { public static void main(String[] args) { TestFramework.runWithFlags("--add-modules=jdk.incubator.vector"); + TestFramework.runWithFlags("--add-modules=jdk.incubator.vector", "-XX:-TieredCompilation"); } } From f29f7a864c3e84746d2b1ce135723b93bc5f0794 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 24 Oct 2024 14:27:50 +0000 Subject: [PATCH 06/41] JDK-8302459: add bug numbers to tests --- .../jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java | 2 +- .../jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java index e5cd538109f74..5a122b6a7de6e 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java @@ -32,7 +32,7 @@ /** * @test - * @bug 8325520 + * @bug 8325520 8302459 * @library /test/lib / * @summary Don't allow folding of Load/Store vectors when using incompatible indices or masks * @modules jdk.incubator.vector diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java index 8bd9d573c7544..f6cc81ac7e367 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java @@ -41,7 +41,7 @@ /** * @test - * @bug 8288294 + * @bug 8288294 8302459 * @key randomness * @library /test/lib / * @summary Add identity transformations for vector logic operations From a78798cc99585eba67eafa1367b2469e05b562b8 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 24 Oct 2024 14:44:26 +0000 Subject: [PATCH 07/41] JDK-8302459: update copyright year --- src/hotspot/share/opto/callGenerator.cpp | 2 +- src/hotspot/share/opto/callGenerator.hpp | 2 +- .../jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index d56b649e2469d..8e7350e280938 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index 4375cce74e166..3c9a9632ce12f 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java index f6cc81ac7e367..61f733429ec9d 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java @@ -1,6 +1,6 @@ /* * Copyright (c) 2022, 2023, Arm Limited. All rights reserved. - * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From bffdbbd7afd73e1ab62bbcfc0db771fbf6de46ec Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 4 Nov 2024 08:39:20 +0000 Subject: [PATCH 08/41] Revert "JDK-8302459: update copyright year" This reverts commit a78798cc99585eba67eafa1367b2469e05b562b8. --- src/hotspot/share/opto/callGenerator.cpp | 2 +- src/hotspot/share/opto/callGenerator.hpp | 2 +- .../jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 8e7350e280938..d56b649e2469d 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index 3c9a9632ce12f..4375cce74e166 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java index 61f733429ec9d..f6cc81ac7e367 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java @@ -1,6 +1,6 @@ /* * Copyright (c) 2022, 2023, Arm Limited. All rights reserved. - * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 7481cc14b97f777affd0d2e2ec68dc48e2de19ae Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 4 Nov 2024 08:39:40 +0000 Subject: [PATCH 09/41] Revert "JDK-8302459: add bug numbers to tests" This reverts commit f29f7a864c3e84746d2b1ce135723b93bc5f0794. --- .../jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java | 2 +- .../jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java index 5a122b6a7de6e..e5cd538109f74 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java @@ -32,7 +32,7 @@ /** * @test - * @bug 8325520 8302459 + * @bug 8325520 * @library /test/lib / * @summary Don't allow folding of Load/Store vectors when using incompatible indices or masks * @modules jdk.incubator.vector diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java index f6cc81ac7e367..8bd9d573c7544 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java @@ -41,7 +41,7 @@ /** * @test - * @bug 8288294 8302459 + * @bug 8288294 * @key randomness * @library /test/lib / * @summary Add identity transformations for vector logic operations From 15cbe153a34449ef804dd3b32bfcfef476df0162 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 4 Nov 2024 08:40:32 +0000 Subject: [PATCH 10/41] Revert "JDK-8302459: add explicit -TieredCompilation to tests" This reverts commit 4131271e923d6b8cd49bf324a209c49d414a6127. --- .../compiler/vectorapi/VectorGatherMaskFoldingTest.java | 6 +----- .../compiler/vectorapi/VectorLogicalOpIdentityTest.java | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java index e5cd538109f74..88175baf8e4a6 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java @@ -1398,11 +1398,7 @@ public static void testFloatVectorLoadMaskedStoreVector() { public static void main(String[] args) { TestFramework testFramework = new TestFramework(); testFramework.setDefaultWarmup(10000) - .addFlags("--add-modules=jdk.incubator.vector") - .start(); - testFramework = new TestFramework(); - testFramework.setDefaultWarmup(10000) - .addFlags("--add-modules=jdk.incubator.vector", "-XX:-TieredCompilation") + .addFlags("--add-modules=jdk.incubator.vector", "-XX:+IgnoreUnrecognizedVMOptions", "-XX:+IncrementalInlineForceCleanup") .start(); } } diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java index 8bd9d573c7544..426dec6701933 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java @@ -761,6 +761,5 @@ public static void testMaskXorSame() { public static void main(String[] args) { TestFramework.runWithFlags("--add-modules=jdk.incubator.vector"); - TestFramework.runWithFlags("--add-modules=jdk.incubator.vector", "-XX:-TieredCompilation"); } } From 0b7dff80e71e41bf63231f26a9c8c1875d565a8f Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 4 Nov 2024 08:40:44 +0000 Subject: [PATCH 11/41] Revert "JDK-8302459: remove unneeded function declaration" This reverts commit 7ec8069d65d38000568ed8d792454e4f92f4e541. --- src/hotspot/share/opto/compile.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 6568828125fe2..e10329274ee2b 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -1127,6 +1127,7 @@ class Compile : public Phase { void remove_root_to_sfpts_edges(PhaseIterGVN& igvn); void inline_vector_reboxing_calls(); + void inline_vector_calls(PhaseIterGVN& igvn); bool has_vbox_nodes(); void process_late_inline_calls_no_inline(PhaseIterGVN& igvn); From f547d89af7b41d930354758fc5e9478b2dc97fa5 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 4 Nov 2024 08:40:52 +0000 Subject: [PATCH 12/41] Revert "JDK-8302459: remove unneeded changes" This reverts commit 80901d424b7431292d83cde41454617d2c574958. --- src/hotspot/share/opto/compile.cpp | 1 + src/hotspot/share/opto/vector.cpp | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index c6b316e527760..dea244df0d536 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2546,6 +2546,7 @@ bool Compile::has_vbox_nodes() { return false; } + //---------------------------- Bitwise operation packing optimization --------------------------- static bool is_vector_unary_bitwise_op(Node* n) { diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index 9ab62c282d124..6f486e8b8a754 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -40,7 +40,6 @@ static bool is_vector_shuffle(ciKlass* klass) { return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); } - void PhaseVector::optimize_vector_boxes() { Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); From 49c8a5102774801db7ea1ae34b04f169769c329b Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 4 Nov 2024 08:40:57 +0000 Subject: [PATCH 13/41] Revert "JDK-8302459: remove unused vector inline queue" This reverts commit 3ba3823b1a2f67b063896ac410b8aa022d44658a. --- src/hotspot/share/opto/compile.cpp | 27 ++++++++++++++++++++++++++- src/hotspot/share/opto/compile.hpp | 5 +++++ src/hotspot/share/opto/phasetype.hpp | 1 + src/hotspot/share/opto/vector.cpp | 3 ++- src/hotspot/share/opto/vector.hpp | 2 +- 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index dea244df0d536..9266e62a3c4f2 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -407,6 +407,7 @@ void Compile::remove_useless_node(Node* dead) { remove_useless_late_inlines( &_string_late_inlines, dead); remove_useless_late_inlines( &_boxing_late_inlines, dead); remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead); + remove_useless_late_inlines( &_vector_late_inlines, dead); if (dead->is_CallStaticJava()) { remove_unstable_if_trap(dead->as_CallStaticJava(), false); @@ -465,6 +466,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis remove_useless_late_inlines( &_string_late_inlines, useful); remove_useless_late_inlines( &_boxing_late_inlines, useful); remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful); + remove_useless_late_inlines( &_vector_late_inlines, useful); debug_only(verify_graph_edges(true/*check for no_dead_code*/);) } @@ -668,6 +670,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _string_late_inlines(comp_arena(), 2, 0, nullptr), _boxing_late_inlines(comp_arena(), 2, 0, nullptr), _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr), + _vector_late_inlines(comp_arena(), 2, 0, nullptr), _late_inlines_pos(0), _number_of_mh_late_inlines(0), _oom(false), @@ -2148,6 +2151,11 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { inline_incrementally_cleanup(igvn); } + if (EnableVectorSupport && _vector_late_inlines.length() > 0) { + inline_vector_calls(igvn); + if (failing()) return; + } + set_inlining_incrementally(false); } @@ -2294,7 +2302,7 @@ void Compile::Optimize() { if (EnableVectorSupport && has_vbox_nodes()) { TracePhase tp("", &timers[_t_vector]); PhaseVector pv(igvn); - pv.optimize_vector_boxes(); + pv.optimize_vector_boxes(igvn); if (failing()) return; print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2); } @@ -2546,6 +2554,23 @@ bool Compile::has_vbox_nodes() { return false; } +void Compile::inline_vector_calls(PhaseIterGVN &igvn) { + if (C->_vector_late_inlines.length() > 0) { + _late_inlines_pos = C->_late_inlines.length(); + while (_vector_late_inlines.length() > 0) { + CallGenerator* cg = _vector_late_inlines.pop(); + bool does_dispatch = cg->is_virtual_late_inline() || cg->is_mh_late_inline(); + if (inlining_incrementally() || does_dispatch) { // a call can be either inlined or strength-reduced to a direct call + cg->do_late_inline(); + if (failing()) { + return; + } + print_method(PHASE_INLINE_VECTOR, 3, cg->call_node()); + } + } + _vector_late_inlines.trunc_to(0); + } +} //---------------------------- Bitwise operation packing optimization --------------------------- diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index e10329274ee2b..13367c87f74d5 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -463,6 +463,7 @@ class Compile : public Phase { GrowableArray _boxing_late_inlines; // same but for boxing operations GrowableArray _vector_reboxing_late_inlines; // same but for vector reboxing operations + GrowableArray _vector_late_inlines; // same but for other vector operations int _late_inlines_pos; // Where in the queue should the next late inlining candidate go (emulate depth first inlining) uint _number_of_mh_late_inlines; // number of method handle late inlining still pending @@ -1090,6 +1091,10 @@ class Compile : public Phase { _vector_reboxing_late_inlines.push(cg); } + void add_vector_late_inline(CallGenerator* cg) { + _vector_late_inlines.insert_before(0, cg); + } + template::value)> void remove_useless_nodes(GrowableArray& node_list, Unique_Node_List& useful); diff --git a/src/hotspot/share/opto/phasetype.hpp b/src/hotspot/share/opto/phasetype.hpp index 7a83f8c7f27d9..f4c07382b0f85 100644 --- a/src/hotspot/share/opto/phasetype.hpp +++ b/src/hotspot/share/opto/phasetype.hpp @@ -44,6 +44,7 @@ flags(EXPAND_VUNBOX, "Expand VectorUnbox") \ flags(SCALARIZE_VBOX, "Scalarize VectorBox") \ flags(INLINE_VECTOR_REBOX, "Inline Vector Rebox Calls") \ + flags(INLINE_VECTOR, "Inline Vector Calls") \ flags(EXPAND_VBOX, "Expand VectorBox") \ flags(ELIMINATE_VBOX_ALLOC, "Eliminate VectorBoxAllocate") \ flags(ITER_GVN_BEFORE_EA, "Iter GVN before EA") \ diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index 6f486e8b8a754..b93312d49b2a5 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -40,7 +40,8 @@ static bool is_vector_shuffle(ciKlass* klass) { return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); } -void PhaseVector::optimize_vector_boxes() { + +void PhaseVector::optimize_vector_boxes(PhaseIterGVN &igvn) { Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); // Signal GraphKit it's post-parse phase. diff --git a/src/hotspot/share/opto/vector.hpp b/src/hotspot/share/opto/vector.hpp index e76b4df376d88..fda5e89b89ecc 100644 --- a/src/hotspot/share/opto/vector.hpp +++ b/src/hotspot/share/opto/vector.hpp @@ -57,7 +57,7 @@ class PhaseVector : public Phase { public: PhaseVector(PhaseIterGVN& igvn) : Phase(Vector), _igvn(igvn) {} - void optimize_vector_boxes(); + void optimize_vector_boxes(PhaseIterGVN &igvn); }; #endif // SHARE_OPTO_VECTOR_HPP From 11c63b57f8065818a035fd3084a86d1722cf8ba1 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 4 Nov 2024 08:41:03 +0000 Subject: [PATCH 14/41] Revert "JDK-8302459: compiler/vectorapi/VectorLogicalOpIdentityTest.java failed with "IRViolationException: There were one or multiple IR rule failures"" This reverts commit 2f10a928a16672141e03b7a24315d0001421a2a7. --- src/hotspot/share/opto/callGenerator.cpp | 22 +------------------ src/hotspot/share/opto/callGenerator.hpp | 2 -- src/hotspot/share/opto/compile.cpp | 28 +----------------------- src/hotspot/share/opto/compile.hpp | 6 ----- src/hotspot/share/opto/doCall.cpp | 2 +- src/hotspot/share/opto/phasetype.hpp | 1 - src/hotspot/share/opto/vector.cpp | 2 +- src/hotspot/share/opto/vector.hpp | 2 +- test/hotspot/jtreg/ProblemList.txt | 1 + 9 files changed, 6 insertions(+), 60 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index d56b649e2469d..36fca9f61b691 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -731,7 +731,7 @@ void CallGenerator::do_late_inline_helper() { C->env()->notice_inlined_method(inline_cg()->method()); } C->set_inlining_progress(true); - C->set_do_cleanup(kit.stopped() || is_vector_late_inline()); // path is dead or vector inlining; needs cleanup + C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup kit.replace_call(call, result, true, do_asserts); } } @@ -823,26 +823,6 @@ CallGenerator* CallGenerator::for_vector_reboxing_late_inline(ciMethod* method, return new LateInlineVectorReboxingCallGenerator(method, inline_cg); } -class LateInlineVectorCallGenerator : public LateInlineCallGenerator { - - public: - LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* inline_cg) : - LateInlineCallGenerator(method, inline_cg) {} - - virtual bool is_vector_late_inline() const { return true; } - - virtual CallGenerator* with_call_node(CallNode* call) { - LateInlineVectorCallGenerator* cg = new LateInlineVectorCallGenerator(method(), _inline_cg); - cg->set_call_node(call->as_CallStaticJava()); - return cg; - } -}; - -// static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* inline_cg); -CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* method, CallGenerator* inline_cg) { - return new LateInlineVectorCallGenerator(method, inline_cg); -} - //------------------------PredictedCallGenerator------------------------------ // Internal class which handles all out-of-line calls checking receiver type. class PredictedCallGenerator : public CallGenerator { diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index 4375cce74e166..77182580207ca 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -76,7 +76,6 @@ class CallGenerator : public ArenaObj { virtual bool is_mh_late_inline() const { return false; } virtual bool is_string_late_inline() const { return false; } virtual bool is_virtual_late_inline() const { return false; } - virtual bool is_vector_late_inline() const { return false; } // Replace the call with an inline version of the code virtual void do_late_inline() { ShouldNotReachHere(); } @@ -142,7 +141,6 @@ class CallGenerator : public ArenaObj { static CallGenerator* for_string_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_boxing_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg); - static CallGenerator* for_vector_late_inline(ciMethod*m , CallGenerator* inline_cg); static CallGenerator* for_late_inline_virtual(ciMethod* m, int vtable_index, float expected_uses); // How to make a call that optimistically assumes a receiver type: diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 9266e62a3c4f2..c6b316e527760 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -407,7 +407,6 @@ void Compile::remove_useless_node(Node* dead) { remove_useless_late_inlines( &_string_late_inlines, dead); remove_useless_late_inlines( &_boxing_late_inlines, dead); remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead); - remove_useless_late_inlines( &_vector_late_inlines, dead); if (dead->is_CallStaticJava()) { remove_unstable_if_trap(dead->as_CallStaticJava(), false); @@ -466,7 +465,6 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis remove_useless_late_inlines( &_string_late_inlines, useful); remove_useless_late_inlines( &_boxing_late_inlines, useful); remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful); - remove_useless_late_inlines( &_vector_late_inlines, useful); debug_only(verify_graph_edges(true/*check for no_dead_code*/);) } @@ -670,7 +668,6 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _string_late_inlines(comp_arena(), 2, 0, nullptr), _boxing_late_inlines(comp_arena(), 2, 0, nullptr), _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr), - _vector_late_inlines(comp_arena(), 2, 0, nullptr), _late_inlines_pos(0), _number_of_mh_late_inlines(0), _oom(false), @@ -2151,11 +2148,6 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { inline_incrementally_cleanup(igvn); } - if (EnableVectorSupport && _vector_late_inlines.length() > 0) { - inline_vector_calls(igvn); - if (failing()) return; - } - set_inlining_incrementally(false); } @@ -2302,7 +2294,7 @@ void Compile::Optimize() { if (EnableVectorSupport && has_vbox_nodes()) { TracePhase tp("", &timers[_t_vector]); PhaseVector pv(igvn); - pv.optimize_vector_boxes(igvn); + pv.optimize_vector_boxes(); if (failing()) return; print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2); } @@ -2554,24 +2546,6 @@ bool Compile::has_vbox_nodes() { return false; } -void Compile::inline_vector_calls(PhaseIterGVN &igvn) { - if (C->_vector_late_inlines.length() > 0) { - _late_inlines_pos = C->_late_inlines.length(); - while (_vector_late_inlines.length() > 0) { - CallGenerator* cg = _vector_late_inlines.pop(); - bool does_dispatch = cg->is_virtual_late_inline() || cg->is_mh_late_inline(); - if (inlining_incrementally() || does_dispatch) { // a call can be either inlined or strength-reduced to a direct call - cg->do_late_inline(); - if (failing()) { - return; - } - print_method(PHASE_INLINE_VECTOR, 3, cg->call_node()); - } - } - _vector_late_inlines.trunc_to(0); - } -} - //---------------------------- Bitwise operation packing optimization --------------------------- static bool is_vector_unary_bitwise_op(Node* n) { diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 13367c87f74d5..6568828125fe2 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -463,7 +463,6 @@ class Compile : public Phase { GrowableArray _boxing_late_inlines; // same but for boxing operations GrowableArray _vector_reboxing_late_inlines; // same but for vector reboxing operations - GrowableArray _vector_late_inlines; // same but for other vector operations int _late_inlines_pos; // Where in the queue should the next late inlining candidate go (emulate depth first inlining) uint _number_of_mh_late_inlines; // number of method handle late inlining still pending @@ -1091,10 +1090,6 @@ class Compile : public Phase { _vector_reboxing_late_inlines.push(cg); } - void add_vector_late_inline(CallGenerator* cg) { - _vector_late_inlines.insert_before(0, cg); - } - template::value)> void remove_useless_nodes(GrowableArray& node_list, Unique_Node_List& useful); @@ -1132,7 +1127,6 @@ class Compile : public Phase { void remove_root_to_sfpts_edges(PhaseIterGVN& igvn); void inline_vector_reboxing_calls(); - void inline_vector_calls(PhaseIterGVN& igvn); bool has_vbox_nodes(); void process_late_inline_calls_no_inline(PhaseIterGVN& igvn); diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index 1967076bf2ded..0a5e27ed5b13f 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -158,7 +158,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool cg_intrinsic = cg; cg = nullptr; } else if (IncrementalInline && should_delay_vector_inlining(callee, jvms)) { - return CallGenerator::for_vector_late_inline(callee, cg); + return CallGenerator::for_late_inline(callee, cg); } else { return cg; } diff --git a/src/hotspot/share/opto/phasetype.hpp b/src/hotspot/share/opto/phasetype.hpp index f4c07382b0f85..7a83f8c7f27d9 100644 --- a/src/hotspot/share/opto/phasetype.hpp +++ b/src/hotspot/share/opto/phasetype.hpp @@ -44,7 +44,6 @@ flags(EXPAND_VUNBOX, "Expand VectorUnbox") \ flags(SCALARIZE_VBOX, "Scalarize VectorBox") \ flags(INLINE_VECTOR_REBOX, "Inline Vector Rebox Calls") \ - flags(INLINE_VECTOR, "Inline Vector Calls") \ flags(EXPAND_VBOX, "Expand VectorBox") \ flags(ELIMINATE_VBOX_ALLOC, "Eliminate VectorBoxAllocate") \ flags(ITER_GVN_BEFORE_EA, "Iter GVN before EA") \ diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index b93312d49b2a5..9ab62c282d124 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -41,7 +41,7 @@ static bool is_vector_shuffle(ciKlass* klass) { } -void PhaseVector::optimize_vector_boxes(PhaseIterGVN &igvn) { +void PhaseVector::optimize_vector_boxes() { Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); // Signal GraphKit it's post-parse phase. diff --git a/src/hotspot/share/opto/vector.hpp b/src/hotspot/share/opto/vector.hpp index fda5e89b89ecc..e76b4df376d88 100644 --- a/src/hotspot/share/opto/vector.hpp +++ b/src/hotspot/share/opto/vector.hpp @@ -57,7 +57,7 @@ class PhaseVector : public Phase { public: PhaseVector(PhaseIterGVN& igvn) : Phase(Vector), _igvn(igvn) {} - void optimize_vector_boxes(PhaseIterGVN &igvn); + void optimize_vector_boxes(); }; #endif // SHARE_OPTO_VECTOR_HPP diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index f1efa66c790e2..3ff450dc3ad92 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -58,6 +58,7 @@ compiler/codecache/jmx/PoolsIndependenceTest.java 8264632 macosx-all compiler/codecache/CheckLargePages.java 8332654 linux-x64 compiler/vectorapi/reshape/TestVectorReinterpret.java 8320897 aix-ppc64,linux-ppc64le +compiler/vectorapi/VectorLogicalOpIdentityTest.java 8302459 linux-x64,windows-x64 compiler/vectorapi/VectorRebracket128Test.java#ZSinglegen 8330538 generic-all compiler/vectorapi/VectorRebracket128Test.java#ZGenerational 8330538 generic-all From 63b749c0098d4e550d3a3f1b2eb3d49cc014306d Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 4 Nov 2024 08:50:38 +0000 Subject: [PATCH 15/41] JDK-8302459: Missing late inline cleanup causes compiler/vectorapi/VectorLogicalOpIdentityTest.java IR failure --- src/hotspot/share/opto/callGenerator.cpp | 4 ++-- test/hotspot/jtreg/ProblemList.txt | 1 - .../compiler/vectorapi/VectorGatherMaskFoldingTest.java | 9 +++++++-- .../compiler/vectorapi/VectorLogicalOpIdentityTest.java | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 36fca9f61b691..d4a6bbae7020f 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -731,7 +731,7 @@ void CallGenerator::do_late_inline_helper() { C->env()->notice_inlined_method(inline_cg()->method()); } C->set_inlining_progress(true); - C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup + C->set_do_cleanup(kit.stopped() || result->Opcode() == Op_VectorBox); // path is dead or vector box; needs cleanup kit.replace_call(call, result, true, do_asserts); } } diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index 3ff450dc3ad92..f1efa66c790e2 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -58,7 +58,6 @@ compiler/codecache/jmx/PoolsIndependenceTest.java 8264632 macosx-all compiler/codecache/CheckLargePages.java 8332654 linux-x64 compiler/vectorapi/reshape/TestVectorReinterpret.java 8320897 aix-ppc64,linux-ppc64le -compiler/vectorapi/VectorLogicalOpIdentityTest.java 8302459 linux-x64,windows-x64 compiler/vectorapi/VectorRebracket128Test.java#ZSinglegen 8330538 generic-all compiler/vectorapi/VectorRebracket128Test.java#ZGenerational 8330538 generic-all diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java index 88175baf8e4a6..c4ad05c9b9693 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java @@ -32,7 +32,7 @@ /** * @test - * @bug 8325520 + * @bug 8325520 8302459 * @library /test/lib / * @summary Don't allow folding of Load/Store vectors when using incompatible indices or masks * @modules jdk.incubator.vector @@ -1398,7 +1398,12 @@ public static void testFloatVectorLoadMaskedStoreVector() { public static void main(String[] args) { TestFramework testFramework = new TestFramework(); testFramework.setDefaultWarmup(10000) - .addFlags("--add-modules=jdk.incubator.vector", "-XX:+IgnoreUnrecognizedVMOptions", "-XX:+IncrementalInlineForceCleanup") + .addFlags("--add-modules=jdk.incubator.vector") .start(); + testFramework = new TestFramework(); + testFramework.setDefaultWarmup(10000) + .addFlags("--add-modules=jdk.incubator.vector", "-XX:-TieredCompilation") + .start(); + } } diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java index 426dec6701933..61f733429ec9d 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java @@ -1,6 +1,6 @@ /* * Copyright (c) 2022, 2023, Arm Limited. All rights reserved. - * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -41,7 +41,7 @@ /** * @test - * @bug 8288294 + * @bug 8288294 8302459 * @key randomness * @library /test/lib / * @summary Add identity transformations for vector logic operations @@ -761,5 +761,6 @@ public static void testMaskXorSame() { public static void main(String[] args) { TestFramework.runWithFlags("--add-modules=jdk.incubator.vector"); + TestFramework.runWithFlags("--add-modules=jdk.incubator.vector", "-XX:-TieredCompilation"); } } From f759c80a987acd384c38a4f46cdf2a69989043ed Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 14 Nov 2024 13:23:37 +0000 Subject: [PATCH 16/41] JDK-8302459: use list of failed inlines --- src/hotspot/share/opto/callGenerator.cpp | 2 +- src/hotspot/share/opto/callnode.cpp | 5 ++++- src/hotspot/share/opto/callnode.hpp | 6 +++++- src/hotspot/share/opto/compile.cpp | 10 ++++++++++ src/hotspot/share/opto/compile.hpp | 5 +++++ 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index d4a6bbae7020f..514fece6bb479 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -731,7 +731,7 @@ void CallGenerator::do_late_inline_helper() { C->env()->notice_inlined_method(inline_cg()->method()); } C->set_inlining_progress(true); - C->set_do_cleanup(kit.stopped() || result->Opcode() == Op_VectorBox); // path is dead or vector box; needs cleanup + C->set_do_cleanup(kit.stopped()); // path is dead; needs cleanup kit.replace_call(call, result, true, do_asserts); } } diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index e800b3c736bf2..a70b74dfc09ff 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1077,7 +1077,10 @@ bool CallStaticJavaNode::cmp( const Node &n ) const { Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { CallGenerator* cg = generator(); - if (can_reshape && cg != nullptr) { + CallGenerator* fcg = failed_generator(); + if (fcg != nullptr) { + phase->C->add_late_inline(fcg); + } else if (can_reshape && cg != nullptr) { assert(IncrementalInlineMH, "required"); assert(cg->call_node() == this, "mismatch"); assert(cg->is_mh_late_inline(), "not virtual"); diff --git a/src/hotspot/share/opto/callnode.hpp b/src/hotspot/share/opto/callnode.hpp index 2d3835b71ad42..5f1a965465f5f 100644 --- a/src/hotspot/share/opto/callnode.hpp +++ b/src/hotspot/share/opto/callnode.hpp @@ -683,6 +683,7 @@ class CallNode : public SafePointNode { float _cnt; // Estimate of number of times called CallGenerator* _generator; // corresponding CallGenerator for some late inline calls const char* _name; // Printable name, if _method is null + CallGenerator* _failed_generator; CallNode(const TypeFunc* tf, address addr, const TypePtr* adr_type, JVMState* jvms = nullptr) : SafePointNode(tf->domain()->cnt(), jvms, adr_type), @@ -690,7 +691,8 @@ class CallNode : public SafePointNode { _entry_point(addr), _cnt(COUNT_UNKNOWN), _generator(nullptr), - _name(nullptr) + _name(nullptr), + _failed_generator(nullptr) { init_class_id(Class_Call); } @@ -699,11 +701,13 @@ class CallNode : public SafePointNode { address entry_point() const { return _entry_point; } float cnt() const { return _cnt; } CallGenerator* generator() const { return _generator; } + CallGenerator* failed_generator() const { return _failed_generator; } void set_tf(const TypeFunc* tf) { _tf = tf; } void set_entry_point(address p) { _entry_point = p; } void set_cnt(float c) { _cnt = c; } void set_generator(CallGenerator* cg) { _generator = cg; } + void set_failed_generator(CallGenerator* cg) { _failed_generator = cg; } virtual const Type* bottom_type() const; virtual const Type* Value(PhaseGVN* phase) const; diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index a452d439a540b..d064bd3e53430 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -665,6 +665,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _types(nullptr), _node_hash(nullptr), _late_inlines(comp_arena(), 2, 0, nullptr), + _failed_late_inlines(comp_arena(), 2, 0, nullptr), _string_late_inlines(comp_arena(), 2, 0, nullptr), _boxing_late_inlines(comp_arena(), 2, 0, nullptr), _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr), @@ -2045,6 +2046,9 @@ bool Compile::inline_incrementally_one() { _late_inlines_pos = i+1; // restore the position in case new elements were inserted print_method(PHASE_INCREMENTAL_INLINE_STEP, 3, cg->call_node()); break; // process one call site at a time + } else { + cg->call_node()->set_failed_generator(cg); + add_failed_late_inline(cg); } } else { // Ignore late inline direct calls when inlining is not allowed. @@ -2076,6 +2080,12 @@ void Compile::inline_incrementally_cleanup(PhaseIterGVN& igvn) { TracePhase tp("incrementalInline_igvn", &timers[_t_incrInline_igvn]); igvn.reset_from_gvn(initial_gvn()); igvn.optimize(); + // Reset failed generator in call node + for (int i = 0; i < _failed_late_inlines.length(); i++) { + CallGenerator* cg = _failed_late_inlines.at(i); + cg->call_node()->set_failed_generator(nullptr); + } + _failed_late_inlines.clear(); if (failing()) return; } print_method(PHASE_INCREMENTAL_INLINE_CLEANUP, 3); diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 05e24bf3f6e1f..1f88f10756a23 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -460,6 +460,7 @@ class Compile : public Phase { NodeHash* _node_hash; GrowableArray _late_inlines; // List of CallGenerators to be revisited after main parsing has finished. + GrowableArray _failed_late_inlines; // List of CallGenerators that have failed inlining GrowableArray _string_late_inlines; // same but for string operations GrowableArray _boxing_late_inlines; // same but for boxing operations @@ -1079,6 +1080,10 @@ class Compile : public Phase { _late_inlines.insert_before(0, cg); } + void add_failed_late_inline(CallGenerator* cg) { + _failed_late_inlines.push(cg); + } + void add_string_late_inline(CallGenerator* cg) { _string_late_inlines.push(cg); } From 7064598710368900f4f8d8a4451e20d049b7fcd2 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Fri, 15 Nov 2024 09:52:12 +0000 Subject: [PATCH 17/41] JDK-8302459: add flag for inline added callnodes --- src/hotspot/share/opto/callnode.cpp | 5 ++++- src/hotspot/share/opto/callnode.hpp | 6 +++++- src/hotspot/share/opto/compile.cpp | 4 +++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index a70b74dfc09ff..2b73e60703e3c 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1079,7 +1079,10 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { CallGenerator* cg = generator(); CallGenerator* fcg = failed_generator(); if (fcg != nullptr) { - phase->C->add_late_inline(fcg); + if (!late_inline_added()) { + phase->C->prepend_late_inline(fcg); + set_late_inline_added(true); + } } else if (can_reshape && cg != nullptr) { assert(IncrementalInlineMH, "required"); assert(cg->call_node() == this, "mismatch"); diff --git a/src/hotspot/share/opto/callnode.hpp b/src/hotspot/share/opto/callnode.hpp index 5f1a965465f5f..649b81f0952d2 100644 --- a/src/hotspot/share/opto/callnode.hpp +++ b/src/hotspot/share/opto/callnode.hpp @@ -684,6 +684,7 @@ class CallNode : public SafePointNode { CallGenerator* _generator; // corresponding CallGenerator for some late inline calls const char* _name; // Printable name, if _method is null CallGenerator* _failed_generator; + bool _late_inline_added; CallNode(const TypeFunc* tf, address addr, const TypePtr* adr_type, JVMState* jvms = nullptr) : SafePointNode(tf->domain()->cnt(), jvms, adr_type), @@ -692,7 +693,8 @@ class CallNode : public SafePointNode { _cnt(COUNT_UNKNOWN), _generator(nullptr), _name(nullptr), - _failed_generator(nullptr) + _failed_generator(nullptr), + _late_inline_added(false) { init_class_id(Class_Call); } @@ -702,12 +704,14 @@ class CallNode : public SafePointNode { float cnt() const { return _cnt; } CallGenerator* generator() const { return _generator; } CallGenerator* failed_generator() const { return _failed_generator; } + bool late_inline_added() const { return _late_inline_added; } void set_tf(const TypeFunc* tf) { _tf = tf; } void set_entry_point(address p) { _entry_point = p; } void set_cnt(float c) { _cnt = c; } void set_generator(CallGenerator* cg) { _generator = cg; } void set_failed_generator(CallGenerator* cg) { _failed_generator = cg; } + void set_late_inline_added(bool added) { _late_inline_added = added; } virtual const Type* bottom_type() const; virtual const Type* Value(PhaseGVN* phase) const; diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index d064bd3e53430..69aa05faaa669 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2083,7 +2083,9 @@ void Compile::inline_incrementally_cleanup(PhaseIterGVN& igvn) { // Reset failed generator in call node for (int i = 0; i < _failed_late_inlines.length(); i++) { CallGenerator* cg = _failed_late_inlines.at(i); - cg->call_node()->set_failed_generator(nullptr); + CallNode* cn = cg->call_node(); + cn->set_failed_generator(nullptr); + cn->set_late_inline_added(false); } _failed_late_inlines.clear(); if (failing()) return; From af130f98e0abac1487a29236cb0cd2a3b2d42f5e Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Wed, 20 Nov 2024 16:44:45 +0000 Subject: [PATCH 18/41] JDK-8302459: increment MH late inline counter when reinserting them --- src/hotspot/share/opto/callnode.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index f415360cd649a..f0e6e951cb972 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1081,6 +1081,7 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (fcg != nullptr) { if (!late_inline_added()) { phase->C->prepend_late_inline(fcg); + phase->C->inc_number_of_mh_late_inlines(); set_late_inline_added(true); } } else if (can_reshape && cg != nullptr) { From ce27ae33f2a8d687e29f3c0a8c93b018f1eac0ec Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Wed, 20 Nov 2024 16:51:58 +0000 Subject: [PATCH 19/41] JDK-8302459: remove unneeded spaces --- src/hotspot/share/opto/callnode.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/callnode.hpp b/src/hotspot/share/opto/callnode.hpp index 649b81f0952d2..33f41fb370bb9 100644 --- a/src/hotspot/share/opto/callnode.hpp +++ b/src/hotspot/share/opto/callnode.hpp @@ -703,7 +703,7 @@ class CallNode : public SafePointNode { address entry_point() const { return _entry_point; } float cnt() const { return _cnt; } CallGenerator* generator() const { return _generator; } - CallGenerator* failed_generator() const { return _failed_generator; } + CallGenerator* failed_generator() const { return _failed_generator; } bool late_inline_added() const { return _late_inline_added; } void set_tf(const TypeFunc* tf) { _tf = tf; } From 00917c5704ceb432cd5bb88ed8b5f92f358236d2 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 21 Nov 2024 10:13:54 +0100 Subject: [PATCH 20/41] JDK-8302459: revert unneeded copyright update --- src/hotspot/share/opto/callGenerator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 514fece6bb479..36fca9f61b691 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From a36199efe301a8fe16fe85391f488a5933a4f894 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Fri, 22 Nov 2024 10:01:28 +0000 Subject: [PATCH 21/41] JDK-8302459: simplify late inline failure conditions --- src/hotspot/share/opto/callGenerator.cpp | 1 - src/hotspot/share/opto/callnode.cpp | 51 ++++++++++++------------ src/hotspot/share/opto/callnode.hpp | 12 ++---- src/hotspot/share/opto/compile.cpp | 7 ++-- 4 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 36fca9f61b691..c834c0658511e 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -484,7 +484,6 @@ class LateInlineVirtualCallGenerator : public VirtualCallGenerator { virtual void do_late_inline(); virtual void set_callee_method(ciMethod* m) { - assert(_callee == nullptr, "repeated inlining attempt"); _callee = m; } diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index f0e6e951cb972..6d6f63e3ebf69 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1077,33 +1077,32 @@ bool CallStaticJavaNode::cmp( const Node &n ) const { Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { CallGenerator* cg = generator(); - CallGenerator* fcg = failed_generator(); - if (fcg != nullptr) { - if (!late_inline_added()) { - phase->C->prepend_late_inline(fcg); + if (cg != nullptr) { + if (late_inline_failed()) { + phase->C->prepend_late_inline(cg); phase->C->inc_number_of_mh_late_inlines(); - set_late_inline_added(true); - } - } else if (can_reshape && cg != nullptr) { - assert(IncrementalInlineMH, "required"); - assert(cg->call_node() == this, "mismatch"); - assert(cg->is_mh_late_inline(), "not virtual"); - - // Check whether this MH handle call becomes a candidate for inlining. - ciMethod* callee = cg->method(); - vmIntrinsics::ID iid = callee->intrinsic_id(); - if (iid == vmIntrinsics::_invokeBasic) { - if (in(TypeFunc::Parms)->Opcode() == Op_ConP) { - phase->C->prepend_late_inline(cg); - set_generator(nullptr); - } - } else if (iid == vmIntrinsics::_linkToNative) { - // never retry - } else { - assert(callee->has_member_arg(), "wrong type of call?"); - if (in(TypeFunc::Parms + callee->arg_size() - 1)->Opcode() == Op_ConP) { - phase->C->prepend_late_inline(cg); - set_generator(nullptr); + set_generator(nullptr); + } else if (can_reshape) { + assert(IncrementalInlineMH, "required"); + assert(cg->call_node() == this, "mismatch"); + assert(cg->is_mh_late_inline(), "not virtual"); + + // Check whether this MH handle call becomes a candidate for inlining. + ciMethod* callee = cg->method(); + vmIntrinsics::ID iid = callee->intrinsic_id(); + if (iid == vmIntrinsics::_invokeBasic) { + if (in(TypeFunc::Parms)->Opcode() == Op_ConP) { + phase->C->prepend_late_inline(cg); + set_generator(nullptr); + } + } else if (iid == vmIntrinsics::_linkToNative) { + // never retry + } else { + assert(callee->has_member_arg(), "wrong type of call?"); + if (in(TypeFunc::Parms + callee->arg_size() - 1)->Opcode() == Op_ConP) { + phase->C->prepend_late_inline(cg); + set_generator(nullptr); + } } } } diff --git a/src/hotspot/share/opto/callnode.hpp b/src/hotspot/share/opto/callnode.hpp index 33f41fb370bb9..1ecc375a9231f 100644 --- a/src/hotspot/share/opto/callnode.hpp +++ b/src/hotspot/share/opto/callnode.hpp @@ -683,8 +683,7 @@ class CallNode : public SafePointNode { float _cnt; // Estimate of number of times called CallGenerator* _generator; // corresponding CallGenerator for some late inline calls const char* _name; // Printable name, if _method is null - CallGenerator* _failed_generator; - bool _late_inline_added; + bool _late_inline_failed; CallNode(const TypeFunc* tf, address addr, const TypePtr* adr_type, JVMState* jvms = nullptr) : SafePointNode(tf->domain()->cnt(), jvms, adr_type), @@ -693,8 +692,7 @@ class CallNode : public SafePointNode { _cnt(COUNT_UNKNOWN), _generator(nullptr), _name(nullptr), - _failed_generator(nullptr), - _late_inline_added(false) + _late_inline_failed(false) { init_class_id(Class_Call); } @@ -703,15 +701,13 @@ class CallNode : public SafePointNode { address entry_point() const { return _entry_point; } float cnt() const { return _cnt; } CallGenerator* generator() const { return _generator; } - CallGenerator* failed_generator() const { return _failed_generator; } - bool late_inline_added() const { return _late_inline_added; } + bool late_inline_failed() const { return _late_inline_failed; } void set_tf(const TypeFunc* tf) { _tf = tf; } void set_entry_point(address p) { _entry_point = p; } void set_cnt(float c) { _cnt = c; } void set_generator(CallGenerator* cg) { _generator = cg; } - void set_failed_generator(CallGenerator* cg) { _failed_generator = cg; } - void set_late_inline_added(bool added) { _late_inline_added = added; } + void set_late_inline_failed(bool failed) { _late_inline_failed = failed; } virtual const Type* bottom_type() const; virtual const Type* Value(PhaseGVN* phase) const; diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 10de0b58dcefc..9cb80fa7e74c4 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2051,7 +2051,8 @@ bool Compile::inline_incrementally_one() { print_method(PHASE_INCREMENTAL_INLINE_STEP, 3, cg->call_node()); break; // process one call site at a time } else { - cg->call_node()->set_failed_generator(cg); + cg->call_node()->set_generator(cg); + cg->call_node()->set_late_inline_failed(true); add_failed_late_inline(cg); } } else { @@ -2088,8 +2089,8 @@ void Compile::inline_incrementally_cleanup(PhaseIterGVN& igvn) { for (int i = 0; i < _failed_late_inlines.length(); i++) { CallGenerator* cg = _failed_late_inlines.at(i); CallNode* cn = cg->call_node(); - cn->set_failed_generator(nullptr); - cn->set_late_inline_added(false); + cn->set_generator(nullptr); + cn->set_late_inline_failed(false); } _failed_late_inlines.clear(); if (failing()) return; From 127a21bd9cecccc64529a14b92744e666f4d7c54 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Fri, 22 Nov 2024 12:54:05 +0000 Subject: [PATCH 22/41] JDK-8302459: copyright year --- src/hotspot/share/opto/callGenerator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index c834c0658511e..8adfe41ad870e 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 99eaf0a69d69e31e1b6b0740031c883957eea2c4 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Fri, 22 Nov 2024 13:26:00 +0000 Subject: [PATCH 23/41] JDK-8032459: fix indentation --- src/hotspot/share/opto/compile.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 9cb80fa7e74c4..d489e7a889bdf 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2087,10 +2087,10 @@ void Compile::inline_incrementally_cleanup(PhaseIterGVN& igvn) { igvn.optimize(); // Reset failed generator in call node for (int i = 0; i < _failed_late_inlines.length(); i++) { - CallGenerator* cg = _failed_late_inlines.at(i); - CallNode* cn = cg->call_node(); - cn->set_generator(nullptr); - cn->set_late_inline_failed(false); + CallGenerator* cg = _failed_late_inlines.at(i); + CallNode* cn = cg->call_node(); + cn->set_generator(nullptr); + cn->set_late_inline_failed(false); } _failed_late_inlines.clear(); if (failing()) return; From 079d4759e82c5199583781dbd5df6a04ef0f876d Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Wed, 27 Nov 2024 12:42:18 +0000 Subject: [PATCH 24/41] JDK-8302459: add failed late inlines handling to dynamic calls --- src/hotspot/share/opto/callGenerator.cpp | 1 + src/hotspot/share/opto/callnode.cpp | 71 +++++++++++++----------- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 8adfe41ad870e..514fece6bb479 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -484,6 +484,7 @@ class LateInlineVirtualCallGenerator : public VirtualCallGenerator { virtual void do_late_inline(); virtual void set_callee_method(ciMethod* m) { + assert(_callee == nullptr, "repeated inlining attempt"); _callee = m; } diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index 6d6f63e3ebf69..32985b38678cc 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1080,7 +1080,9 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (cg != nullptr) { if (late_inline_failed()) { phase->C->prepend_late_inline(cg); - phase->C->inc_number_of_mh_late_inlines(); + if (cg->is_mh_late_inline()) { + phase->C->inc_number_of_mh_late_inlines(); + } set_generator(nullptr); } else if (can_reshape) { assert(IncrementalInlineMH, "required"); @@ -1171,40 +1173,45 @@ bool CallDynamicJavaNode::cmp( const Node &n ) const { Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { CallGenerator* cg = generator(); - if (can_reshape && cg != nullptr) { - assert(IncrementalInlineVirtual, "required"); - assert(cg->call_node() == this, "mismatch"); - assert(cg->is_virtual_late_inline(), "not virtual"); - - // Recover symbolic info for method resolution. - ciMethod* caller = jvms()->method(); - ciBytecodeStream iter(caller); - iter.force_bci(jvms()->bci()); - - bool not_used1; - ciSignature* not_used2; - ciMethod* orig_callee = iter.get_method(not_used1, ¬_used2); // callee in the bytecode - ciKlass* holder = iter.get_declared_method_holder(); - if (orig_callee->is_method_handle_intrinsic()) { - assert(_override_symbolic_info, "required"); - orig_callee = method(); - holder = method()->holder(); - } + if (cg != nullptr) { + if (late_inline_failed()) { + phase->C->prepend_late_inline(cg); + set_generator(nullptr); + } else if (can_reshape) { + assert(IncrementalInlineVirtual, "required"); + assert(cg->call_node() == this, "mismatch"); + assert(cg->is_virtual_late_inline(), "not virtual"); + + // Recover symbolic info for method resolution. + ciMethod* caller = jvms()->method(); + ciBytecodeStream iter(caller); + iter.force_bci(jvms()->bci()); + + bool not_used1; + ciSignature* not_used2; + ciMethod* orig_callee = iter.get_method(not_used1, ¬_used2); // callee in the bytecode + ciKlass* holder = iter.get_declared_method_holder(); + if (orig_callee->is_method_handle_intrinsic()) { + assert(_override_symbolic_info, "required"); + orig_callee = method(); + holder = method()->holder(); + } - ciInstanceKlass* klass = ciEnv::get_instance_klass_for_declared_method_holder(holder); + ciInstanceKlass* klass = ciEnv::get_instance_klass_for_declared_method_holder(holder); - Node* receiver_node = in(TypeFunc::Parms); - const TypeOopPtr* receiver_type = phase->type(receiver_node)->isa_oopptr(); + Node* receiver_node = in(TypeFunc::Parms); + const TypeOopPtr* receiver_type = phase->type(receiver_node)->isa_oopptr(); - int not_used3; - bool call_does_dispatch; - ciMethod* callee = phase->C->optimize_virtual_call(caller, klass, holder, orig_callee, receiver_type, true /*is_virtual*/, - call_does_dispatch, not_used3); // out-parameters - if (!call_does_dispatch) { - // Register for late inlining. - cg->set_callee_method(callee); - phase->C->prepend_late_inline(cg); // MH late inlining prepends to the list, so do the same - set_generator(nullptr); + int not_used3; + bool call_does_dispatch; + ciMethod* callee = phase->C->optimize_virtual_call(caller, klass, holder, orig_callee, receiver_type, true /*is_virtual*/, + call_does_dispatch, not_used3); // out-parameters + if (!call_does_dispatch) { + // Register for late inlining. + cg->set_callee_method(callee); + phase->C->prepend_late_inline(cg); // MH late inlining prepends to the list, so do the same + set_generator(nullptr); + } } } return CallNode::Ideal(phase, can_reshape); From 15246bcab04613436269a0046ba131513d4ea9bf Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Wed, 27 Nov 2024 16:20:32 +0100 Subject: [PATCH 25/41] JDK-8302459: remove unneeded copyright year change --- src/hotspot/share/opto/callGenerator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 514fece6bb479..36fca9f61b691 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From b05308f7f2aad68acf7ca3c1c70f158ba18aa51d Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Mon, 9 Dec 2024 19:58:15 -0800 Subject: [PATCH 26/41] remove failed late inline tracking --- src/hotspot/share/opto/callGenerator.cpp | 6 +++- src/hotspot/share/opto/callnode.cpp | 35 ++++++++++++------------ src/hotspot/share/opto/callnode.hpp | 6 +--- src/hotspot/share/opto/compile.cpp | 16 +++-------- src/hotspot/share/opto/compile.hpp | 5 ---- 5 files changed, 28 insertions(+), 40 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 36fca9f61b691..88dae90a4a859 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -696,7 +696,11 @@ void CallGenerator::do_late_inline_helper() { return; } if (C->print_inlining() && (is_mh_late_inline() || is_virtual_late_inline())) { - C->print_inlining_update_delayed(this); + if (inline_cg()->is_intrinsic()) { + assert(C->print_inlining_stream()->is_empty(), ""); // FIXME: fix for 8320237 is incomplete (observed w/ -XX:+AlwaysIncrementalInlining) + } else { + C->print_inlining_update_delayed(this); + } } // Setup default node notes to be picked up by the inlining diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index 32985b38678cc..ed36e91636891 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1077,19 +1077,12 @@ bool CallStaticJavaNode::cmp( const Node &n ) const { Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { CallGenerator* cg = generator(); - if (cg != nullptr) { - if (late_inline_failed()) { - phase->C->prepend_late_inline(cg); - if (cg->is_mh_late_inline()) { - phase->C->inc_number_of_mh_late_inlines(); - } - set_generator(nullptr); - } else if (can_reshape) { + if (cg != nullptr && can_reshape) { + if (cg->is_mh_late_inline()) { + // Check whether this MH handle call becomes a candidate for inlining. assert(IncrementalInlineMH, "required"); assert(cg->call_node() == this, "mismatch"); - assert(cg->is_mh_late_inline(), "not virtual"); - - // Check whether this MH handle call becomes a candidate for inlining. + assert(cg->method()->is_method_handle_intrinsic(), "required"); ciMethod* callee = cg->method(); vmIntrinsics::ID iid = callee->intrinsic_id(); if (iid == vmIntrinsics::_invokeBasic) { @@ -1103,9 +1096,16 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { assert(callee->has_member_arg(), "wrong type of call?"); if (in(TypeFunc::Parms + callee->arg_size() - 1)->Opcode() == Op_ConP) { phase->C->prepend_late_inline(cg); + phase->C->inc_number_of_mh_late_inlines(); set_generator(nullptr); } } + } else { + // TODO: additional checks to ensure the call is actually "inlineable" now + assert(IncrementalInline, "required"); + assert(cg->method()->is_method_handle_intrinsic() == false, "required"); + phase->C->prepend_late_inline(cg); + set_generator(nullptr); } } return CallNode::Ideal(phase, can_reshape); @@ -1173,14 +1173,10 @@ bool CallDynamicJavaNode::cmp( const Node &n ) const { Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { CallGenerator* cg = generator(); - if (cg != nullptr) { - if (late_inline_failed()) { - phase->C->prepend_late_inline(cg); - set_generator(nullptr); - } else if (can_reshape) { + if (cg != nullptr && can_reshape) { + if (cg->is_virtual_late_inline()) { assert(IncrementalInlineVirtual, "required"); assert(cg->call_node() == this, "mismatch"); - assert(cg->is_virtual_late_inline(), "not virtual"); // Recover symbolic info for method resolution. ciMethod* caller = jvms()->method(); @@ -1212,6 +1208,11 @@ Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { phase->C->prepend_late_inline(cg); // MH late inlining prepends to the list, so do the same set_generator(nullptr); } + } else { + // TODO: additional checks to ensure the call is "inlineable" now + assert(IncrementalInline, "required"); + phase->C->prepend_late_inline(cg); + set_generator(nullptr); } } return CallNode::Ideal(phase, can_reshape); diff --git a/src/hotspot/share/opto/callnode.hpp b/src/hotspot/share/opto/callnode.hpp index 1ecc375a9231f..2d3835b71ad42 100644 --- a/src/hotspot/share/opto/callnode.hpp +++ b/src/hotspot/share/opto/callnode.hpp @@ -683,7 +683,6 @@ class CallNode : public SafePointNode { float _cnt; // Estimate of number of times called CallGenerator* _generator; // corresponding CallGenerator for some late inline calls const char* _name; // Printable name, if _method is null - bool _late_inline_failed; CallNode(const TypeFunc* tf, address addr, const TypePtr* adr_type, JVMState* jvms = nullptr) : SafePointNode(tf->domain()->cnt(), jvms, adr_type), @@ -691,8 +690,7 @@ class CallNode : public SafePointNode { _entry_point(addr), _cnt(COUNT_UNKNOWN), _generator(nullptr), - _name(nullptr), - _late_inline_failed(false) + _name(nullptr) { init_class_id(Class_Call); } @@ -701,13 +699,11 @@ class CallNode : public SafePointNode { address entry_point() const { return _entry_point; } float cnt() const { return _cnt; } CallGenerator* generator() const { return _generator; } - bool late_inline_failed() const { return _late_inline_failed; } void set_tf(const TypeFunc* tf) { _tf = tf; } void set_entry_point(address p) { _entry_point = p; } void set_cnt(float c) { _cnt = c; } void set_generator(CallGenerator* cg) { _generator = cg; } - void set_late_inline_failed(bool failed) { _late_inline_failed = failed; } virtual const Type* bottom_type() const; virtual const Type* Value(PhaseGVN* phase) const; diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 94935e47c832b..3720da8c81235 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -665,7 +665,6 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _types(nullptr), _node_hash(nullptr), _late_inlines(comp_arena(), 2, 0, nullptr), - _failed_late_inlines(comp_arena(), 2, 0, nullptr), _string_late_inlines(comp_arena(), 2, 0, nullptr), _boxing_late_inlines(comp_arena(), 2, 0, nullptr), _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr), @@ -2040,6 +2039,7 @@ bool Compile::inline_incrementally_one() { for (int i = 0; i < _late_inlines.length(); i++) { _late_inlines_pos = i+1; CallGenerator* cg = _late_inlines.at(i); + DEBUG_ONLY( bool is_scheduled_for_igvn_before = C->igvn_worklist()->member(cg->call_node()); ) bool does_dispatch = cg->is_virtual_late_inline() || cg->is_mh_late_inline(); if (inlining_incrementally() || does_dispatch) { // a call can be either inlined or strength-reduced to a direct call cg->do_late_inline(); @@ -2050,10 +2050,10 @@ bool Compile::inline_incrementally_one() { _late_inlines_pos = i+1; // restore the position in case new elements were inserted print_method(PHASE_INCREMENTAL_INLINE_STEP, 3, cg->call_node()); break; // process one call site at a time - } else { + } else /*if (UseNewCode)*/ { + assert(C->igvn_worklist()->member(cg->call_node()) == is_scheduled_for_igvn_before, + "call node shouldn't be scheduled for IGVN"); // avoid infinite loop cg->call_node()->set_generator(cg); - cg->call_node()->set_late_inline_failed(true); - add_failed_late_inline(cg); } } else { // Ignore late inline direct calls when inlining is not allowed. @@ -2085,14 +2085,6 @@ void Compile::inline_incrementally_cleanup(PhaseIterGVN& igvn) { TracePhase tp(_t_incrInline_igvn); igvn.reset_from_gvn(initial_gvn()); igvn.optimize(); - // Reset failed generator in call node - for (int i = 0; i < _failed_late_inlines.length(); i++) { - CallGenerator* cg = _failed_late_inlines.at(i); - CallNode* cn = cg->call_node(); - cn->set_generator(nullptr); - cn->set_late_inline_failed(false); - } - _failed_late_inlines.clear(); if (failing()) return; } print_method(PHASE_INCREMENTAL_INLINE_CLEANUP, 3); diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 4265fccc15ad3..7c251bd4bd934 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -461,7 +461,6 @@ class Compile : public Phase { NodeHash* _node_hash; GrowableArray _late_inlines; // List of CallGenerators to be revisited after main parsing has finished. - GrowableArray _failed_late_inlines; // List of CallGenerators that have failed inlining GrowableArray _string_late_inlines; // same but for string operations GrowableArray _boxing_late_inlines; // same but for boxing operations @@ -1083,10 +1082,6 @@ class Compile : public Phase { _late_inlines.insert_before(0, cg); } - void add_failed_late_inline(CallGenerator* cg) { - _failed_late_inlines.push(cg); - } - void add_string_late_inline(CallGenerator* cg) { _string_late_inlines.push(cg); } From 4f0a890243016229e85657e4108423853c76c07f Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Tue, 10 Dec 2024 16:14:58 -0800 Subject: [PATCH 27/41] Support repeated inlining attempts for virtual late inlines --- src/hotspot/share/opto/callGenerator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 88dae90a4a859..d8da3137d0975 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -484,7 +484,7 @@ class LateInlineVirtualCallGenerator : public VirtualCallGenerator { virtual void do_late_inline(); virtual void set_callee_method(ciMethod* m) { - assert(_callee == nullptr, "repeated inlining attempt"); + assert(_callee == nullptr || _callee == m, ""); // for repeated inlining attempts _callee = m; } From 1c747253ab388de1b376569b8c9ddb5640f90054 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 12 Dec 2024 10:15:18 +0000 Subject: [PATCH 28/41] JDK-8302459: add check to avoid infinite loop --- src/hotspot/share/opto/compile.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 7628787a88ab1..cd9c62a3bf3ef 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2039,7 +2039,7 @@ bool Compile::inline_incrementally_one() { for (int i = 0; i < _late_inlines.length(); i++) { _late_inlines_pos = i+1; CallGenerator* cg = _late_inlines.at(i); - DEBUG_ONLY( bool is_scheduled_for_igvn_before = C->igvn_worklist()->member(cg->call_node()); ) + bool is_scheduled_for_igvn_before = C->igvn_worklist()->member(cg->call_node()); bool does_dispatch = cg->is_virtual_late_inline() || cg->is_mh_late_inline(); if (inlining_incrementally() || does_dispatch) { // a call can be either inlined or strength-reduced to a direct call cg->do_late_inline(); @@ -2050,10 +2050,12 @@ bool Compile::inline_incrementally_one() { _late_inlines_pos = i+1; // restore the position in case new elements were inserted print_method(PHASE_INCREMENTAL_INLINE_STEP, 3, cg->call_node()); break; // process one call site at a time - } else /*if (UseNewCode)*/ { - assert(C->igvn_worklist()->member(cg->call_node()) == is_scheduled_for_igvn_before, - "call node shouldn't be scheduled for IGVN"); // avoid infinite loop - cg->call_node()->set_generator(cg); + } else { + if (C->igvn_worklist()->member(cg->call_node()) == is_scheduled_for_igvn_before) { // avoid potential infinite loop + cg->call_node()->set_generator(cg); + } else { + assert(false, "call node shouldn't be scheduled for IGVN"); + } } } else { // Ignore late inline direct calls when inlining is not allowed. From a0fb8007963fb1a0ddb2bb75d8f0468db59c2b67 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 12 Dec 2024 12:51:47 +0000 Subject: [PATCH 29/41] JDK-8302459: remove todos --- src/hotspot/share/opto/callnode.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index ed36e91636891..e0ae070e55ac6 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1101,7 +1101,6 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { } } } else { - // TODO: additional checks to ensure the call is actually "inlineable" now assert(IncrementalInline, "required"); assert(cg->method()->is_method_handle_intrinsic() == false, "required"); phase->C->prepend_late_inline(cg); @@ -1209,7 +1208,6 @@ Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { set_generator(nullptr); } } else { - // TODO: additional checks to ensure the call is "inlineable" now assert(IncrementalInline, "required"); phase->C->prepend_late_inline(cg); set_generator(nullptr); From 65e7de024abc39e331eb858acb5ed9c8e262bf32 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 12 Dec 2024 16:21:55 +0000 Subject: [PATCH 30/41] JDK-8302459: add logging --- src/hotspot/share/opto/callnode.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index e0ae070e55ac6..867cd567bd036 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1103,6 +1103,9 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { } else { assert(IncrementalInline, "required"); assert(cg->method()->is_method_handle_intrinsic() == false, "required"); + if (phase->C->print_inlining()) { + cg->print_inlining_late(InliningResult::FAILURE, "static call node changed: trying again"); + } phase->C->prepend_late_inline(cg); set_generator(nullptr); } @@ -1209,6 +1212,9 @@ Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { } } else { assert(IncrementalInline, "required"); + if (phase->C->print_inlining()) { + cg->print_inlining_late(InliningResult::FAILURE, "dynamic call node changed: trying again"); + } phase->C->prepend_late_inline(cg); set_generator(nullptr); } From f36c5b7924606ad972d137595a203ca1336fc769 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 20 Feb 2025 12:22:02 +0000 Subject: [PATCH 31/41] JDK-8302459: fix after merge --- src/hotspot/share/opto/callGenerator.cpp | 7 ------- src/hotspot/share/opto/callnode.cpp | 6 ++++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 42db0c501e601..0947b9da65fed 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -662,13 +662,6 @@ void CallGenerator::do_late_inline_helper() { map->disconnect_inputs(C); return; } - if (C->print_inlining() && (is_mh_late_inline() || is_virtual_late_inline())) { - if (inline_cg()->is_intrinsic()) { - assert(C->print_inlining_stream()->is_empty(), ""); // FIXME: fix for 8320237 is incomplete (observed w/ -XX:+AlwaysIncrementalInlining) - } else { - C->print_inlining_update_delayed(this); - } - } // Setup default node notes to be picked up by the inlining Node_Notes* old_nn = C->node_notes_at(call->_idx); diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index cead9e9378bb0..d1b7381abda0c 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1105,7 +1105,8 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { assert(IncrementalInline, "required"); assert(cg->method()->is_method_handle_intrinsic() == false, "required"); if (phase->C->print_inlining()) { - cg->print_inlining_late(InliningResult::FAILURE, "static call node changed: trying again"); + phase->C->inline_printer()->record(cg->method(), cg->call_node()->jvms(), InliningResult::FAILURE, + "static call node changed: trying again"); } phase->C->prepend_late_inline(cg); set_generator(nullptr); @@ -1214,7 +1215,8 @@ Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { } else { assert(IncrementalInline, "required"); if (phase->C->print_inlining()) { - cg->print_inlining_late(InliningResult::FAILURE, "dynamic call node changed: trying again"); + phase->C->inline_printer()->record(cg->method(), cg->call_node()->jvms(), InliningResult::FAILURE, + "dynamic call node changed: trying again"); } phase->C->prepend_late_inline(cg); set_generator(nullptr); From 1b2292355fcdca9d72ff1fdf29b5f52bc3938ebf Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 20 Feb 2025 12:24:02 +0000 Subject: [PATCH 32/41] JDK-8302459: fix copyright year --- .../jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java | 2 +- .../jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java index c4ad05c9b9693..7e16c5e5e8645 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorGatherMaskFoldingTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java index 61f733429ec9d..1a2402222ed84 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorLogicalOpIdentityTest.java @@ -1,6 +1,6 @@ /* * Copyright (c) 2022, 2023, Arm Limited. All rights reserved. - * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From ba3547daf89916162bd208b8b8aee575b6991cea Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 27 Feb 2025 12:33:47 +0000 Subject: [PATCH 33/41] JDK-8302459: update assert string --- src/hotspot/share/opto/callGenerator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 0947b9da65fed..5f010d5c42799 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -470,7 +470,7 @@ class LateInlineVirtualCallGenerator : public VirtualCallGenerator { virtual void do_late_inline(); virtual void set_callee_method(ciMethod* m) { - assert(_callee == nullptr || _callee == m, ""); // for repeated inlining attempts + assert(_callee == nullptr || _callee == m, "repeated inline attempt with different callee"); _callee = m; } From 2521e85592f4b797ff708318652a0515ad91763c Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 27 Feb 2025 13:50:03 +0000 Subject: [PATCH 34/41] JDK-8302459: unneeded changes --- src/hotspot/share/opto/callnode.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index d1b7381abda0c..a52a57159b95d 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1078,12 +1078,12 @@ bool CallStaticJavaNode::cmp( const Node &n ) const { Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { CallGenerator* cg = generator(); - if (cg != nullptr && can_reshape) { + if (can_reshape && cg != nullptr) { if (cg->is_mh_late_inline()) { - // Check whether this MH handle call becomes a candidate for inlining. assert(IncrementalInlineMH, "required"); assert(cg->call_node() == this, "mismatch"); assert(cg->method()->is_method_handle_intrinsic(), "required"); + // Check whether this MH handle call becomes a candidate for inlining. ciMethod* callee = cg->method(); vmIntrinsics::ID iid = callee->intrinsic_id(); if (iid == vmIntrinsics::_invokeBasic) { @@ -1177,7 +1177,7 @@ bool CallDynamicJavaNode::cmp( const Node &n ) const { Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { CallGenerator* cg = generator(); - if (cg != nullptr && can_reshape) { + if (can_reshape && cg != nullptr) { if (cg->is_virtual_late_inline()) { assert(IncrementalInlineVirtual, "required"); assert(cg->call_node() == this, "mismatch"); From e71e72f52a245e122cee8efa32915724bcb67ad3 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Thu, 27 Feb 2025 13:54:41 +0000 Subject: [PATCH 35/41] JDK-8302459: unneeded changes --- src/hotspot/share/opto/callnode.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index a52a57159b95d..15960d27fe6e6 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1083,6 +1083,7 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { assert(IncrementalInlineMH, "required"); assert(cg->call_node() == this, "mismatch"); assert(cg->method()->is_method_handle_intrinsic(), "required"); + // Check whether this MH handle call becomes a candidate for inlining. ciMethod* callee = cg->method(); vmIntrinsics::ID iid = callee->intrinsic_id(); From d688fcfe407af8107610e3e73a608f28ab04f3ed Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Fri, 7 Mar 2025 10:25:31 +0000 Subject: [PATCH 36/41] JDK-8302459: create method to prepend and reset generator --- src/hotspot/share/opto/callnode.cpp | 20 ++++++++++---------- src/hotspot/share/opto/callnode.hpp | 1 + 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index 15960d27fe6e6..56805bea38129 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1069,6 +1069,11 @@ void CallJavaNode::dump_compact_spec(outputStream* st) const { } #endif +void CallJavaNode::prepend_and_reset_generator(PhaseGVN* phase, CallGenerator* cg) { + phase->C->prepend_late_inline(cg); + set_generator(nullptr); +} + //============================================================================= uint CallStaticJavaNode::size_of() const { return sizeof(*this); } bool CallStaticJavaNode::cmp( const Node &n ) const { @@ -1089,17 +1094,15 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { vmIntrinsics::ID iid = callee->intrinsic_id(); if (iid == vmIntrinsics::_invokeBasic) { if (in(TypeFunc::Parms)->Opcode() == Op_ConP) { - phase->C->prepend_late_inline(cg); - set_generator(nullptr); + prepend_and_reset_generator(phase, cg); } } else if (iid == vmIntrinsics::_linkToNative) { // never retry } else { assert(callee->has_member_arg(), "wrong type of call?"); if (in(TypeFunc::Parms + callee->arg_size() - 1)->Opcode() == Op_ConP) { - phase->C->prepend_late_inline(cg); + prepend_and_reset_generator(phase, cg); phase->C->inc_number_of_mh_late_inlines(); - set_generator(nullptr); } } } else { @@ -1109,8 +1112,7 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { phase->C->inline_printer()->record(cg->method(), cg->call_node()->jvms(), InliningResult::FAILURE, "static call node changed: trying again"); } - phase->C->prepend_late_inline(cg); - set_generator(nullptr); + prepend_and_reset_generator(phase, cg); } } return CallNode::Ideal(phase, can_reshape); @@ -1210,8 +1212,7 @@ Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (!call_does_dispatch) { // Register for late inlining. cg->set_callee_method(callee); - phase->C->prepend_late_inline(cg); // MH late inlining prepends to the list, so do the same - set_generator(nullptr); + prepend_and_reset_generator(phase, cg); // MH late inlining prepends to the list, so do the same } } else { assert(IncrementalInline, "required"); @@ -1219,8 +1220,7 @@ Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { phase->C->inline_printer()->record(cg->method(), cg->call_node()->jvms(), InliningResult::FAILURE, "dynamic call node changed: trying again"); } - phase->C->prepend_late_inline(cg); - set_generator(nullptr); + prepend_and_reset_generator(phase, cg); } } return CallNode::Ideal(phase, can_reshape); diff --git a/src/hotspot/share/opto/callnode.hpp b/src/hotspot/share/opto/callnode.hpp index ad5e9daa8f69d..867469e08d680 100644 --- a/src/hotspot/share/opto/callnode.hpp +++ b/src/hotspot/share/opto/callnode.hpp @@ -794,6 +794,7 @@ class CallJavaNode : public CallNode { void set_arg_escape(bool f) { _arg_escape = f; } bool arg_escape() const { return _arg_escape; } void copy_call_debug_info(PhaseIterGVN* phase, SafePointNode *sfpt); + void prepend_and_reset_generator(PhaseGVN* phase, CallGenerator* cg); DEBUG_ONLY( bool validate_symbolic_info() const; ) From 887dd96c55dd02822e6d5d8f9c88b0102fa1c432 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 10 Mar 2025 09:22:32 +0000 Subject: [PATCH 37/41] JDK-8302459: reshape infinite loop check --- src/hotspot/share/opto/compile.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index a95cc5e0f48cb..21ee569271d9c 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2041,10 +2041,12 @@ bool Compile::inline_incrementally_one() { print_method(PHASE_INCREMENTAL_INLINE_STEP, 3, cg->call_node()); break; // process one call site at a time } else { - if (C->igvn_worklist()->member(cg->call_node()) == is_scheduled_for_igvn_before) { // avoid potential infinite loop - cg->call_node()->set_generator(cg); + bool is_scheduled_for_igvn_after = C->igvn_worklist()->member(cg->call_node()); + if (!is_scheduled_for_igvn_before && is_scheduled_for_igvn_after) { // avoid potential infinite loop + assert(false, "scheduled for IGVN during inlining attempt"); } else { - assert(false, "call node shouldn't be scheduled for IGVN"); + assert(is_scheduled_for_igvn_before == is_scheduled_for_igvn_after, "call node removed from IGVN list during inlining pass"); + cg->call_node()->set_generator(cg); } } } else { From 9406c6e2baebabe7c1d0baf8f170b7d802450b3e Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 10 Mar 2025 09:42:54 +0000 Subject: [PATCH 38/41] JDK-8302459: refactor helper method --- src/hotspot/share/opto/callnode.cpp | 20 ++++++++++++-------- src/hotspot/share/opto/callnode.hpp | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index 56805bea38129..3697df5d6513c 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1069,9 +1069,13 @@ void CallJavaNode::dump_compact_spec(outputStream* st) const { } #endif -void CallJavaNode::prepend_and_reset_generator(PhaseGVN* phase, CallGenerator* cg) { - phase->C->prepend_late_inline(cg); - set_generator(nullptr); +void CallJavaNode::register_for_late_inline() { + if (generator() != nullptr) { + Compile::current()->prepend_late_inline(generator()); + set_generator(nullptr); + } else { + assert(false, "repeated inline attempt"); + } } //============================================================================= @@ -1094,14 +1098,14 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { vmIntrinsics::ID iid = callee->intrinsic_id(); if (iid == vmIntrinsics::_invokeBasic) { if (in(TypeFunc::Parms)->Opcode() == Op_ConP) { - prepend_and_reset_generator(phase, cg); + register_for_late_inline(); } } else if (iid == vmIntrinsics::_linkToNative) { // never retry } else { assert(callee->has_member_arg(), "wrong type of call?"); if (in(TypeFunc::Parms + callee->arg_size() - 1)->Opcode() == Op_ConP) { - prepend_and_reset_generator(phase, cg); + register_for_late_inline(); phase->C->inc_number_of_mh_late_inlines(); } } @@ -1112,7 +1116,7 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { phase->C->inline_printer()->record(cg->method(), cg->call_node()->jvms(), InliningResult::FAILURE, "static call node changed: trying again"); } - prepend_and_reset_generator(phase, cg); + register_for_late_inline(); } } return CallNode::Ideal(phase, can_reshape); @@ -1212,7 +1216,7 @@ Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (!call_does_dispatch) { // Register for late inlining. cg->set_callee_method(callee); - prepend_and_reset_generator(phase, cg); // MH late inlining prepends to the list, so do the same + register_for_late_inline(); // MH late inlining prepends to the list, so do the same } } else { assert(IncrementalInline, "required"); @@ -1220,7 +1224,7 @@ Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { phase->C->inline_printer()->record(cg->method(), cg->call_node()->jvms(), InliningResult::FAILURE, "dynamic call node changed: trying again"); } - prepend_and_reset_generator(phase, cg); + register_for_late_inline(); } } return CallNode::Ideal(phase, can_reshape); diff --git a/src/hotspot/share/opto/callnode.hpp b/src/hotspot/share/opto/callnode.hpp index 867469e08d680..bc1b06060c6bc 100644 --- a/src/hotspot/share/opto/callnode.hpp +++ b/src/hotspot/share/opto/callnode.hpp @@ -794,7 +794,7 @@ class CallJavaNode : public CallNode { void set_arg_escape(bool f) { _arg_escape = f; } bool arg_escape() const { return _arg_escape; } void copy_call_debug_info(PhaseIterGVN* phase, SafePointNode *sfpt); - void prepend_and_reset_generator(PhaseGVN* phase, CallGenerator* cg); + void register_for_late_inline(); DEBUG_ONLY( bool validate_symbolic_info() const; ) From bbaf98595c35ed0318fed0779fd152d3df5c2c83 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 24 Mar 2025 08:53:21 +0100 Subject: [PATCH 39/41] Update src/hotspot/share/opto/callnode.cpp Co-authored-by: Tobias Hartmann --- src/hotspot/share/opto/callnode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index 3697df5d6513c..6907f8b4ed327 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1111,7 +1111,7 @@ Node* CallStaticJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { } } else { assert(IncrementalInline, "required"); - assert(cg->method()->is_method_handle_intrinsic() == false, "required"); + assert(!cg->method()->is_method_handle_intrinsic(), "required"); if (phase->C->print_inlining()) { phase->C->inline_printer()->record(cg->method(), cg->call_node()->jvms(), InliningResult::FAILURE, "static call node changed: trying again"); From cfa5252bbf1c956fbb713b348326eee254b44b38 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 24 Mar 2025 08:57:00 +0100 Subject: [PATCH 40/41] Update src/hotspot/share/opto/callnode.cpp Co-authored-by: Tobias Hartmann --- src/hotspot/share/opto/callnode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp index 6907f8b4ed327..46e6b6aa1700e 100644 --- a/src/hotspot/share/opto/callnode.cpp +++ b/src/hotspot/share/opto/callnode.cpp @@ -1212,7 +1212,7 @@ Node* CallDynamicJavaNode::Ideal(PhaseGVN* phase, bool can_reshape) { int not_used3; bool call_does_dispatch; ciMethod* callee = phase->C->optimize_virtual_call(caller, klass, holder, orig_callee, receiver_type, true /*is_virtual*/, - call_does_dispatch, not_used3); // out-parameters + call_does_dispatch, not_used3); // out-parameters if (!call_does_dispatch) { // Register for late inlining. cg->set_callee_method(callee); From f84cafbe614577cdc771eb8b4679589c38b040c1 Mon Sep 17 00:00:00 2001 From: Damon Fenacci Date: Mon, 24 Mar 2025 12:06:06 +0000 Subject: [PATCH 41/41] JDK-8302459: add comments to asserts --- src/hotspot/share/opto/compile.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 21ee569271d9c..542ca6d6532ed 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2042,10 +2042,12 @@ bool Compile::inline_incrementally_one() { break; // process one call site at a time } else { bool is_scheduled_for_igvn_after = C->igvn_worklist()->member(cg->call_node()); - if (!is_scheduled_for_igvn_before && is_scheduled_for_igvn_after) { // avoid potential infinite loop + if (!is_scheduled_for_igvn_before && is_scheduled_for_igvn_after) { + // Avoid potential infinite loop if node already in the IGVN list assert(false, "scheduled for IGVN during inlining attempt"); } else { - assert(is_scheduled_for_igvn_before == is_scheduled_for_igvn_after, "call node removed from IGVN list during inlining pass"); + // Ensure call node has not disappeared from IGVN worklist during a failed inlining attempt + assert(!is_scheduled_for_igvn_before || is_scheduled_for_igvn_after, "call node removed from IGVN list during inlining pass"); cg->call_node()->set_generator(cg); } }