8254108: ciReplay: Support incremental inlining

Reviewed-by: dlong, thartmann

Author: chhagedorn

src/hotspot/share/ci/ciReplay.cpp

@@ -103,6 +103,7 @@ typedef struct _ciInlineRecord {
 
   int _inline_depth;
   int _inline_bci;
+  bool _inline_late;
 } ciInlineRecord;
 
 class  CompileReplay;
@@ -720,7 +721,7 @@ class CompileReplay : public StackObj {
     return NULL;
   }
 
-  // compile <klass> <name> <signature> <entry_bci> <comp_level> inline <count> (<depth> <bci> <klass> <name> <signature>)*
+  // compile <klass> <name> <signature> <entry_bci> <comp_level> inline <count> (<depth> <bci> <inline_late> <klass> <name> <signature>)*
   void process_compile(TRAPS) {
     Method* method = parse_method(CHECK);
     if (had_error()) return;
@@ -762,11 +763,19 @@ class CompileReplay : public StackObj {
         if (had_error()) {
           break;
         }
+        int inline_late = 0;
+        if (_version >= 2) {
+          inline_late = parse_int("inline_late");
+          if (had_error()) {
+              break;
+          }
+        }
+
         Method* inl_method = parse_method(CHECK);
         if (had_error()) {
           break;
         }
-        new_ciInlineRecord(inl_method, bci, depth);
+        new_ciInlineRecord(inl_method, bci, depth, inline_late);
       }
     }
     if (_imethod != NULL) {
@@ -1227,13 +1236,14 @@ class CompileReplay : public StackObj {
   }
 
   // Create and initialize a record for a ciInlineRecord
-  ciInlineRecord* new_ciInlineRecord(Method* method, int bci, int depth) {
+  ciInlineRecord* new_ciInlineRecord(Method* method, int bci, int depth, int inline_late) {
     ciInlineRecord* rec = NEW_RESOURCE_OBJ(ciInlineRecord);
     rec->_klass_name =  method->method_holder()->name()->as_utf8();
     rec->_method_name = method->name()->as_utf8();
     rec->_signature = method->signature()->as_utf8();
     rec->_inline_bci = bci;
     rec->_inline_depth = depth;
+    rec->_inline_late = inline_late;
     _ci_inline_records->append(rec);
     return rec;
   }
@@ -1470,23 +1480,33 @@ bool ciReplay::should_not_inline(ciMethod* method) {
   return replay_state->find_ciMethodRecord(method->get_Method()) == NULL;
 }
 
-bool ciReplay::should_inline(void* data, ciMethod* method, int bci, int inline_depth) {
+bool ciReplay::should_inline(void* data, ciMethod* method, int bci, int inline_depth, bool& should_delay) {
   if (data != NULL) {
-    GrowableArray<ciInlineRecord*>*  records = (GrowableArray<ciInlineRecord*>*)data;
+    GrowableArray<ciInlineRecord*>* records = (GrowableArray<ciInlineRecord*>*)data;
     VM_ENTRY_MARK;
     // Inline record are ordered by bci and depth.
-    return CompileReplay::find_ciInlineRecord(records, method->get_Method(), bci, inline_depth) != NULL;
+    ciInlineRecord* record = CompileReplay::find_ciInlineRecord(records, method->get_Method(), bci, inline_depth);
+    if (record == NULL) {
+      return false;
+    }
+    should_delay = record->_inline_late;
+    return true;
   } else if (replay_state != NULL) {
     VM_ENTRY_MARK;
     // Inline record are ordered by bci and depth.
-    return replay_state->find_ciInlineRecord(method->get_Method(), bci, inline_depth) != NULL;
+    ciInlineRecord* record = replay_state->find_ciInlineRecord(method->get_Method(), bci, inline_depth);
+    if (record == NULL) {
+      return false;
+    }
+    should_delay = record->_inline_late;
+    return true;
   }
   return false;
 }
 
 bool ciReplay::should_not_inline(void* data, ciMethod* method, int bci, int inline_depth) {
   if (data != NULL) {
-    GrowableArray<ciInlineRecord*>*  records = (GrowableArray<ciInlineRecord*>*)data;
+    GrowableArray<ciInlineRecord*>* records = (GrowableArray<ciInlineRecord*>*)data;
     VM_ENTRY_MARK;
     // Inline record are ordered by bci and depth.
     return CompileReplay::find_ciInlineRecord(records, method->get_Method(), bci, inline_depth) == NULL;

src/hotspot/share/ci/ciReplay.hpp

@@ -121,7 +121,7 @@ class ciReplay {
   static bool is_loaded(Method* method);
 
   static bool should_not_inline(ciMethod* method);
-  static bool should_inline(void* data, ciMethod* method, int bci, int inline_depth);
+  static bool should_inline(void* data, ciMethod* method, int bci, int inline_depth, bool& should_delay);
   static bool should_not_inline(void* data, ciMethod* method, int bci, int inline_depth);
 #endif
 
@@ -135,6 +135,7 @@ class ciReplay {
 // 0: legacy (no version number)
 // 1: first instanceKlass sets protection domain (8275868)
 //    replace current_mileage with invocation_count (8276095)
-#define REPLAY_VERSION 1 // current version, bump up for incompatible changes
+// 2: incremental inlining support (8254108)
+#define REPLAY_VERSION 2 // current version, bump up for incompatible changes
 
 #endif // SHARE_CI_CIREPLAY_HPP

src/hotspot/share/opto/bytecodeInfo.cpp

@@ -46,6 +46,7 @@ InlineTree::InlineTree(Compile* c,
   C(c),
   _caller_jvms(caller_jvms),
   _method(callee),
+  _late_inline(false),
   _caller_tree((InlineTree*) caller_tree),
   _count_inline_bcs(method()->code_size_for_inlining()),
   _max_inline_level(max_inline_level),
@@ -113,7 +114,7 @@ static bool is_unboxing_method(ciMethod* callee_method, Compile* C) {
 
 // positive filter: should callee be inlined?
 bool InlineTree::should_inline(ciMethod* callee_method, ciMethod* caller_method,
-                               int caller_bci, ciCallProfile& profile) {
+                               int caller_bci, NOT_PRODUCT_ARG(bool& should_delay) ciCallProfile& profile) {
   // Allows targeted inlining
   if (C->directive()->should_inline(callee_method)) {
     set_msg("force inline by CompileCommand");
@@ -128,9 +129,13 @@ bool InlineTree::should_inline(ciMethod* callee_method, ciMethod* caller_method,
   }
 
 #ifndef PRODUCT
-  int inline_depth = inline_level()+1;
-  if (ciReplay::should_inline(C->replay_inline_data(), callee_method, caller_bci, inline_depth)) {
-    set_msg("force inline by ciReplay");
+  int inline_depth = inline_level() + 1;
+  if (ciReplay::should_inline(C->replay_inline_data(), callee_method, caller_bci, inline_depth, should_delay)) {
+    if (should_delay) {
+      set_msg("force (incremental) inline by ciReplay");
+    } else {
+      set_msg("force inline by ciReplay");
+    }
     _forced_inline = true;
     return true;
   }
@@ -194,7 +199,7 @@ bool InlineTree::should_inline(ciMethod* callee_method, ciMethod* caller_method,
 
 // negative filter: should callee NOT be inlined?
 bool InlineTree::should_not_inline(ciMethod* callee_method, ciMethod* caller_method,
-                                   int caller_bci, ciCallProfile& profile) {
+                                   int caller_bci, NOT_PRODUCT_ARG(bool& should_delay) ciCallProfile& profile) {
   const char* fail_msg = NULL;
 
   // First check all inlining restrictions which are required for correctness
@@ -232,9 +237,13 @@ bool InlineTree::should_not_inline(ciMethod* callee_method, ciMethod* caller_met
   }
 
 #ifndef PRODUCT
-  int inline_depth = inline_level()+1;
-  if (ciReplay::should_inline(C->replay_inline_data(), callee_method, caller_bci, inline_depth)) {
-    set_msg("force inline by ciReplay");
+  int inline_depth = inline_level() + 1;
+  if (ciReplay::should_inline(C->replay_inline_data(), callee_method, caller_bci, inline_depth, should_delay)) {
+    if (should_delay) {
+      set_msg("force (incremental) inline by ciReplay");
+    } else {
+      set_msg("force inline by ciReplay");
+    }
     return false;
   }
 
@@ -369,10 +378,13 @@ bool InlineTree::try_to_inline(ciMethod* callee_method, ciMethod* caller_method,
   }
 
   _forced_inline = false; // Reset
-  if (!should_inline(callee_method, caller_method, caller_bci, profile)) {
+
+  // 'should_delay' can be overridden during replay compilation
+  if (!should_inline(callee_method, caller_method, caller_bci, NOT_PRODUCT_ARG(should_delay) profile)) {
     return false;
   }
-  if (should_not_inline(callee_method, caller_method, caller_bci, profile)) {
+  // 'should_delay' can be overridden during replay compilation
+  if (should_not_inline(callee_method, caller_method, caller_bci, NOT_PRODUCT_ARG(should_delay) profile)) {
     return false;
   }
 
@@ -557,9 +569,8 @@ void InlineTree::print_inlining(ciMethod* callee_method, int caller_bci,
 //------------------------------ok_to_inline-----------------------------------
 bool InlineTree::ok_to_inline(ciMethod* callee_method, JVMState* jvms, ciCallProfile& profile,
                               bool& should_delay) {
-  assert(callee_method != NULL, "caller checks for optimized virtual!");
-  assert(!should_delay, "should be initialized to false");
 #ifdef ASSERT
+  assert(callee_method != NULL, "caller checks for optimized virtual!");
   // Make sure the incoming jvms has the same information content as me.
   // This means that we can eventually make this whole class AllStatic.
   if (jvms->caller() == NULL) {
@@ -595,7 +606,11 @@ bool InlineTree::ok_to_inline(ciMethod* callee_method, JVMState* jvms, ciCallPro
       set_msg("inline (hot)");
     }
     print_inlining(callee_method, caller_bci, caller_method, true /* success */);
-    build_inline_tree_for_callee(callee_method, jvms, caller_bci);
+    InlineTree* callee_tree = build_inline_tree_for_callee(callee_method, jvms, caller_bci);
+    if (should_delay) {
+      // Record late inlining decision in order to dump it for compiler replay
+      callee_tree->set_late_inline();
+    }
     return true;
   } else {
     // Do not inline
@@ -700,7 +715,7 @@ int InlineTree::count() const {
 }
 
 void InlineTree::dump_replay_data(outputStream* out) {
-  out->print(" %d %d ", inline_level(), caller_bci());
+  out->print(" %d %d %d ", inline_level(), caller_bci(), _late_inline);
   method()->dump_name_as_ascii(out);
   for (int i = 0 ; i < _subtrees.length(); i++) {
     _subtrees.at(i)->dump_replay_data(out);

src/hotspot/share/opto/doCall.cpp

@@ -164,7 +164,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool
     // Try inlining a bytecoded method:
     if (!call_does_dispatch) {
       InlineTree* ilt = InlineTree::find_subtree_from_root(this->ilt(), jvms->caller(), jvms->method());
-      bool should_delay = false;
+      bool should_delay = AlwaysIncrementalInline;
       if (ilt->ok_to_inline(callee, jvms, profile, should_delay)) {
         CallGenerator* cg = CallGenerator::for_inline(callee, expected_uses);
         // For optimized virtual calls assert at runtime that receiver object
@@ -189,7 +189,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool
             return CallGenerator::for_boxing_late_inline(callee, cg);
           } else if (should_delay_vector_reboxing_inlining(callee, jvms)) {
             return CallGenerator::for_vector_reboxing_late_inline(callee, cg);
-          } else if ((should_delay || AlwaysIncrementalInline)) {
+          } else if (should_delay) {
             return CallGenerator::for_late_inline(callee, cg);
           } else {
             return cg;

src/hotspot/share/opto/parse.hpp

@@ -46,6 +46,7 @@ class InlineTree : public ResourceObj {
   Compile*    C;                  // cache
   JVMState*   _caller_jvms;       // state of caller
   ciMethod*   _method;            // method being called by the caller_jvms
+  bool        _late_inline;       // method is inlined incrementally
   InlineTree* _caller_tree;
   uint        _count_inline_bcs;  // Accumulated count of inlined bytecodes
   const int   _max_inline_level;  // the maximum inline level for this sub-tree (may be adjusted)
@@ -75,10 +76,12 @@ class InlineTree : public ResourceObj {
   bool        should_inline(ciMethod* callee_method,
                             ciMethod* caller_method,
                             int caller_bci,
+                            NOT_PRODUCT_ARG(bool& should_delay)
                             ciCallProfile& profile);
   bool        should_not_inline(ciMethod* callee_method,
                                 ciMethod* caller_method,
                                 int caller_bci,
+                                NOT_PRODUCT_ARG(bool& should_delay)
                                 ciCallProfile& profile);
   bool        is_not_reached(ciMethod* callee_method,
                              ciMethod* caller_method,
@@ -112,6 +115,10 @@ class InlineTree : public ResourceObj {
   // The call_method is an optimized virtual method candidate otherwise.
   bool ok_to_inline(ciMethod *call_method, JVMState* caller_jvms, ciCallProfile& profile, bool& should_delay);
 
+  void set_late_inline() {
+    _late_inline = true;
+  }
+
   // Information about inlined method
   JVMState*   caller_jvms()       const { return _caller_jvms; }
   ciMethod   *method()            const { return _method; }

test/hotspot/jtreg/compiler/ciReplay/CiReplayBase.java

@@ -24,10 +24,17 @@
 package compiler.ciReplay;
 
 import compiler.whitebox.CompilerWhiteBoxTest;
-import java.io.IOException;
-import java.io.File;
+import jdk.test.lib.Asserts;
+import jdk.test.lib.Platform;
+import jdk.test.lib.Utils;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+import jdk.test.lib.util.CoreUtils;
+
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileReader;
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -36,14 +43,6 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
-import java.util.regex.Pattern;
-import java.util.regex.Matcher;
-import jdk.test.lib.Platform;
-import jdk.test.lib.process.ProcessTools;
-import jdk.test.lib.process.OutputAnalyzer;
-import jdk.test.lib.Asserts;
-import jdk.test.lib.Utils;
-import jdk.test.lib.util.CoreUtils;
 
 public abstract class CiReplayBase {
     public static final String REPLAY_FILE_NAME = "test_replay.txt";
@@ -296,4 +295,41 @@ private String[] getTestJvmCommandlineWithPrefix(String prefix, String... args)
             throw new Error("Can't create process builder: " + t, t);
         }
     }
+
+    protected void removeVersionFromReplayFile() {
+        setNewVersionLineInReplayFile(null);
+    }
+
+    protected void setNewVersionInReplayFile(int newVersionNumber) {
+        setNewVersionLineInReplayFile("version " + newVersionNumber);
+    }
+
+    private void setNewVersionLineInReplayFile(String firstLineString) {
+        List<String> newLines = new ArrayList<>();
+        Path replayFilePath = Paths.get(getReplayFileName());
+        try (var br = Files.newBufferedReader(replayFilePath)) {
+            String line;
+            boolean firstLine = true;
+            while ((line = br.readLine()) != null) {
+                if (firstLine) {
+                    firstLine = false;
+                    Asserts.assertTrue(line.startsWith("version"), "version number must exist in a proper replay file");
+                    if (firstLineString != null) {
+                        newLines.add(firstLineString);
+                    }
+                    // Else: Remove first line by skipping it.
+                } else {
+                    newLines.add(line);
+                }
+            }
+            Asserts.assertFalse(firstLine, replayFilePath + " should not be empty");
+        } catch (IOException e) {
+            throw new Error("Failed to read replay data: " + e, e);
+        }
+        try {
+            Files.write(replayFilePath, newLines, StandardOpenOption.TRUNCATE_EXISTING);
+        } catch (IOException e) {
+            throw new Error("Failed to write replay data: " + e, e);
+        }
+    }
 }