8326615: C1/C2 don't handle allocation failure properly during initia…

…lization (RuntimeStub::new_runtime_stub fatal crash)

Reviewed-by: thartmann, kvn

src/hotspot/share/code/codeCache.cpp

@@ -205,7 +205,7 @@ void CodeCache::initialize_heaps() {
   const bool cache_size_set   = FLAG_IS_CMDLINE(ReservedCodeCacheSize);
   const size_t ps             = page_size(false, 8);
   const size_t min_size       = MAX2(os::vm_allocation_granularity(), ps);
-  const size_t min_cache_size = CodeCacheMinimumUseSpace DEBUG_ONLY(* 3); // Make sure we have enough space for VM internal code
+  const size_t min_cache_size = CompilerConfig::min_code_cache_size(); // Make sure we have enough space for VM internal code
   size_t cache_size           = align_up(ReservedCodeCacheSize, min_size);
 
   // Prerequisites

src/hotspot/share/compiler/compilationPolicy.cpp

@@ -455,7 +455,7 @@ void CompilationPolicy::initialize() {
       c2_size = C2Compiler::initial_code_buffer_size();
 #endif
       size_t buffer_size = c1_only ? c1_size : (c1_size/3 + 2*c2_size/3);
-      int max_count = (ReservedCodeCacheSize - (CodeCacheMinimumUseSpace DEBUG_ONLY(* 3))) / (int)buffer_size;
+      int max_count = (ReservedCodeCacheSize - (int)CompilerConfig::min_code_cache_size()) / (int)buffer_size;
       if (count > max_count) {
         // Lower the compiler count such that all buffers fit into the code cache
         count = MAX2(max_count, c1_only ? 1 : 2);

src/hotspot/share/compiler/compilerDefinitions.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -475,8 +475,7 @@ void CompilerConfig::set_jvmci_specific_flags() {
 
 bool CompilerConfig::check_args_consistency(bool status) {
   // Check lower bounds of the code cache
-  // Template Interpreter code is approximately 3X larger in debug builds.
-  uint min_code_cache_size = CodeCacheMinimumUseSpace DEBUG_ONLY(* 3);
+  size_t min_code_cache_size = CompilerConfig::min_code_cache_size();
   if (ReservedCodeCacheSize < InitialCodeCacheSize) {
     jio_fprintf(defaultStream::error_stream(),
                 "Invalid ReservedCodeCacheSize: %dK. Must be at least InitialCodeCacheSize=%dK.\n",

src/hotspot/share/compiler/compilerDefinitions.hpp

@@ -148,6 +148,8 @@ class CompilerConfig : public AllStatic {
   inline static bool is_c2_or_jvmci_compiler_only();
   inline static bool is_c2_or_jvmci_compiler_enabled();
 
+  inline static size_t min_code_cache_size();
+
 private:
   static bool is_compilation_mode_selected();
   static void set_compilation_policy_flags();

src/hotspot/share/compiler/compilerDefinitions.inline.hpp

@@ -25,6 +25,12 @@
 #ifndef SHARE_COMPILER_COMPILERDEFINITIONS_INLINE_HPP
 #define SHARE_COMPILER_COMPILERDEFINITIONS_INLINE_HPP
 
+#ifdef COMPILER1
+#include "c1/c1_Compiler.hpp"
+#endif
+#ifdef COMPILER2
+#include "opto/c2compiler.hpp"
+#endif
 #include "compiler/compilerDefinitions.hpp"
 
 #include "compiler/compiler_globals.hpp"
@@ -132,4 +138,13 @@ inline bool CompilerConfig::is_c2_or_jvmci_compiler_enabled() {
   return is_c2_enabled() || is_jvmci_compiler_enabled();
 }
 
+inline size_t CompilerConfig::min_code_cache_size() {
+  size_t min_code_cache_size = CodeCacheMinimumUseSpace;
+  // Template Interpreter code is approximately 3X larger in debug builds.
+  DEBUG_ONLY(min_code_cache_size *= 3);
+  COMPILER1_PRESENT(min_code_cache_size += Compiler::code_buffer_size());
+  COMPILER2_PRESENT(min_code_cache_size += C2Compiler::initial_code_buffer_size());
+  return min_code_cache_size;
+}
+
 #endif // SHARE_COMPILER_COMPILERDEFINITIONS_INLINE_HPP

test/hotspot/jtreg/ProblemList.txt

@@ -73,8 +73,6 @@ compiler/jvmci/jdk.vm.ci.code.test/src/jdk/vm/ci/code/test/VirtualObjectDebugInf
 compiler/floatingpoint/TestSubnormalFloat.java 8317810 generic-i586
 compiler/floatingpoint/TestSubnormalDouble.java 8317810 generic-i586
 
-compiler/startup/StartupOutput.java 8326615 generic-x64
-
 compiler/codecache/CodeCacheFullCountTest.java 8332954 generic-all
 
 compiler/interpreter/Test6833129.java 8335266 generic-i586

test/hotspot/jtreg/compiler/codecache/CheckSegmentedCodeCache.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -187,8 +187,17 @@ public static void main(String[] args) throws Exception {
 
         // Fails if not enough space for VM internal code
         long minUseSpace = WHITE_BOX.getUintxVMFlag("CodeCacheMinimumUseSpace");
-        // minimum size: CodeCacheMinimumUseSpace DEBUG_ONLY(* 3)
-        long minSize = (Platform.isDebugBuild() ? 3 : 1) * minUseSpace;
+        long nMethodSizeLimit = WHITE_BOX.getIntxVMFlag("NMethodSizeLimit");
+        long codeEntryAlignment = WHITE_BOX.getIntxVMFlag("CodeEntryAlignment");
+        long c1MinCodeCacheSize = 11 * nMethodSizeLimit / 10;
+        long c2MinCodeCacheSize = 2048 /* PhaseOutput::MAX_inst_size */ +
+                                  128 /* PhaseOutput::MAX_stubs_size */ +
+                                  4 * 1024 /* initial_const_capacity */ +
+                                  2 * Math.max(64, codeEntryAlignment) /* 2 * CodeSection::end_slop() */ +
+                                  2 * 128 /* sizeof(relocInfo) * PhaseOutput::MAX_locs_size */;
+        // minimum size: CompilerConfig::min_code_cache_size =
+        // CodeCacheMinimumUseSpace DEBUG_ONLY(* 3) + Compiler::code_buffer_size() + C2Compiler::initial_code_buffer_size())
+        long minSize = minUseSpace * (Platform.isDebugBuild() ? 3 : 1) + c1MinCodeCacheSize + c2MinCodeCacheSize;
         pb = ProcessTools.createLimitedTestJavaProcessBuilder("-XX:+SegmentedCodeCache",
                                                               "-XX:NonNMethodCodeHeapSize=" + minSize,
                                                               "-XX:ReservedCodeCacheSize=" + minSize,