8316653: Large NMethodSizeLimit triggers assert during C1 code buffer allocation

Reviewed-by: kvn, rcastanedalo, thartmann

Author: dlunde

src/hotspot/share/c1/c1_Compilation.hpp

@@ -213,10 +213,10 @@ class Compilation: public StackObj {
   bool bailed_out() const                        { return _bailout_msg != nullptr; }
   const char* bailout_msg() const                { return _bailout_msg; }
 
-  static int desired_max_code_buffer_size() {
-    return (int)NMethodSizeLimit;  // default 64K
+  static uint desired_max_code_buffer_size() {
+    return (uint)NMethodSizeLimit;  // default 64K
   }
-  static int desired_max_constant_size() {
+  static uint desired_max_constant_size() {
     return desired_max_code_buffer_size() / 10;
   }
 

src/hotspot/share/c1/c1_Compiler.cpp

@@ -77,7 +77,7 @@ void Compiler::initialize() {
   }
 }
 
-int Compiler::code_buffer_size() {
+uint Compiler::code_buffer_size() {
   return Compilation::desired_max_code_buffer_size() + Compilation::desired_max_constant_size();
 }
 

src/hotspot/share/c1/c1_Compiler.hpp

@@ -60,7 +60,7 @@ class Compiler: public AbstractCompiler {
   static bool is_intrinsic_supported(vmIntrinsics::ID id);
 
   // Size of the code buffer
-  static int code_buffer_size();
+  static uint code_buffer_size();
 };
 
 #endif // SHARE_C1_C1_COMPILER_HPP

src/hotspot/share/code/codeBlob.cpp

@@ -249,7 +249,7 @@ BufferBlob::BufferBlob(const char* name, int size)
 : RuntimeBlob(name, sizeof(BufferBlob), size, CodeOffsets::frame_never_safe, /*locs_size:*/ 0)
 {}
 
-BufferBlob* BufferBlob::create(const char* name, int buffer_size) {
+BufferBlob* BufferBlob::create(const char* name, uint buffer_size) {
   ThreadInVMfromUnknown __tiv;  // get to VM state in case we block on CodeCache_lock
 
   BufferBlob* blob = nullptr;

src/hotspot/share/code/codeBlob.hpp

@@ -410,7 +410,7 @@ class BufferBlob: public RuntimeBlob {
 
  public:
   // Creation
-  static BufferBlob* create(const char* name, int buffer_size);
+  static BufferBlob* create(const char* name, uint buffer_size);
   static BufferBlob* create(const char* name, CodeBuffer* cb);
 
   static void free(BufferBlob* buf);

src/hotspot/share/code/codeCache.cpp

@@ -520,10 +520,10 @@ CodeBlob* CodeCache::next_blob(CodeHeap* heap, CodeBlob* cb) {
  * run the constructor for the CodeBlob subclass he is busy
  * instantiating.
  */
-CodeBlob* CodeCache::allocate(int size, CodeBlobType code_blob_type, bool handle_alloc_failure, CodeBlobType orig_code_blob_type) {
+CodeBlob* CodeCache::allocate(uint size, CodeBlobType code_blob_type, bool handle_alloc_failure, CodeBlobType orig_code_blob_type) {
   assert_locked_or_safepoint(CodeCache_lock);
-  assert(size > 0, "Code cache allocation request must be > 0 but is %d", size);
-  if (size <= 0) {
+  assert(size > 0, "Code cache allocation request must be > 0");
+  if (size == 0) {
     return nullptr;
   }
   CodeBlob* cb = nullptr;
@@ -1575,10 +1575,14 @@ void CodeCache::print_memory_overhead() {
 
 #ifndef PRODUCT
 
-void CodeCache::print_trace(const char* event, CodeBlob* cb, int size) {
+void CodeCache::print_trace(const char* event, CodeBlob* cb, uint size) {
   if (PrintCodeCache2) {  // Need to add a new flag
     ResourceMark rm;
-    if (size == 0)  size = cb->size();
+    if (size == 0) {
+      int s = cb->size();
+      assert(s >= 0, "CodeBlob size is negative: %d", s);
+      size = (uint) s;
+    }
     tty->print_cr("CodeCache %s:  addr: " INTPTR_FORMAT ", size: 0x%x", event, p2i(cb), size);
   }
 }

src/hotspot/share/code/codeCache.hpp

@@ -149,7 +149,7 @@ class CodeCache : AllStatic {
   static const GrowableArray<CodeHeap*>* nmethod_heaps() { return _nmethod_heaps; }
 
   // Allocation/administration
-  static CodeBlob* allocate(int size, CodeBlobType code_blob_type, bool handle_alloc_failure = true, CodeBlobType orig_code_blob_type = CodeBlobType::All); // allocates a new CodeBlob
+  static CodeBlob* allocate(uint size, CodeBlobType code_blob_type, bool handle_alloc_failure = true, CodeBlobType orig_code_blob_type = CodeBlobType::All); // allocates a new CodeBlob
   static void commit(CodeBlob* cb);                        // called when the allocated CodeBlob has been filled
   static void free(CodeBlob* cb);                          // frees a CodeBlob
   static void free_unused_tail(CodeBlob* cb, size_t used); // frees the unused tail of a CodeBlob (only used by TemplateInterpreter::initialize())
@@ -226,7 +226,7 @@ class CodeCache : AllStatic {
   static void print_internals();
   static void print_memory_overhead();
   static void verify();                          // verifies the code cache
-  static void print_trace(const char* event, CodeBlob* cb, int size = 0) PRODUCT_RETURN;
+  static void print_trace(const char* event, CodeBlob* cb, uint size = 0) PRODUCT_RETURN;
   static void print_summary(outputStream* st, bool detailed = true); // Prints a summary of the code cache usage
   static void log_state(outputStream* st);
   LINUX_ONLY(static void write_perf_map();)

test/hotspot/jtreg/compiler/arguments/TestC1Globals.java

@@ -41,6 +41,16 @@
  *                   compiler.arguments.TestC1Globals
  */
 
+/**
+ * @test
+ * @bug 8316653
+ * @requires vm.debug
+ * @summary Test flag with max value
+ *
+ * @run main/othervm -XX:NMethodSizeLimit=2147483647
+ *                   compiler.arguments.TestC1Globals
+ */
+
 /**
  * @test
  * @bug 8318817