don't handle code cache exhaustion fully during inline cache patching

src/hotspot/share/code/codeBlob.cpp

@@ -304,6 +304,16 @@ AdapterBlob* AdapterBlob::create(CodeBuffer* cb) {
   return blob;
 }
 
+void* VtableBlob::operator new(size_t s, unsigned size) throw() {
+  // Handling of allocation failure stops compilation and prints a bunch of
+  // stuff, which requires unlocking the CodeCache_lock, so that the Compile_lock
+  // can be locked, and then re-locking the CodeCache_lock. That is not safe in
+  // this context as we hold the CompiledICLocker. So we just don't handle code
+  // cache exhaustion here; we leave that for a later allocation that does not
+  // hold the CompiledICLocker.
+  return CodeCache::allocate(size, CodeBlobType::NonNMethod, false /* handle_alloc_failure */);
+}
+
 VtableBlob::VtableBlob(const char* name, int size) :
   BufferBlob(name, size) {
 }

src/hotspot/share/code/codeBlob.hpp

@@ -441,6 +441,8 @@ class VtableBlob: public BufferBlob {
 private:
   VtableBlob(const char*, int);
 
+  void* operator new(size_t s, unsigned size) throw();
+
 public:
   // Creation
   static VtableBlob* create(const char* name, int buffer_size);

src/hotspot/share/code/codeCache.cpp

@@ -483,7 +483,7 @@ CodeBlob* CodeCache::next_blob(CodeHeap* heap, CodeBlob* cb) {
  * run the constructor for the CodeBlob subclass he is busy
  * instantiating.
  */
-CodeBlob* CodeCache::allocate(int size, int code_blob_type, int orig_code_blob_type) {
+CodeBlob* CodeCache::allocate(int size, int code_blob_type, bool handle_alloc_failure, int orig_code_blob_type) {
   // Possibly wakes up the sweeper thread.
   NMethodSweeper::report_allocation(code_blob_type);
   assert_locked_or_safepoint(CodeCache_lock);
@@ -531,11 +531,13 @@ CodeBlob* CodeCache::allocate(int size, int code_blob_type, int orig_code_blob_t
             tty->print_cr("Extension of %s failed. Trying to allocate in %s.",
                           heap->name(), get_code_heap(type)->name());
           }
-          return allocate(size, type, orig_code_blob_type);
+          return allocate(size, type, handle_alloc_failure, orig_code_blob_type);
         }
       }
-      MutexUnlocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
-      CompileBroker::handle_full_code_cache(orig_code_blob_type);
+      if (handle_alloc_failure) {
+        MutexUnlocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
+        CompileBroker::handle_full_code_cache(orig_code_blob_type);
+      }
       return NULL;
     }
     if (PrintCodeCacheExtension) {

src/hotspot/share/code/codeCache.hpp

@@ -136,7 +136,7 @@ class CodeCache : AllStatic {
   static const GrowableArray<CodeHeap*>* nmethod_heaps() { return _nmethod_heaps; }
 
   // Allocation/administration
-  static CodeBlob* allocate(int size, int code_blob_type, int orig_code_blob_type = CodeBlobType::All); // allocates a new CodeBlob
+  static CodeBlob* allocate(int size, int code_blob_type, bool handle_alloc_failure = true, int orig_code_blob_type = CodeBlobType::All); // allocates a new CodeBlob
   static void commit(CodeBlob* cb);                        // called when the allocated CodeBlob has been filled
   static int  alignment_unit();                            // guaranteed alignment of all CodeBlobs
   static int  alignment_offset();                          // guaranteed offset of first CodeBlob byte within alignment unit (i.e., allocation header)