8248411: [aarch64] Insufficient error handling when CodeBuffer is exhausted

Reviewed-by: adinn

Author: Patric Hedlin

src/hotspot/cpu/aarch64/aarch64.ad

@@ -3765,23 +3765,28 @@ encode %{
     if (!_method) {
       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
+      if (call == NULL) {
+        ciEnv::current()->record_failure("CodeCache is full");
+        return;
+      }
     } else {
       int method_index = resolved_method_index(cbuf);
       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
                                                   : static_call_Relocation::spec(method_index);
       call = __ trampoline_call(Address(addr, rspec), &cbuf);
-
+      if (call == NULL) {
+        ciEnv::current()->record_failure("CodeCache is full");
+        return;
+      }
       // Emit stub for static call
       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
       if (stub == NULL) {
         ciEnv::current()->record_failure("CodeCache is full");
         return;
       }
     }
-    if (call == NULL) {
-      ciEnv::current()->record_failure("CodeCache is full");
-      return;
-    } else if (UseSVE > 0 && Compile::current()->max_vector_size() >= 16) {
+
+    if (UseSVE > 0 && Compile::current()->max_vector_size() >= 16) {
       // Only non uncommon_trap calls need to reinitialize ptrue.
       if (uncommon_trap_request() == 0) {
         __ reinitialize_ptrue();
@@ -14725,7 +14730,11 @@ instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlag
   format %{ "ClearArray $cnt, $base" %}
 
   ins_encode %{
-    __ zero_words($base$$Register, $cnt$$Register);
+    address tpc = __ zero_words($base$$Register, $cnt$$Register);
+    if (tpc == NULL) {
+      ciEnv::current()->record_failure("CodeCache is full");
+      return;
+    }
   %}
 
   ins_pipe(pipe_class_memory);
@@ -16003,8 +16012,8 @@ instruct CallStaticJavaDirect(method meth)
 
   format %{ "call,static $meth \t// ==> " %}
 
-  ins_encode( aarch64_enc_java_static_call(meth),
-              aarch64_enc_call_epilog );
+  ins_encode(aarch64_enc_java_static_call(meth),
+             aarch64_enc_call_epilog);
 
   ins_pipe(pipe_class_call);
 %}
@@ -16022,8 +16031,8 @@ instruct CallDynamicJavaDirect(method meth)
 
   format %{ "CALL,dynamic $meth \t// ==> " %}
 
-  ins_encode( aarch64_enc_java_dynamic_call(meth),
-               aarch64_enc_call_epilog );
+  ins_encode(aarch64_enc_java_dynamic_call(meth),
+             aarch64_enc_call_epilog);
 
   ins_pipe(pipe_class_call);
 %}
@@ -16489,10 +16498,14 @@ instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
 
   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
   ins_encode %{
-    __ arrays_equals($ary1$$Register, $ary2$$Register,
-                     $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
-                     $result$$Register, $tmp$$Register, 1);
-    %}
+    address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
+                                   $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
+                                   $result$$Register, $tmp$$Register, 1);
+    if (tpc == NULL) {
+      ciEnv::current()->record_failure("CodeCache is full");
+      return;
+    }
+  %}
   ins_pipe(pipe_class_memory);
 %}
 
@@ -16506,9 +16519,13 @@ instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
 
   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
   ins_encode %{
-    __ arrays_equals($ary1$$Register, $ary2$$Register,
-                     $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
-                     $result$$Register, $tmp$$Register, 2);
+    address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
+                                   $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
+                                   $result$$Register, $tmp$$Register, 2);
+    if (tpc == NULL) {
+      ciEnv::current()->record_failure("CodeCache is full");
+      return;
+    }
   %}
   ins_pipe(pipe_class_memory);
 %}
@@ -16519,7 +16536,11 @@ instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg c
   effect(USE_KILL ary1, USE_KILL len, KILL cr);
   format %{ "has negatives byte[] $ary1,$len -> $result" %}
   ins_encode %{
-    __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
+    address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
+    if (tpc == NULL) {
+      ciEnv::current()->record_failure("CodeCache is full");
+      return;
+    }
   %}
   ins_pipe( pipe_slow );
 %}
@@ -16552,8 +16573,13 @@ instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len
 
   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
   ins_encode %{
-    __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
-                          $tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister, $tmp4$$Register);
+    address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
+                                        $tmp1$$FloatRegister, $tmp2$$FloatRegister,
+                                        $tmp3$$FloatRegister, $tmp4$$Register);
+    if (tpc == NULL) {
+      ciEnv::current()->record_failure("CodeCache is full");
+      return;
+    }
   %}
   ins_pipe(pipe_class_memory);
 %}

src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp

@@ -36,6 +36,9 @@
 
 #define __ _masm.
 address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+  precond(cbuf.stubs()->start() != badAddress);
+  precond(cbuf.stubs()->end() != badAddress);
+
   // Stub is fixed up when the corresponding call is converted from
   // calling compiled code to calling interpreted code.
   // mov rmethod, 0

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -705,7 +705,7 @@ void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, in
 // Maybe emit a call via a trampoline.  If the code cache is small
 // trampolines won't be emitted.
 
-address MacroAssembler::trampoline_call(Address entry, CodeBuffer *cbuf) {
+address MacroAssembler::trampoline_call(Address entry, CodeBuffer* cbuf) {
   assert(JavaThread::current()->is_Compiler_thread(), "just checking");
   assert(entry.rspec().type() == relocInfo::runtime_call_type
          || entry.rspec().type() == relocInfo::opt_virtual_call_type
@@ -726,6 +726,7 @@ address MacroAssembler::trampoline_call(Address entry, CodeBuffer *cbuf) {
     if (!in_scratch_emit_size) {
       address stub = emit_trampoline_stub(offset(), entry.target());
       if (stub == NULL) {
+        postcond(pc() == badAddress);
         return NULL; // CodeCache is full
       }
     }
@@ -739,6 +740,7 @@ address MacroAssembler::trampoline_call(Address entry, CodeBuffer *cbuf) {
     bl(pc());
   }
   // just need to return a non-null address
+  postcond(pc() != badAddress);
   return pc();
 }
 
@@ -4490,7 +4492,7 @@ void MacroAssembler::remove_frame(int framesize) {
 
 
 // This method checks if provided byte array contains byte with highest bit set.
-void MacroAssembler::has_negatives(Register ary1, Register len, Register result) {
+address MacroAssembler::has_negatives(Register ary1, Register len, Register result) {
     // Simple and most common case of aligned small array which is not at the
     // end of memory page is placed here. All other cases are in stub.
     Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE;
@@ -4527,27 +4529,38 @@ void MacroAssembler::has_negatives(Register ary1, Register len, Register result)
     b(SET_RESULT);
 
   BIND(STUB);
-    RuntimeAddress has_neg =  RuntimeAddress(StubRoutines::aarch64::has_negatives());
+    RuntimeAddress has_neg = RuntimeAddress(StubRoutines::aarch64::has_negatives());
     assert(has_neg.target() != NULL, "has_negatives stub has not been generated");
-    trampoline_call(has_neg);
+    address tpc1 = trampoline_call(has_neg);
+    if (tpc1 == NULL) {
+      DEBUG_ONLY(reset_labels(STUB_LONG, SET_RESULT, DONE));
+      postcond(pc() == badAddress);
+      return NULL;
+    }
     b(DONE);
 
   BIND(STUB_LONG);
-    RuntimeAddress has_neg_long =  RuntimeAddress(
-            StubRoutines::aarch64::has_negatives_long());
+    RuntimeAddress has_neg_long = RuntimeAddress(StubRoutines::aarch64::has_negatives_long());
     assert(has_neg_long.target() != NULL, "has_negatives stub has not been generated");
-    trampoline_call(has_neg_long);
+    address tpc2 = trampoline_call(has_neg_long);
+    if (tpc2 == NULL) {
+      DEBUG_ONLY(reset_labels(SET_RESULT, DONE));
+      postcond(pc() == badAddress);
+      return NULL;
+    }
     b(DONE);
 
   BIND(SET_RESULT);
     cset(result, NE); // set true or false
 
   BIND(DONE);
+  postcond(pc() != badAddress);
+  return pc();
 }
 
-void MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
-                                   Register tmp4, Register tmp5, Register result,
-                                   Register cnt1, int elem_size) {
+address MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
+                                      Register tmp4, Register tmp5, Register result,
+                                      Register cnt1, int elem_size) {
   Label DONE, SAME;
   Register tmp1 = rscratch1;
   Register tmp2 = rscratch2;
@@ -4651,7 +4664,7 @@ void MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
       }
     }
   } else {
-    Label NEXT_DWORD, SHORT, TAIL, TAIL2, STUB, EARLY_OUT,
+    Label NEXT_DWORD, SHORT, TAIL, TAIL2, STUB,
         CSET_EQ, LAST_CHECK;
     mov(result, false);
     cbz(a1, DONE);
@@ -4710,10 +4723,14 @@ void MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
     cbnz(tmp5, DONE);
     RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_array_equals());
     assert(stub.target() != NULL, "array_equals_long stub has not been generated");
-    trampoline_call(stub);
+    address tpc = trampoline_call(stub);
+    if (tpc == NULL) {
+      DEBUG_ONLY(reset_labels(SHORT, LAST_CHECK, CSET_EQ, SAME, DONE));
+      postcond(pc() == badAddress);
+      return NULL;
+    }
     b(DONE);
 
-    bind(EARLY_OUT);
     // (a1 != null && a2 == null) || (a1 != null && a2 != null && a1 == a2)
     // so, if a2 == null => return false(0), else return true, so we can return a2
     mov(result, a2);
@@ -4740,6 +4757,8 @@ void MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
   bind(DONE);
 
   BLOCK_COMMENT("} array_equals");
+  postcond(pc() != badAddress);
+  return pc();
 }
 
 // Compare Strings
@@ -4847,7 +4866,7 @@ const int MacroAssembler::zero_words_block_size = 8;
 // cnt:   Count in HeapWords.
 //
 // ptr, cnt, rscratch1, and rscratch2 are clobbered.
-void MacroAssembler::zero_words(Register ptr, Register cnt)
+address MacroAssembler::zero_words(Register ptr, Register cnt)
 {
   assert(is_power_of_2(zero_words_block_size), "adjust this");
   assert(ptr == r10 && cnt == r11, "mismatch in register usage");
@@ -4857,10 +4876,15 @@ void MacroAssembler::zero_words(Register ptr, Register cnt)
   Label around;
   br(LO, around);
   {
-    RuntimeAddress zero_blocks =  RuntimeAddress(StubRoutines::aarch64::zero_blocks());
+    RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::aarch64::zero_blocks());
     assert(zero_blocks.target() != NULL, "zero_blocks stub has not been generated");
     if (StubRoutines::aarch64::complete()) {
-      trampoline_call(zero_blocks);
+      address tpc = trampoline_call(zero_blocks);
+      if (tpc == NULL) {
+        DEBUG_ONLY(reset_labels(around));
+        postcond(pc() == badAddress);
+        return NULL;
+      }
     } else {
       bl(zero_blocks);
     }
@@ -4881,6 +4905,8 @@ void MacroAssembler::zero_words(Register ptr, Register cnt)
     bind(l);
   }
   BLOCK_COMMENT("} zero_words");
+  postcond(pc() != badAddress);
+  return pc();
 }
 
 // base:         Address of a buffer to be zeroed, 8 bytes aligned.
@@ -4893,14 +4919,15 @@ void MacroAssembler::zero_words(Register base, uint64_t cnt)
   if (i) str(zr, Address(base));
 
   if (cnt <= SmallArraySize / BytesPerLong) {
-    for (; i < (int)cnt; i += 2)
+    for (; i < (int)cnt; i += 2) {
       stp(zr, zr, Address(base, i * wordSize));
+    }
   } else {
     const int unroll = 4; // Number of stp(zr, zr) instructions we'll unroll
     int remainder = cnt % (2 * unroll);
-    for (; i < remainder; i += 2)
+    for (; i < remainder; i += 2) {
       stp(zr, zr, Address(base, i * wordSize));
-
+    }
     Label loop;
     Register cnt_reg = rscratch1;
     Register loop_base = rscratch2;
@@ -4910,8 +4937,9 @@ void MacroAssembler::zero_words(Register base, uint64_t cnt)
     add(loop_base, base, (remainder - 2) * wordSize);
     bind(loop);
     sub(cnt_reg, cnt_reg, 2 * unroll);
-    for (i = 1; i < unroll; i++)
+    for (i = 1; i < unroll; i++) {
       stp(zr, zr, Address(loop_base, 2 * i * wordSize));
+    }
     stp(zr, zr, Address(pre(loop_base, 2 * unroll * wordSize)));
     cbnz(cnt_reg, loop);
   }
@@ -5127,9 +5155,9 @@ void MacroAssembler::encode_iso_array(Register src, Register dst,
 
 
 // Inflate byte[] array to char[].
-void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
-                                        FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3,
-                                        Register tmp4) {
+address MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
+                                           FloatRegister vtmp1, FloatRegister vtmp2,
+                                           FloatRegister vtmp3, Register tmp4) {
   Label big, done, after_init, to_stub;
 
   assert_different_registers(src, dst, len, tmp4, rscratch1);
@@ -5166,9 +5194,14 @@ void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len
 
   if (SoftwarePrefetchHintDistance >= 0) {
     bind(to_stub);
-      RuntimeAddress stub =  RuntimeAddress(StubRoutines::aarch64::large_byte_array_inflate());
+      RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_byte_array_inflate());
       assert(stub.target() != NULL, "large_byte_array_inflate stub has not been generated");
-      trampoline_call(stub);
+      address tpc = trampoline_call(stub);
+      if (tpc == NULL) {
+        DEBUG_ONLY(reset_labels(big, done));
+        postcond(pc() == badAddress);
+        return NULL;
+      }
       b(after_init);
   }
 
@@ -5222,6 +5255,8 @@ void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len
   strq(vtmp3, Address(dst, -16));
 
   bind(done);
+  postcond(pc() != badAddress);
+  return pc();
 }
 
 // Compress char[] array to byte[].