8277204: Implement PAC-RET branch protection on Linux/AArch64

make/autoconf/flags-cflags.m4

@@ -805,17 +805,19 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_CPU_DEP],
   fi
   AC_SUBST(FILE_MACRO_CFLAGS)
 
+  FLAGS_SETUP_BRANCH_PROTECTION
+
   # EXPORT to API
   CFLAGS_JVM_COMMON="$ALWAYS_CFLAGS_JVM $ALWAYS_DEFINES_JVM \
       $TOOLCHAIN_CFLAGS_JVM ${$1_TOOLCHAIN_CFLAGS_JVM} \
       $OS_CFLAGS $OS_CFLAGS_JVM $CFLAGS_OS_DEF_JVM $DEBUG_CFLAGS_JVM \
       $WARNING_CFLAGS $WARNING_CFLAGS_JVM $JVM_PICFLAG $FILE_MACRO_CFLAGS \
-      $REPRODUCIBLE_CFLAGS"
+      $REPRODUCIBLE_CFLAGS $BRANCH_PROTECTION_CFLAGS"
 
   CFLAGS_JDK_COMMON="$ALWAYS_CFLAGS_JDK $ALWAYS_DEFINES_JDK $TOOLCHAIN_CFLAGS_JDK \
       $OS_CFLAGS $CFLAGS_OS_DEF_JDK $DEBUG_CFLAGS_JDK $DEBUG_OPTIONS_FLAGS_JDK \
       $WARNING_CFLAGS $WARNING_CFLAGS_JDK $DEBUG_SYMBOLS_CFLAGS_JDK \
-      $FILE_MACRO_CFLAGS $REPRODUCIBLE_CFLAGS"
+      $FILE_MACRO_CFLAGS $REPRODUCIBLE_CFLAGS $BRANCH_PROTECTION_CFLAGS"
 
   # Use ${$2EXTRA_CFLAGS} to block EXTRA_CFLAGS to be added to build flags.
   # (Currently we don't have any OPENJDK_BUILD_EXTRA_CFLAGS, but that might
@@ -881,3 +883,24 @@ AC_DEFUN([FLAGS_SETUP_GCC6_COMPILER_FLAGS],
       PREFIX: $2, IF_FALSE: [NO_LIFETIME_DSE_CFLAG=""])
   $1_GCC6_CFLAGS="${NO_DELETE_NULL_POINTER_CHECKS_CFLAG} ${NO_LIFETIME_DSE_CFLAG}"
 ])
+
+AC_DEFUN_ONCE([FLAGS_SETUP_BRANCH_PROTECTION],
+[
+  # Is branch protection available?
+  BRANCH_PROTECTION_AVAILABLE=false
+  BRANCH_PROTECTION_FLAG="-mbranch-protection=standard"
+
+  if test "x$OPENJDK_TARGET_CPU" = xaarch64; then
+    if test "x$TOOLCHAIN_TYPE" = xgcc || test "x$TOOLCHAIN_TYPE" = xclang; then
+      FLAGS_COMPILER_CHECK_ARGUMENTS(ARGUMENT: [${BRANCH_PROTECTION_FLAG}],
+          IF_TRUE: [BRANCH_PROTECTION_AVAILABLE=true])
+    fi
+  fi
+
+  BRANCH_PROTECTION_CFLAGS=""
+  UTIL_ARG_ENABLE(NAME: branch-protection, DEFAULT: auto,
+      RESULT: USE_BRANCH_PROTECTION, AVAILABLE: $BRANCH_PROTECTION_AVAILABLE,
+      DESC: [enable branch protection when compiling C/C++],
+      IF_ENABLED: [ BRANCH_PROTECTION_CFLAGS=${BRANCH_PROTECTION_FLAG}])
+  AC_SUBST(BRANCH_PROTECTION_CFLAGS)
+])

make/autoconf/spec.gmk.in

@@ -403,6 +403,7 @@ LIBFFI_CFLAGS:=@LIBFFI_CFLAGS@
 ENABLE_LIBFFI_BUNDLING:=@ENABLE_LIBFFI_BUNDLING@
 LIBFFI_LIB_FILE:=@LIBFFI_LIB_FILE@
 FILE_MACRO_CFLAGS := @FILE_MACRO_CFLAGS@
+BRANCH_PROTECTION_CFLAGS := @BRANCH_PROTECTION_CFLAGS@
 
 STATIC_LIBS_CFLAGS := @STATIC_LIBS_CFLAGS@
 

src/hotspot/cpu/aarch64/aarch64.ad

@@ -1366,8 +1366,8 @@ source %{
       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
     }
 
-    // r29 is not allocatable when PreserveFramePointer is on
-    if (PreserveFramePointer) {
+    // r29 is not allocatable when PreserveFramePointer or UseROPProtection is on
+    if (PreserveFramePointer || UseROPProtection) {
       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);

src/hotspot/cpu/aarch64/assembler_aarch64.hpp

@@ -990,33 +990,36 @@ class Assembler : public AbstractAssembler {
     rf(rt, 0);
   }
 
-  void hint(int imm) {
-    system(0b00, 0b011, 0b0010, 0b0000, imm);
-  }
-
-  void nop() {
-    hint(0);
-  }
-
-  void yield() {
-    hint(1);
-  }
+  // Hint instructions
 
-  void wfe() {
-    hint(2);
+#define INSN(NAME, crm, op2)               \
+  void NAME() {                            \
+    system(0b00, 0b011, 0b0010, crm, op2); \
   }
 
-  void wfi() {
-    hint(3);
-  }
+  INSN(nop,   0b000, 0b0000);
+  INSN(yield, 0b000, 0b0001);
+  INSN(wfe,   0b000, 0b0010);
+  INSN(wfi,   0b000, 0b0011);
+  INSN(sev,   0b000, 0b0100);
+  INSN(sevl,  0b000, 0b0101);
 
-  void sev() {
-    hint(4);
-  }
+  // PAC instructions
+  INSN(autia1716, 0b0001, 0b100);
+  INSN(autiasp,   0b0011, 0b101);
+  INSN(autiaz,    0b0011, 0b100);
+  INSN(autib1716, 0b0001, 0b110);
+  INSN(autibsp,   0b0011, 0b111);
+  INSN(autibz,    0b0011, 0b110);
+  INSN(pacia1716, 0b0001, 0b000);
+  INSN(paciasp,   0b0011, 0b001);
+  INSN(paciaz,    0b0011, 0b000);
+  INSN(pacib1716, 0b0001, 0b010);
+  INSN(pacibsp,   0b0011, 0b011);
+  INSN(pacibz,    0b0011, 0b010);
+  INSN(xpaclri,   0b0000, 0b111);
 
-  void sevl() {
-    hint(5);
-  }
+#undef INSN
 
   // we only provide mrs and msr for the special purpose system
   // registers where op1 (instr[20:19]) == 11 and, (currently) only
@@ -1102,18 +1105,20 @@ class Assembler : public AbstractAssembler {
   }
 
   // Unconditional branch (register)
-  void branch_reg(Register R, int opc) {
+  void branch_reg(int OP, int A, int M, Register RN, Register RM) {
     starti;
     f(0b1101011, 31, 25);
-    f(opc, 24, 21);
-    f(0b11111000000, 20, 10);
-    rf(R, 5);
-    f(0b00000, 4, 0);
+    f(OP, 24, 21);
+    f(0b111110000, 20, 12);
+    f(A, 11, 11);
+    f(M, 10, 10);
+    rf(RN, 5);
+    rf(RM, 0);
   }
 
-#define INSN(NAME, opc)                         \
-  void NAME(Register R) {                       \
-    branch_reg(R, opc);                         \
+#define INSN(NAME, opc)              \
+  void NAME(Register RN) {           \
+    branch_reg(opc, 0, 0, RN, 0);    \
   }
 
   INSN(br, 0b0000);
@@ -1124,16 +1129,54 @@ class Assembler : public AbstractAssembler {
 
 #undef INSN
 
-#define INSN(NAME, opc)                         \
-  void NAME() {                 \
-    branch_reg(dummy_reg, opc);         \
+#define INSN(NAME, opc)                     \
+  void NAME() {                             \
+    branch_reg(opc, 0, 0, dummy_reg, 0);    \
   }
 
   INSN(eret, 0b0100);
   INSN(drps, 0b0101);
 
 #undef INSN
 
+  // PAC return instructions
+#define INSN(NAME, M)                                  \
+  void NAME() {                                        \
+    branch_reg(0b0010, 1, M, dummy_reg, dummy_reg);    \
+  }
+
+  INSN(retaa, 0);
+  INSN(retab, 1);
+
+#undef INSN
+
+  // PAC branch instructions (with zero modifier)
+#define INSN(NAME, OP, M)                   \
+  void NAME(Register rn) {                  \
+    branch_reg(OP, 1, M, rn, dummy_reg);    \
+  }
+
+  INSN(braaz,  0b0000, 0);
+  INSN(brabz,  0b0000, 1);
+  INSN(blraaz, 0b0001, 0);
+  INSN(blrabz, 0b0001, 1);
+
+#undef INSN
+
+  // PAC branch instructions (with register modifier)
+#define INSN(NAME, OP, M)                  \
+  void NAME(Register rn, Register rm) {    \
+    branch_reg(OP, 1, M, rn, rm);          \
+  }
+
+  INSN(braa,  0b1000, 0);
+  INSN(brab,  0b1000, 1);
+  INSN(blraa, 0b1001, 0);
+  INSN(blrab, 0b1001, 1);
+
+#undef INSN
+
+
   // Load/store exclusive
   enum operand_size { byte, halfword, word, xword };
 
@@ -1793,6 +1836,37 @@ void mvnw(Register Rd, Register Rm,
   INSN(clz,    0b110, 0b00000, 0b00100);
   INSN(cls,    0b110, 0b00000, 0b00101);
 
+  // PAC instructions
+  INSN(pacia,  0b110, 0b00001, 0b00000);
+  INSN(pacib,  0b110, 0b00001, 0b00001);
+  INSN(pacda,  0b110, 0b00001, 0b00010);
+  INSN(pacdb,  0b110, 0b00001, 0b00011);
+  INSN(autia,  0b110, 0b00001, 0b00100);
+  INSN(autib,  0b110, 0b00001, 0b00101);
+  INSN(autda,  0b110, 0b00001, 0b00110);
+  INSN(autdb,  0b110, 0b00001, 0b00111);
+
+#undef INSN
+
+#define INSN(NAME, op29, opcode2, opcode)                       \
+  void NAME(Register Rd) {                                      \
+    starti;                                                     \
+    f(opcode2, 20, 16);                                         \
+    data_processing(current_insn, op29, opcode, Rd, dummy_reg); \
+  }
+
+  // PAC instructions (with zero modifier)
+  INSN(paciza,  0b110, 0b00001, 0b01000);
+  INSN(pacizb,  0b110, 0b00001, 0b01001);
+  INSN(pacdza,  0b110, 0b00001, 0b01010);
+  INSN(pacdzb,  0b110, 0b00001, 0b01011);
+  INSN(autiza,  0b110, 0b00001, 0b01100);
+  INSN(autizb,  0b110, 0b00001, 0b01101);
+  INSN(autdza,  0b110, 0b00001, 0b01110);
+  INSN(autdzb,  0b110, 0b00001, 0b01111);
+  INSN(xpaci,   0b110, 0b00001, 0b10000);
+  INSN(xpacd,   0b110, 0b00001, 0b10001);
+
 #undef INSN
 
   // (2 sources)

src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp

@@ -385,6 +385,7 @@ OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
 
     // load issuing PC (the return address for this stub) into r3
     __ ldr(exception_pc, Address(rfp, 1*BytesPerWord));
+    __ authenticate_return_address(exception_pc, rscratch1);
 
     // make sure that the vm_results are cleared (may be unnecessary)
     __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
@@ -433,6 +434,7 @@ OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
   __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));
 
   // patch throwing pc into return address (has bci & oop map)
+  __ protect_return_address(exception_pc, rscratch1);
   __ str(exception_pc, Address(rfp, 1*BytesPerWord));
 
   // compute the exception handler.
@@ -443,13 +445,12 @@ OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
   // r0: handler address
   //      will be the deopt blob if nmethod was deoptimized while we looked up
   //      handler regardless of whether handler existed in the nmethod.
+  //      Move it into LR ready for returning.
+  __ mov(lr, r0);
 
   // only r0 is valid at this time, all other registers have been destroyed by the runtime call
   __ invalidate_registers(false, true, true, true, true, true);
 
-  // patch the return address, this stub will directly return to the exception handler
-  __ str(r0, Address(rfp, 1*BytesPerWord));
-
   switch (id) {
   case forward_exception_id:
   case handle_exception_nofpu_id:
@@ -462,6 +463,11 @@ OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
   default:  ShouldNotReachHere();
   }
 
+  // Restore the frame (without overwriting lr), and return.
+  __ mov(sp, rfp);
+  __ ldr(rfp, Address(__ post(sp, 2 * wordSize)));
+  __ ret(lr);
+
   return oop_maps;
 }
 
@@ -496,10 +502,12 @@ void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
   // Save our return address because
   // exception_handler_for_return_address will destroy it.  We also
   // save exception_oop
+  __ mov(r3, lr);
+  __ protect_return_address();
   __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize)));
 
   // search the exception handler address of the caller (using the return address)
-  __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, lr);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, r3);
   // r0: exception handler address of the caller
 
   // Only R0 is valid at this time; all other registers have been
@@ -512,6 +520,7 @@ void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
   // get throwing pc (= return address).
   // lr has been destroyed by the call
   __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize)));
+  __ authenticate_return_address();
   __ mov(r3, lr);
 
   __ verify_not_null_oop(exception_oop);
@@ -617,8 +626,6 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
     case forward_exception_id:
       {
         oop_maps = generate_handle_exception(id, sasm);
-        __ leave();
-        __ ret(lr);
       }
       break;
 
@@ -1043,13 +1050,13 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
 
     case handle_exception_nofpu_id:
     case handle_exception_id:
-      { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
+      { StubFrame f(sasm, "handle_exception", dont_gc_arguments, does_not_return);
         oop_maps = generate_handle_exception(id, sasm);
       }
       break;
 
     case handle_exception_from_callee_id:
-      { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
+      { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments, does_not_return);
         oop_maps = generate_handle_exception(id, sasm);
       }
       break;