8258384: AArch64: SVE verify_ptrue fails on some tests

Reviewed-by: adinn, ngasson

src/hotspot/cpu/aarch64/aarch64.ad

@@ -1,6 +1,6 @@
 //
-// Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2014, 2020, Red Hat, Inc. All rights reserved.
+// Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -1911,7 +1911,7 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
     __ bind(L_skip_barrier);
   }
 
-  if (UseSVE > 0 && C->max_vector_size() >= 16) {
+  if (C->max_vector_size() >= 16) {
     __ reinitialize_ptrue();
   }
 
@@ -3793,11 +3793,9 @@ encode %{
       }
     }
 
-    if (UseSVE > 0 && Compile::current()->max_vector_size() >= 16) {
-      // Only non uncommon_trap calls need to reinitialize ptrue.
-      if (uncommon_trap_request() == 0) {
-        __ reinitialize_ptrue();
-      }
+    // Only non uncommon_trap calls need to reinitialize ptrue.
+    if (Compile::current()->max_vector_size() >= 16 && uncommon_trap_request() == 0) {
+      __ reinitialize_ptrue();
     }
   %}
 
@@ -3808,7 +3806,7 @@ encode %{
     if (call == NULL) {
       ciEnv::current()->record_failure("CodeCache is full");
       return;
-    } else if (UseSVE > 0 && Compile::current()->max_vector_size() >= 16) {
+    } else if (Compile::current()->max_vector_size() >= 16) {
       __ reinitialize_ptrue();
     }
   %}
@@ -3846,7 +3844,7 @@ encode %{
       __ bind(retaddr);
       __ add(sp, sp, 2 * wordSize);
     }
-    if (UseSVE > 0 && Compile::current()->max_vector_size() >= 16) {
+    if (Compile::current()->max_vector_size() >= 16) {
       __ reinitialize_ptrue();
     }
   %}
@@ -3859,7 +3857,7 @@ encode %{
   enc_class aarch64_enc_ret() %{
     C2_MacroAssembler _masm(&cbuf);
 #ifdef ASSERT
-    if (UseSVE > 0 && Compile::current()->max_vector_size() >= 16) {
+    if (Compile::current()->max_vector_size() >= 16) {
       __ verify_ptrue();
     }
 #endif

src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -354,6 +354,10 @@ class ZSaveLiveRegisters {
   ~ZSaveLiveRegisters() {
     // Restore registers
     __ pop_fp(_fp_regs, sp);
+
+    // External runtime call may clobber ptrue reg
+    __ reinitialize_ptrue();
+
     __ pop(_gp_regs, sp);
   }
 };
@@ -428,11 +432,6 @@ void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, Z
     ZSetupArguments setup_arguments(masm, stub);
     __ mov(rscratch1, stub->slow_path());
     __ blr(rscratch1);
-    if (UseSVE > 0) {
-      // Reinitialize the ptrue predicate register, in case the external runtime
-      // call clobbers ptrue reg, as we may return to SVE compiled code.
-      __ reinitialize_ptrue();
-    }
   }
   // Stub exit
   __ b(*stub->continuation());

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
+ * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2659,6 +2659,8 @@ void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) {
           as_FloatRegister(i+3), T1D, Address(post(sp, 4 * wordSize)));
   }
 
+  reinitialize_ptrue();
+
   pop(call_clobbered_registers() - exclude, sp);
 }
 
@@ -2695,6 +2697,11 @@ void MacroAssembler::pop_CPU_state(bool restore_vectors, bool use_sve,
       ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
           as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
   }
+
+  if (restore_vectors) {
+    reinitialize_ptrue();
+  }
+
   pop(0x3fffffff, sp);         // integer registers except lr & sp
 }
 
@@ -5304,7 +5311,9 @@ void MacroAssembler::verify_sve_vector_length() {
 
 void MacroAssembler::verify_ptrue() {
   Label verify_ok;
-  assert(UseSVE > 0, "should only be used for SVE");
+  if (!UseSVE) {
+    return;
+  }
   sve_cntp(rscratch1, B, ptrue, ptrue); // get true elements count.
   sve_dec(rscratch1, B);
   cbz(rscratch1, verify_ok);

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
+ * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -967,7 +967,9 @@ class MacroAssembler: public Assembler {
 
   void verify_sve_vector_length();
   void reinitialize_ptrue() {
-    sve_ptrue(ptrue, B);
+    if (UseSVE > 0) {
+      sve_ptrue(ptrue, B);
+    }
   }
   void verify_ptrue();
 

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
+ * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2791,12 +2791,6 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t
 
   __ membar(Assembler::LoadLoad | Assembler::LoadStore);
 
-  if (UseSVE > 0 && save_vectors) {
-    // Reinitialize the ptrue predicate register, in case the external runtime
-    // call clobbers ptrue reg, as we may return to SVE compiled code.
-    __ reinitialize_ptrue();
-  }
-
   __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
   __ cbz(rscratch1, noException);
 
@@ -3019,6 +3013,9 @@ void OptoRuntime::generate_exception_blob() {
   // handle_exception_C is a special VM call which does not require an explicit
   // instruction sync afterwards.
 
+  // May jump to SVE compiled code
+  __ reinitialize_ptrue();
+
   // Set an oopmap for the call site.  This oopmap will only be used if we
   // are unwinding the stack.  Hence, all locations will be dead.
   // Callee-saved registers will be the same as the frame above (i.e.,

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
+ * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -488,11 +488,10 @@ class StubGenerator: public StubCodeGenerator {
     __ call_VM_leaf(CAST_FROM_FN_PTR(address,
                          SharedRuntime::exception_handler_for_return_address),
                     rthread, c_rarg1);
-    if (UseSVE > 0 ) {
-      // Reinitialize the ptrue predicate register, in case the external runtime
-      // call clobbers ptrue reg, as we may return to SVE compiled code.
-      __ reinitialize_ptrue();
-    }
+    // Reinitialize the ptrue predicate register, in case the external runtime
+    // call clobbers ptrue reg, as we may return to SVE compiled code.
+    __ reinitialize_ptrue();
+
     // we should not really care that lr is no longer the callee
     // address. we saved the value the handler needs in r19 so we can
     // just copy it to r3. however, the C2 handler will push its own
@@ -5653,11 +5652,9 @@ class StubGenerator: public StubCodeGenerator {
 
     __ reset_last_Java_frame(true);
 
-    if (UseSVE > 0) {
-      // Reinitialize the ptrue predicate register, in case the external runtime
-      // call clobbers ptrue reg, as we may return to SVE compiled code.
-      __ reinitialize_ptrue();
-    }
+    // Reinitialize the ptrue predicate register, in case the external runtime
+    // call clobbers ptrue reg, as we may return to SVE compiled code.
+    __ reinitialize_ptrue();
 
     __ leave();