8354668: Missing REX2 prefix accounting in ZGC barriers leads to incorrect encoding

Reviewed-by: aboldtch, sviswanathan

Author: Jatin Bhateja

src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp

@@ -391,8 +391,8 @@ void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm,
     if (rnew_zaddress != noreg) {
       // noreg means null; no need to color
       __ movptr(rnew_zpointer, rnew_zaddress);
-      __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
       __ shlq(rnew_zpointer, barrier_Relocation::unpatched);
+      __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodAfterShX);
       __ orq_imm32(rnew_zpointer, barrier_Relocation::unpatched);
       __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodAfterOr);
     }
@@ -969,13 +969,13 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
 #define __ ce->masm()->
 
 static void z_uncolor(LIR_Assembler* ce, LIR_Opr ref) {
-  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
   __ shrq(ref->as_register(), barrier_Relocation::unpatched);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodAfterShX);
 }
 
 static void z_color(LIR_Assembler* ce, LIR_Opr ref) {
-  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
   __ shlq(ref->as_register(), barrier_Relocation::unpatched);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodAfterShX);
   __ orq_imm32(ref->as_register(), barrier_Relocation::unpatched);
   __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodAfterOr);
 }
@@ -1278,8 +1278,8 @@ void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm,
 
 static int patch_barrier_relocation_offset(int format) {
   switch (format) {
-  case ZBarrierRelocationFormatLoadGoodBeforeShl:
-    return 3;
+  case ZBarrierRelocationFormatLoadGoodAfterShX:
+    return -1;
 
   case ZBarrierRelocationFormatStoreGoodAfterCmp:
     return -2;
@@ -1300,7 +1300,7 @@ static int patch_barrier_relocation_offset(int format) {
 
 static uint16_t patch_barrier_relocation_value(int format) {
   switch (format) {
-  case ZBarrierRelocationFormatLoadGoodBeforeShl:
+  case ZBarrierRelocationFormatLoadGoodAfterShX:
     return (uint16_t)ZPointerLoadShift;
 
   case ZBarrierRelocationFormatMarkBadAfterTest:
@@ -1327,7 +1327,7 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
   const int offset = patch_barrier_relocation_offset(format);
   const uint16_t value = patch_barrier_relocation_value(format);
   uint8_t* const patch_addr = (uint8_t*)addr + offset;
-  if (format == ZBarrierRelocationFormatLoadGoodBeforeShl) {
+  if (format == ZBarrierRelocationFormatLoadGoodAfterShX) {
     *patch_addr = (uint8_t)value;
   } else {
     *(uint16_t*)patch_addr = value;

src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp

@@ -49,7 +49,7 @@ class ZLoadBarrierStubC2;
 class ZStoreBarrierStubC2;
 #endif // COMPILER2
 
-const int ZBarrierRelocationFormatLoadGoodBeforeShl = 0;
+const int ZBarrierRelocationFormatLoadGoodAfterShX  = 0;
 const int ZBarrierRelocationFormatLoadBadAfterTest  = 1;
 const int ZBarrierRelocationFormatMarkBadAfterTest  = 2;
 const int ZBarrierRelocationFormatStoreGoodAfterCmp = 3;

src/hotspot/cpu/x86/gc/z/z_x86_64.ad

@@ -35,15 +35,15 @@ source %{
 #include "gc/z/zBarrierSetAssembler.hpp"
 
 static void z_color(MacroAssembler* masm, const MachNode* node, Register ref) {
-  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
   __ shlq(ref, barrier_Relocation::unpatched);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodAfterShX);
   __ orq_imm32(ref, barrier_Relocation::unpatched);
   __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodAfterOr);
 }
 
 static void z_uncolor(MacroAssembler* masm, const MachNode* node, Register ref) {
-  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
   __ shrq(ref, barrier_Relocation::unpatched);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodAfterShX);
 }
 
 static void z_keep_alive_load_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register ref) {

src/hotspot/cpu/x86/jvmciCodeInstaller_x86.cpp

@@ -221,7 +221,7 @@ bool CodeInstaller::pd_relocate(address pc, jint mark) {
       return true;
 #if INCLUDE_ZGC
     case Z_BARRIER_RELOCATION_FORMAT_LOAD_GOOD_BEFORE_SHL:
-      _instructions->relocate(pc, barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
+      _instructions->relocate(pc, barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodAfterShX);
       return true;
     case Z_BARRIER_RELOCATION_FORMAT_LOAD_BAD_AFTER_TEST:
       _instructions->relocate(pc, barrier_Relocation::spec(), ZBarrierRelocationFormatLoadBadAfterTest);