8224974: Implement JEP 352

Non-Volatile Mapped Byte Buffers

Reviewed-by: alanb, kvn, bpb, gromero, darcy, shade, bulasevich, dchuyko

Author: adinn

make/common/Modules.gmk

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -64,6 +64,7 @@ BOOT_MODULES += \
     jdk.management.jfr \
     jdk.management.agent \
     jdk.net \
+    jdk.nio.mapmode \
     jdk.sctp \
     jdk.unsupported \
     #

src/hotspot/cpu/aarch64/aarch64.ad

@@ -2185,17 +2185,21 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 //=============================================================================
 
 const bool Matcher::match_rule_supported(int opcode) {
+  if (!has_match_rule(opcode))
+    return false;
 
+  bool ret_value = true;
   switch (opcode) {
-  default:
-    break;
-  }
-
-  if (!has_match_rule(opcode)) {
-    return false;
+    case Op_CacheWB:
+    case Op_CacheWBPreSync:
+    case Op_CacheWBPostSync:
+      if (!VM_Version::supports_data_cache_line_flush()) {
+        ret_value = false;
+      }
+      break;
   }
 
-  return true;  // Per default match rules are supported.
+  return ret_value; // Per default match rules are supported.
 }
 
 const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
@@ -7769,6 +7773,47 @@ instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 
 //  ---------------- end of volatile loads and stores ----------------
 
+instruct cacheWB(indirect addr)
+%{
+  predicate(VM_Version::supports_data_cache_line_flush());
+  match(CacheWB addr);
+
+  ins_cost(100);
+  format %{"cache wb $addr" %}
+  ins_encode %{
+    assert($addr->index_position() < 0, "should be");
+    assert($addr$$disp == 0, "should be");
+    __ cache_wb(Address($addr$$base$$Register, 0));
+  %}
+  ins_pipe(pipe_slow); // XXX
+%}
+
+instruct cacheWBPreSync()
+%{
+  predicate(VM_Version::supports_data_cache_line_flush());
+  match(CacheWBPreSync);
+
+  ins_cost(100);
+  format %{"cache wb presync" %}
+  ins_encode %{
+    __ cache_wbsync(true);
+  %}
+  ins_pipe(pipe_slow); // XXX
+%}
+
+instruct cacheWBPostSync()
+%{
+  predicate(VM_Version::supports_data_cache_line_flush());
+  match(CacheWBPostSync);
+
+  ins_cost(100);
+  format %{"cache wb postsync" %}
+  ins_encode %{
+    __ cache_wbsync(false);
+  %}
+  ins_pipe(pipe_slow); // XXX
+%}
+
 // ============================================================================
 // BSWAP Instructions
 

src/hotspot/cpu/aarch64/assembler_aarch64.hpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
+ * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1058,12 +1058,13 @@ class Assembler : public AbstractAssembler {
   //            op1    CRn    CRm    op2
   // IC IVAU     3      7      5      1
   // DC CVAC     3      7      10     1
+  // DC CVAP     3      7      12     1
   // DC CVAU     3      7      11     1
   // DC CIVAC    3      7      14     1
   // DC ZVA      3      7      4      1
   // So only deal with the CRm field.
   enum icache_maintenance {IVAU = 0b0101};
-  enum dcache_maintenance {CVAC = 0b1010, CVAU = 0b1011, CIVAC = 0b1110, ZVA = 0b100};
+  enum dcache_maintenance {CVAC = 0b1010, CVAP = 0b1100, CVAU = 0b1011, CIVAC = 0b1110, ZVA = 0b100};
 
   void dc(dcache_maintenance cm, Register Rt) {
     sys(0b011, 0b0111, cm, 0b001, Rt);

src/hotspot/cpu/aarch64/globals_aarch64.hpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2015, Red Hat Inc. All rights reserved.
+ * Copyright (c) 2015, 2019, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -5833,3 +5833,25 @@ void MacroAssembler::get_thread(Register dst) {
 
   pop(saved_regs, sp);
 }
+
+void MacroAssembler::cache_wb(Address line) {
+  assert(line.getMode() == Address::base_plus_offset, "mode should be base_plus_offset");
+  assert(line.index() == noreg, "index should be noreg");
+  assert(line.offset() == 0, "offset should be 0");
+  // would like to assert this
+  // assert(line._ext.shift == 0, "shift should be zero");
+  if (VM_Version::supports_dcpop()) {
+    // writeback using clear virtual address to point of persistence
+    dc(Assembler::CVAP, line.base());
+  } else {
+    // no need to generate anything as Unsafe.writebackMemory should
+    // never invoke this stub
+  }
+}
+
+void MacroAssembler::cache_wbsync(bool is_pre) {
+  // we only need a barrier post sync
+  if (!is_pre) {
+    membar(Assembler::AnyAny);
+  }
+}

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
+ * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1344,6 +1344,9 @@ class MacroAssembler: public Assembler {
       spill(tmp1, true, dst_offset+8);
     }
   }
+
+  void cache_wb(Address line);
+  void cache_wbsync(bool is_pre);
 };
 
 #ifdef ASSERT

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

@@ -2350,6 +2350,44 @@ class StubGenerator: public StubCodeGenerator {
     return start;
   }
 
+  address generate_data_cache_writeback() {
+    const Register line        = c_rarg0;  // address of line to write back
+
+    __ align(CodeEntryAlignment);
+
+    StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback");
+
+    address start = __ pc();
+    __ enter();
+    __ cache_wb(Address(line, 0));
+    __ leave();
+    __ ret(lr);
+
+    return start;
+  }
+
+  address generate_data_cache_writeback_sync() {
+    const Register is_pre     = c_rarg0;  // pre or post sync
+
+    __ align(CodeEntryAlignment);
+
+    StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback_sync");
+
+    // pre wbsync is a no-op
+    // post wbsync translates to an sfence
+
+    Label skip;
+    address start = __ pc();
+    __ enter();
+    __ cbnz(is_pre, skip);
+    __ cache_wbsync(false);
+    __ bind(skip);
+    __ leave();
+    __ ret(lr);
+
+    return start;
+  }
+
   void generate_arraycopy_stubs() {
     address entry;
     address entry_jbyte_arraycopy;
@@ -5739,6 +5777,10 @@ class StubGenerator: public StubCodeGenerator {
       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
     }
 
+    // data cache line writeback
+    StubRoutines::_data_cache_writeback = generate_data_cache_writeback();
+    StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
+
     if (UseAESIntrinsics) {
       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();

src/hotspot/cpu/aarch64/vm_version_aarch64.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2015, Red Hat Inc. All rights reserved.
+ * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
 #include "asm/macroAssembler.inline.hpp"
 #include "memory/resourceArea.hpp"
 #include "runtime/java.hpp"
+#include "runtime/os.hpp"
 #include "runtime/stubCodeGenerator.hpp"
 #include "utilities/macros.hpp"
 #include "vm_version_aarch64.hpp"
@@ -67,6 +68,7 @@ int VM_Version::_model2;
 int VM_Version::_variant;
 int VM_Version::_revision;
 int VM_Version::_stepping;
+bool VM_Version::_dcpop;
 VM_Version::PsrInfo VM_Version::_psr_info   = { 0, };
 
 static BufferBlob* stub_blob;
@@ -167,7 +169,8 @@ void VM_Version::get_processor_features() {
 
   int cpu_lines = 0;
   if (FILE *f = fopen("/proc/cpuinfo", "r")) {
-    char buf[128], *p;
+    // need a large buffer as the flags line may include lots of text
+    char buf[1024], *p;
     while (fgets(buf, sizeof (buf), f) != NULL) {
       if ((p = strchr(buf, ':')) != NULL) {
         long v = strtol(p+1, NULL, 0);
@@ -181,12 +184,25 @@ void VM_Version::get_processor_features() {
           _model = v;
         } else if (strncmp(buf, "CPU revision", sizeof "CPU revision" - 1) == 0) {
           _revision = v;
+        } else if (strncmp(buf, "flags", sizeof("flags") - 1) == 0) {
+          if (strstr(p+1, "dcpop")) {
+            _dcpop = true;
+          }
         }
       }
     }
     fclose(f);
   }
 
+  if (os::supports_map_sync()) {
+    // if dcpop is available publish data cache line flush size via
+    // generic field, otherwise let if default to zero thereby
+    // disabling writeback
+    if (_dcpop) {
+      _data_cache_line_flush_size = dcache_line;
+    }
+  }
+
   // Enable vendor specific features
 
   // Ampere eMAG

src/hotspot/cpu/aarch64/vm_version_aarch64.hpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Red Hat Inc. All rights reserved.
+ * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,7 +40,7 @@ class VM_Version : public Abstract_VM_Version {
   static int _variant;
   static int _revision;
   static int _stepping;
-
+  static bool _dcpop;
   struct PsrInfo {
     uint32_t dczid_el0;
     uint32_t ctr_el0;
@@ -106,6 +106,7 @@ class VM_Version : public Abstract_VM_Version {
   static int cpu_model2()                     { return _model2; }
   static int cpu_variant()                    { return _variant; }
   static int cpu_revision()                   { return _revision; }
+  static bool supports_dcpop()                { return _dcpop; }
   static ByteSize dczid_el0_offset() { return byte_offset_of(PsrInfo, dczid_el0); }
   static ByteSize ctr_el0_offset()   { return byte_offset_of(PsrInfo, ctr_el0); }
   static bool is_zva_enabled() {

src/hotspot/cpu/x86/assembler_x86.cpp

@@ -2274,6 +2274,14 @@ void Assembler::mfence() {
   emit_int8((unsigned char)0xF0);
 }
 
+// Emit sfence instruction
+void Assembler::sfence() {
+  NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
+  emit_int8(0x0F);
+  emit_int8((unsigned char)0xAE);
+  emit_int8((unsigned char)0xF8);
+}
+
 void Assembler::mov(Register dst, Register src) {
   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 }
@@ -8617,12 +8625,45 @@ void Assembler::cdqq() {
 }
 
 void Assembler::clflush(Address adr) {
+  assert(VM_Version::supports_clflush(), "should do");
   prefix(adr);
   emit_int8(0x0F);
   emit_int8((unsigned char)0xAE);
   emit_operand(rdi, adr);
 }
 
+void Assembler::clflushopt(Address adr) {
+  assert(VM_Version::supports_clflushopt(), "should do!");
+  // adr should be base reg only with no index or offset
+  assert(adr.index() == noreg, "index should be noreg");
+  assert(adr.scale() == Address::no_scale, "scale should be no_scale");
+  assert(adr.disp() == 0, "displacement should be 0");
+  // instruction prefix is 0x66
+  emit_int8(0x66);
+  prefix(adr);
+  // opcode family is 0x0f 0xAE
+  emit_int8(0x0F);
+  emit_int8((unsigned char)0xAE);
+  // extended opcode byte is 7 == rdi
+  emit_operand(rdi, adr);
+}
+
+void Assembler::clwb(Address adr) {
+  assert(VM_Version::supports_clwb(), "should do!");
+  // adr should be base reg only with no index or offset
+  assert(adr.index() == noreg, "index should be noreg");
+  assert(adr.scale() == Address::no_scale, "scale should be no_scale");
+  assert(adr.disp() == 0, "displacement should be 0");
+  // instruction prefix is 0x66
+  emit_int8(0x66);
+  prefix(adr);
+  // opcode family is 0x0f 0xAE
+  emit_int8(0x0F);
+  emit_int8((unsigned char)0xAE);
+  // extended opcode byte is 6 == rsi
+  emit_operand(rsi, adr);
+}
+
 void Assembler::cmovq(Condition cc, Register dst, Register src) {
   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
   emit_int8(0x0F);