Merge branch 'upstream-master'

src/hotspot/cpu/aarch64/aarch64.ad

@@ -7780,7 +7780,7 @@ instruct membar_acquire() %{
   ins_cost(VOLATILE_REF_COST);
 
   format %{ "membar_acquire\n\t"
-            "dmb ish" %}
+            "dmb ishld" %}
 
   ins_encode %{
     __ block_comment("membar_acquire");
@@ -7834,11 +7834,13 @@ instruct membar_release() %{
   ins_cost(VOLATILE_REF_COST);
 
   format %{ "membar_release\n\t"
-            "dmb ish" %}
+            "dmb ishst\n\tdmb ishld" %}
 
   ins_encode %{
     __ block_comment("membar_release");
-    __ membar(Assembler::LoadStore|Assembler::StoreStore);
+    // These will be merged if AlwaysMergeDMB is enabled.
+    __ membar(Assembler::StoreStore);
+    __ membar(Assembler::LoadStore);
   %}
   ins_pipe(pipe_serial);
 %}

src/hotspot/cpu/aarch64/globals_aarch64.hpp

@@ -124,6 +124,8 @@ define_pd_global(intx, InlineSmallCode,          1000);
           range(1, 99)                                                  \
   product(ccstr, UseBranchProtection, "none",                           \
           "Branch Protection to use: none, standard, pac-ret")          \
+  product(bool, AlwaysMergeDMB, true, DIAGNOSTIC,                       \
+          "Always merge DMB instructions in code emission")             \
 
 // end of ARCH_FLAGS
 

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -2350,14 +2350,36 @@ void MacroAssembler::membar(Membar_mask_bits order_constraint) {
   address last = code()->last_insn();
   if (last != nullptr && nativeInstruction_at(last)->is_Membar() && prev == last) {
     NativeMembar *bar = NativeMembar_at(prev);
-    // We are merging two memory barrier instructions.  On AArch64 we
-    // can do this simply by ORing them together.
-    bar->set_kind(bar->get_kind() | order_constraint);
-    BLOCK_COMMENT("merged membar");
-  } else {
-    code()->set_last_insn(pc());
-    dmb(Assembler::barrier(order_constraint));
+    if (AlwaysMergeDMB) {
+      bar->set_kind(bar->get_kind() | order_constraint);
+      BLOCK_COMMENT("merged membar(always)");
+      return;
+    }
+    // Don't promote DMB ST|DMB LD to DMB (a full barrier) because
+    // doing so would introduce a StoreLoad which the caller did not
+    // intend
+    if (bar->get_kind() == order_constraint
+        || bar->get_kind() == AnyAny
+        || order_constraint == AnyAny) {
+      // We are merging two memory barrier instructions.  On AArch64 we
+      // can do this simply by ORing them together.
+      bar->set_kind(bar->get_kind() | order_constraint);
+      BLOCK_COMMENT("merged membar");
+      return;
+    } else {
+      // A special case like "DMB ST;DMB LD;DMB ST", the last DMB can be skipped
+      // We need check the last 2 instructions
+      address prev2 = prev - NativeMembar::instruction_size;
+      if (last != code()->last_label() && nativeInstruction_at(prev2)->is_Membar()) {
+        NativeMembar *bar2 = NativeMembar_at(prev2);
+        assert(bar2->get_kind() == order_constraint, "it should be merged before");
+        BLOCK_COMMENT("merged membar(elided)");
+        return;
+      }
+    }
   }
+  code()->set_last_insn(pc());
+  dmb(Assembler::barrier(order_constraint));
 }
 
 bool MacroAssembler::try_merge_ldst(Register rt, const Address &adr, size_t size_in_bytes, bool is_store) {

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -150,6 +150,7 @@ class MacroAssembler: public Assembler {
   void bind(Label& L) {
     Assembler::bind(L);
     code()->clear_last_insn();
+    code()->set_last_label(pc());
   }
 
   void membar(Membar_mask_bits order_constraint);

src/hotspot/cpu/aarch64/vm_version_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2015, 2020, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -230,6 +230,9 @@ void VM_Version::initialize() {
     if (FLAG_IS_DEFAULT(OnSpinWaitInstCount)) {
       FLAG_SET_DEFAULT(OnSpinWaitInstCount, 1);
     }
+    if (FLAG_IS_DEFAULT(AlwaysMergeDMB)) {
+      FLAG_SET_DEFAULT(AlwaysMergeDMB, false);
+    }
   }
 
   if (_cpu == CPU_ARM) {

src/hotspot/cpu/riscv/assembler_riscv.hpp

@@ -1828,10 +1828,12 @@ enum Nf {
   // Vector unordered indexed load instructions
   INSN( vluxei8_v, 0b0000111, 0b000, 0b01, 0b0);
   INSN(vluxei32_v, 0b0000111, 0b110, 0b01, 0b0);
+  INSN(vluxei64_v, 0b0000111, 0b111, 0b01, 0b0);
 
   // Vector unordered indexed store instructions
   INSN( vsuxei8_v, 0b0100111, 0b000, 0b01, 0b0);
   INSN(vsuxei32_v, 0b0100111, 0b110, 0b01, 0b0);
+  INSN(vsuxei64_v, 0b0100111, 0b111, 0b01, 0b0);
 
 #undef INSN
 

src/hotspot/cpu/riscv/riscv_v.ad

@@ -4795,12 +4795,11 @@ instruct vcountTrailingZeros(vReg dst, vReg src) %{
 
 // ------------------------------ Vector Load Gather ---------------------------
 
-instruct gather_load(vReg dst, indirect mem, vReg idx) %{
-  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 4 ||
-            type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
+instruct gather_loadS(vReg dst, indirect mem, vReg idx) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
   match(Set dst (LoadVectorGather mem idx));
   effect(TEMP_DEF dst);
-  format %{ "gather_load $dst, $mem, $idx" %}
+  format %{ "gather_loadS $dst, $mem, $idx" %}
   ins_encode %{
     __ vmv1r_v(as_VectorRegister($dst$$reg), as_VectorRegister($idx$$reg));
     BasicType bt = Matcher::vector_element_basic_type(this);
@@ -4813,12 +4812,28 @@ instruct gather_load(vReg dst, indirect mem, vReg idx) %{
   ins_pipe(pipe_slow);
 %}
 
-instruct gather_load_masked(vReg dst, indirect mem, vReg idx, vRegMask_V0 v0, vReg tmp) %{
-  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 4 ||
-            type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
+instruct gather_loadD(vReg dst, indirect mem, vReg idx) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
+  match(Set dst (LoadVectorGather mem idx));
+  effect(TEMP_DEF dst);
+  format %{ "gather_loadD $dst, $mem, $idx" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vzext_vf2(as_VectorRegister($dst$$reg), as_VectorRegister($idx$$reg));
+    __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), (int)sew);
+    __ vluxei64_v(as_VectorRegister($dst$$reg), as_Register($mem$$base),
+                  as_VectorRegister($dst$$reg));
+ %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct gather_loadS_masked(vReg dst, indirect mem, vReg idx, vRegMask_V0 v0, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
   match(Set dst (LoadVectorGatherMasked mem (Binary idx v0)));
   effect(TEMP_DEF dst, TEMP tmp);
-  format %{ "gather_load_masked $dst, $mem, $idx, $v0\t# KILL $tmp" %}
+  format %{ "gather_loadS_masked $dst, $mem, $idx, $v0\t# KILL $tmp" %}
   ins_encode %{
     __ vmv1r_v(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
     BasicType bt = Matcher::vector_element_basic_type(this);
@@ -4833,14 +4848,32 @@ instruct gather_load_masked(vReg dst, indirect mem, vReg idx, vRegMask_V0 v0, vR
   ins_pipe(pipe_slow);
 %}
 
+instruct gather_loadD_masked(vReg dst, indirect mem, vReg idx, vRegMask_V0 v0, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
+  match(Set dst (LoadVectorGatherMasked mem (Binary idx v0)));
+  effect(TEMP_DEF dst, TEMP tmp);
+  format %{ "gather_loadD_masked $dst, $mem, $idx, $v0\t# KILL $tmp" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vzext_vf2(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
+    __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), (int)sew);
+    __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg),
+               as_VectorRegister($dst$$reg));
+    __ vluxei64_v(as_VectorRegister($dst$$reg), as_Register($mem$$base),
+                  as_VectorRegister($tmp$$reg), Assembler::v0_t);
+ %}
+  ins_pipe(pipe_slow);
+%}
+
 // ------------------------------ Vector Store Scatter -------------------------
 
-instruct scatter_store(indirect mem, vReg src, vReg idx, vReg tmp) %{
-  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4 ||
-            type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
+instruct scatter_storeS(indirect mem, vReg src, vReg idx, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
   match(Set mem (StoreVectorScatter mem (Binary src idx)));
   effect(TEMP tmp);
-  format %{ "scatter_store $mem, $idx, $src\t# KILL $tmp" %}
+  format %{ "scatter_storeS $mem, $idx, $src\t# KILL $tmp" %}
   ins_encode %{
     __ vmv1r_v(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
     BasicType bt = Matcher::vector_element_basic_type(this, $src);
@@ -4853,12 +4886,28 @@ instruct scatter_store(indirect mem, vReg src, vReg idx, vReg tmp) %{
   ins_pipe(pipe_slow);
 %}
 
-instruct scatter_store_masked(indirect mem, vReg src, vReg idx, vRegMask_V0 v0, vReg tmp) %{
-  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4 ||
-            type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
+instruct scatter_storeD(indirect mem, vReg src, vReg idx, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
+  match(Set mem (StoreVectorScatter mem (Binary src idx)));
+  effect(TEMP tmp);
+  format %{ "scatter_storeD $mem, $idx, $src\t# KILL $tmp" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this, $src);
+    Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
+    __ vsetvli_helper(bt, Matcher::vector_length(this, $src));
+    __ vzext_vf2(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
+    __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), (int)sew);
+    __ vsuxei64_v(as_VectorRegister($src$$reg), as_Register($mem$$base),
+                  as_VectorRegister($tmp$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct scatter_storeS_masked(indirect mem, vReg src, vReg idx, vRegMask_V0 v0, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx v0))));
   effect(TEMP tmp);
-  format %{ "scatter_store_masked $mem, $idx, $src, $v0\t# KILL $tmp" %}
+  format %{ "scatter_storeS_masked $mem, $idx, $src, $v0\t# KILL $tmp" %}
   ins_encode %{
     __ vmv1r_v(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
     BasicType bt = Matcher::vector_element_basic_type(this, $src);
@@ -4871,6 +4920,23 @@ instruct scatter_store_masked(indirect mem, vReg src, vReg idx, vRegMask_V0 v0,
   ins_pipe(pipe_slow);
 %}
 
+instruct scatter_storeD_masked(indirect mem, vReg src, vReg idx, vRegMask_V0 v0, vReg tmp) %{
+  predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
+  match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx v0))));
+  effect(TEMP tmp);
+  format %{ "scatter_storeD_masked $mem, $idx, $src, $v0\t# KILL $tmp" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this, $src);
+    Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
+    __ vsetvli_helper(bt, Matcher::vector_length(this, $src));
+    __ vzext_vf2(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
+    __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), (int)sew);
+    __ vsuxei64_v(as_VectorRegister($src$$reg), as_Register($mem$$base),
+                  as_VectorRegister($tmp$$reg), Assembler::v0_t);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 // ------------------------------ Populate Index to a Vector -------------------
 
 instruct populateindex(vReg dst, iRegIorL2I src1, iRegIorL2I src2, vReg tmp) %{

src/hotspot/os/linux/os_linux.cpp

@@ -314,7 +314,10 @@ static jlong host_free_swap() {
 }
 
 jlong os::free_swap_space() {
-  jlong host_free_swap_val = host_free_swap();
+  // os::total_swap_space() might return the containerized limit which might be
+  // less than host_free_swap(). The upper bound of free swap needs to be the lower of the two.
+  jlong host_free_swap_val = MIN2(os::total_swap_space(), host_free_swap());
+  assert(host_free_swap_val >= 0, "sysinfo failed?");
   if (OSContainer::is_containerized()) {
     jlong mem_swap_limit = OSContainer::memory_and_swap_limit_in_bytes();
     jlong mem_limit = OSContainer::memory_limit_in_bytes();

src/hotspot/share/asm/codeBuffer.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -928,6 +928,10 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) {
   // Move all the code and relocations to the new blob:
   relocate_code_to(&cb);
 
+  // some internal addresses, _last_insn _last_label, are used during code emission,
+  // adjust them in expansion
+  adjust_internal_address(insts_begin(), cb.insts_begin());
+
   // Copy the temporary code buffer into the current code buffer.
   // Basically, do {*this = cb}, except for some control information.
   this->take_over_code_from(&cb);
@@ -949,6 +953,15 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) {
 #endif //PRODUCT
 }
 
+void CodeBuffer::adjust_internal_address(address from, address to) {
+  if (_last_insn != nullptr) {
+    _last_insn += to - from;
+  }
+  if (_last_label != nullptr) {
+    _last_label += to - from;
+  }
+}
+
 void CodeBuffer::take_over_code_from(CodeBuffer* cb) {
   // Must already have disposed of the old blob somehow.
   assert(blob() == nullptr, "must be empty");

src/hotspot/share/asm/codeBuffer.hpp

@@ -433,6 +433,7 @@ class CodeBuffer: public StackObj DEBUG_ONLY(COMMA private Scrubber) {
   Arena*       _overflow_arena;
 
   address      _last_insn;      // used to merge consecutive memory barriers, loads or stores.
+  address      _last_label;     // record last bind label address, it's also the start of current bb.
 
   SharedStubToInterpRequests* _shared_stub_to_interp_requests; // used to collect requests for shared iterpreter stubs
   SharedTrampolineRequests*   _shared_trampoline_requests;     // used to collect requests for shared trampolines
@@ -457,6 +458,7 @@ class CodeBuffer: public StackObj DEBUG_ONLY(COMMA private Scrubber) {
     _oop_recorder    = nullptr;
     _overflow_arena  = nullptr;
     _last_insn       = nullptr;
+    _last_label      = nullptr;
     _finalize_stubs  = false;
     _shared_stub_to_interp_requests = nullptr;
     _shared_trampoline_requests = nullptr;
@@ -510,6 +512,9 @@ class CodeBuffer: public StackObj DEBUG_ONLY(COMMA private Scrubber) {
   // moves code sections to new buffer (assumes relocs are already in there)
   void relocate_code_to(CodeBuffer* cb) const;
 
+  // adjust some internal address during expand
+  void adjust_internal_address(address from, address to);
+
   // set up a model of the final layout of my contents
   void compute_final_layout(CodeBuffer* dest) const;
 
@@ -679,6 +684,9 @@ class CodeBuffer: public StackObj DEBUG_ONLY(COMMA private Scrubber) {
   void set_last_insn(address a) { _last_insn = a; }
   void clear_last_insn() { set_last_insn(nullptr); }
 
+  address last_label() const { return _last_label; }
+  void set_last_label(address a) { _last_label = a; }
+
 #ifndef PRODUCT
   AsmRemarks &asm_remarks() { return _asm_remarks; }
   DbgStrings &dbg_strings() { return _dbg_strings; }

src/hotspot/share/compiler/compilerDirectives.cpp

@@ -761,7 +761,7 @@ DirectiveSet* DirectivesStack::getMatchingDirective(const methodHandle& method,
       if (dir->is_default_directive() || dir->match(method)) {
         match = dir->get_for(comp);
         assert(match != nullptr, "Consistency");
-        if (match->EnableOption) {
+        if (match->EnableOption || dir->is_default_directive()) {
           // The directiveSet for this compile is also enabled -> success
           dir->inc_refcount();
           break;

src/hotspot/share/nmt/memReporter.cpp

@@ -51,13 +51,7 @@ static ssize_t counter_diff(size_t c1, size_t c2) {
 }
 
 MemReporterBase::MemReporterBase(outputStream* out, size_t scale) :
-  _scale(scale), _output(out) {
-  _output->set_autoindent(true);
-}
-
-MemReporterBase::~MemReporterBase() {
-  _output->set_autoindent(false);
-}
+  _scale(scale), _output(out), _auto_indentor(out) {}
 
 size_t MemReporterBase::reserved_total(const MallocMemory* malloc, const VirtualMemory* vm) {
   return malloc->malloc_size() + malloc->arena_size() + vm->reserved();

src/hotspot/share/nmt/memReporter.hpp

@@ -38,14 +38,14 @@ class MemReporterBase : public StackObj {
  private:
   const size_t  _scale;         // report in this scale
   outputStream* const _output;  // destination
+  StreamAutoIndentor _auto_indentor;
 
  public:
 
   // Default scale to use if no scale given.
   static const size_t default_scale = K;
 
   MemReporterBase(outputStream* out, size_t scale = default_scale);
-  ~MemReporterBase();
 
   // Helper functions
   // Calculate total reserved and committed amount

src/hotspot/share/prims/upcallLinker.cpp

@@ -48,7 +48,6 @@ extern struct JavaVM_ main_vm;
 struct UpcallContext {
   Thread* attachedThread;
 
-  UpcallContext() {} // Explicit constructor to address XL C compiler bug.
   ~UpcallContext() {
     if (attachedThread != nullptr) {
       JavaVM_ *vm = (JavaVM *)(&main_vm);