8309130: x86_64 AVX512 intrinsics for Arrays.sort methods (int, long, float and double arrays)

make/modules/java.base/Lib.gmk

@@ -233,3 +233,23 @@ ifeq ($(ENABLE_FALLBACK_LINKER), true)
 
   TARGETS += $(BUILD_LIBFALLBACKLINKER)
 endif
+
+################################################################################
+
+ifeq ($(call isTargetOs, linux)+$(call isTargetCpu, x86_64)+$(INCLUDE_COMPILER2), true+true+true)
+  $(eval $(call SetupJdkLibrary, BUILD_LIB_X86_64, \
+      NAME := x86_64, \
+      OPTIMIZATION := HIGH, \
+      CFLAGS := $(CFLAGS_JDKLIB), \
+      CXXFLAGS := $(CXXFLAGS_JDKLIB), \
+      LDFLAGS := $(LDFLAGS_JDKLIB) \
+          $(call SET_SHARED_LIBRARY_ORIGIN), \
+      LDFLAGS_linux := -Wl$(COMMA)--no-as-needed, \
+      LIBS := $(LIBCXX), \
+      LIBS_linux := -lc -lm -ldl, \
+  ))
+
+  TARGETS += $(BUILD_LIB_X86_64)
+endif
+
+################################################################################

src/hotspot/cpu/x86/stubGenerator_x86_64.cpp

@@ -4130,6 +4130,45 @@ void StubGenerator::generate_compiler_stubs() {
       = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square);
   }
 
+  // Load x86_64 library on supported hardware to enable avx512 sort and partition intrinsics
+    if (UseAVX > 2 && VM_Version::supports_avx512dq()) {
+
+      void *libx86_64 = nullptr;
+      char ebuf_x86_64[1024];
+      char dll_name_avx512[JVM_MAXPATHLEN];
+      if (os::dll_locate_lib(dll_name_avx512, sizeof(dll_name_avx512), Arguments::get_dll_dir(), "x86_64")) {
+        libx86_64 = os::dll_load(dll_name_avx512, ebuf_x86_64, sizeof ebuf_x86_64);
+      }
+    // Get addresses for avx512 sort and partition routines
+    if (libx86_64 != nullptr) {
+      log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "x86_64" JNI_LIB_SUFFIX, p2i(libx86_64));
+
+      snprintf(ebuf_x86_64, sizeof(ebuf_x86_64), "avx512_sort_int");
+      StubRoutines::_arraysort_int = (address)os::dll_lookup(libx86_64, ebuf_x86_64);
+
+      snprintf(ebuf_x86_64, sizeof(ebuf_x86_64), "avx512_sort_long");
+      StubRoutines::_arraysort_long = (address)os::dll_lookup(libx86_64, ebuf_x86_64);
+
+      snprintf(ebuf_x86_64, sizeof(ebuf_x86_64), "avx512_sort_float");
+      StubRoutines::_arraysort_float = (address)os::dll_lookup(libx86_64, ebuf_x86_64);
+
+      snprintf(ebuf_x86_64, sizeof(ebuf_x86_64), "avx512_sort_double");
+      StubRoutines::_arraysort_double = (address)os::dll_lookup(libx86_64, ebuf_x86_64);
+
+      snprintf(ebuf_x86_64, sizeof(ebuf_x86_64), "avx512_partition_int");
+      StubRoutines::_array_partition_int = (address)os::dll_lookup(libx86_64, ebuf_x86_64);
+
+      snprintf(ebuf_x86_64, sizeof(ebuf_x86_64), "avx512_partition_long");
+      StubRoutines::_array_partition_long = (address)os::dll_lookup(libx86_64, ebuf_x86_64);
+
+      snprintf(ebuf_x86_64, sizeof(ebuf_x86_64), "avx512_partition_float");
+      StubRoutines::_array_partition_float = (address)os::dll_lookup(libx86_64, ebuf_x86_64);
+
+      snprintf(ebuf_x86_64, sizeof(ebuf_x86_64), "avx512_partition_double");
+      StubRoutines::_array_partition_double = (address)os::dll_lookup(libx86_64, ebuf_x86_64);
+    }
+  }
+
   // Get svml stub routine addresses
   void *libjsvml = nullptr;
   char ebuf[1024];

src/hotspot/share/classfile/vmIntrinsics.hpp

@@ -341,6 +341,15 @@ class methodHandle;
    do_name(     copyOf_name,                                     "copyOf")                                              \
    do_signature(copyOf_signature,             "([Ljava/lang/Object;ILjava/lang/Class;)[Ljava/lang/Object;")             \
                                                                                                                         \
+  do_intrinsic(_arraySort,                java_util_DualPivotQuicksort,       arraySort_name, arraySort_signature, F_S) \
+   do_name(     arraySort_name,                                  "arraySort")                                           \
+   do_signature(arraySort_signature,          "(Ljava/lang/Class;Ljava/lang/Object;JIII)V")                             \
+                                                                                                                        \
+  do_intrinsic(_arrayPartition, java_util_DualPivotQuicksort, arrayPartition_name, arrayPartition_signature, F_S)       \
+   do_name(     arrayPartition_name,                             "arrayPartition")                                      \
+  do_signature(arrayPartition_signature,      "(Ljava/lang/Class;Ljava/lang/Object;JII[IZ)V")                           \
+                                                                                                                        \
+                                                                                                                        \
   do_intrinsic(_copyOfRange,              java_util_Arrays,       copyOfRange_name, copyOfRange_signature,       F_S)   \
    do_name(     copyOfRange_name,                                "copyOfRange")                                         \
    do_signature(copyOfRange_signature,        "([Ljava/lang/Object;IILjava/lang/Class;)[Ljava/lang/Object;")            \

src/hotspot/share/classfile/vmSymbols.hpp

@@ -143,6 +143,7 @@
   template(java_util_Vector,                          "java/util/Vector")                         \
   template(java_util_AbstractList,                    "java/util/AbstractList")                   \
   template(java_util_Hashtable,                       "java/util/Hashtable")                      \
+  template(java_util_DualPivotQuicksort,              "java/util/DualPivotQuicksort")             \
   template(java_lang_Compiler,                        "java/lang/Compiler")                       \
   template(jdk_internal_misc_Signal,                  "jdk/internal/misc/Signal")                 \
   template(jdk_internal_util_Preconditions,           "jdk/internal/util/Preconditions")          \

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

@@ -387,6 +387,11 @@ void ShenandoahBarrierC2Support::verify(RootNode* root) {
           verify_type t;
         } args[6];
       } calls[] = {
+        "array_partition_stub",
+        { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
+          { -1, ShenandoahNone }, { -1, ShenandoahNone } },
+        "arraysort_stub",
+        { { TypeFunc::Parms, ShenandoahStore }, { -1, ShenandoahNone }, { -1, ShenandoahNone } },
         "aescrypt_encryptBlock",
         { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },
           { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

src/hotspot/share/jvmci/vmStructs_jvmci.cpp

@@ -327,6 +327,14 @@
   static_field(StubRoutines,                _checkcast_arraycopy_uninit,                      address)                               \
   static_field(StubRoutines,                _unsafe_arraycopy,                                address)                               \
   static_field(StubRoutines,                _generic_arraycopy,                               address)                               \
+  static_field(StubRoutines,                _arraysort_int,                                   address)                               \
+  static_field(StubRoutines,                _arraysort_long,                                  address)                               \
+  static_field(StubRoutines,                _arraysort_float,                                 address)                               \
+  static_field(StubRoutines,                _arraysort_double,                                address)                               \
+  static_field(StubRoutines,                _array_partition_int,                             address)                               \
+  static_field(StubRoutines,                _array_partition_long,                            address)                               \
+  static_field(StubRoutines,                _array_partition_float,                           address)                               \
+  static_field(StubRoutines,                _array_partition_double,                          address)                               \
                                                                                                                                      \
   static_field(StubRoutines,                _aescrypt_encryptBlock,                           address)                               \
   static_field(StubRoutines,                _aescrypt_decryptBlock,                           address)                               \

src/hotspot/share/opto/c2compiler.cpp

@@ -568,6 +568,8 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method) {
   case vmIntrinsics::_min_strict:
   case vmIntrinsics::_max_strict:
   case vmIntrinsics::_arraycopy:
+  case vmIntrinsics::_arraySort:
+  case vmIntrinsics::_arrayPartition:
   case vmIntrinsics::_indexOfL:
   case vmIntrinsics::_indexOfU:
   case vmIntrinsics::_indexOfUL:

src/hotspot/share/opto/escape.cpp

@@ -1190,6 +1190,8 @@ void ConnectionGraph::process_call_arguments(CallNode *call) {
                   strcmp(call->as_CallLeaf()->_name, "bigIntegerRightShiftWorker") == 0 ||
                   strcmp(call->as_CallLeaf()->_name, "bigIntegerLeftShiftWorker") == 0 ||
                   strcmp(call->as_CallLeaf()->_name, "vectorizedMismatch") == 0 ||
+                  strcmp(call->as_CallLeaf()->_name, "arraysort_stub") == 0 ||
+                  strcmp(call->as_CallLeaf()->_name, "array_partition_stub") == 0 ||
                   strcmp(call->as_CallLeaf()->_name, "get_class_id_intrinsic") == 0)
                  ))) {
             call->dump();

src/hotspot/share/opto/library_call.cpp

@@ -292,6 +292,9 @@ bool LibraryCallKit::try_to_inline(int predicate) {
 
   case vmIntrinsics::_arraycopy:                return inline_arraycopy();
 
+  case vmIntrinsics::_arraySort:                return inline_arraysort();
+  case vmIntrinsics::_arrayPartition:           return inline_array_partition();
+
   case vmIntrinsics::_compareToL:               return inline_string_compareTo(StrIntrinsicNode::LL);
   case vmIntrinsics::_compareToU:               return inline_string_compareTo(StrIntrinsicNode::UU);
   case vmIntrinsics::_compareToLU:              return inline_string_compareTo(StrIntrinsicNode::LU);
@@ -5192,6 +5195,82 @@ void LibraryCallKit::create_new_uncommon_trap(CallStaticJavaNode* uncommon_trap_
   uncommon_trap_call->set_req(0, top()); // not used anymore, kill it
 }
 
+//------------------------------inline_array_partition-----------------------
+bool LibraryCallKit::inline_array_partition() {
+
+  address stubAddr = nullptr;
+  const char *stubName;
+  stubName = "array_partition_stub";
+
+  Node* elementType     = null_check(argument(0));
+  Node* obj             = argument(1);
+  Node* offset          = argument(2);
+  Node* fromIndex       = argument(4);
+  Node* toIndex         = argument(5);
+  Node* pivot_indices   = argument(6);
+  Node* isDualPivot     = argument(7);
+
+  const TypeInstPtr* elem_klass = gvn().type(elementType)->isa_instptr();
+  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
+  BasicType bt = elem_type->basic_type();
+  stubAddr = StubRoutines::select_array_partition_function(bt);
+  if (stubAddr == nullptr) return false;
+
+  const TypeAryPtr* obj_t = _gvn.type(obj)->isa_aryptr();
+  if (obj_t == nullptr || obj_t->elem() == Type::BOTTOM ) {
+    return false; // failed input validation
+  }
+  Node* obj_adr = make_unsafe_address(obj, offset);
+
+  pivot_indices = must_be_not_null(pivot_indices, true);
+  const TypeAryPtr* pivot_indices_type = pivot_indices->Value(&_gvn)->isa_aryptr();
+  if (pivot_indices_type == nullptr || pivot_indices_type->elem() == Type::BOTTOM ) {
+    return false; // failed input validation
+  }
+  Node* pivot_indices_adr = array_element_address(pivot_indices, intcon(0), T_INT);
+
+  // Call the stub.
+  make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::array_partition_Type(),
+                    stubAddr, stubName, TypePtr::BOTTOM,
+                    obj_adr, fromIndex, toIndex, pivot_indices_adr, isDualPivot);
+
+  return true;
+}
+
+
+//------------------------------inline_arraysort-----------------------
+bool LibraryCallKit::inline_arraysort() {
+
+  address stubAddr = nullptr;
+  const char *stubName;
+  stubName = "arraysort_stub";
+
+  Node* elementType     = null_check(argument(0));
+  Node* obj             = argument(1);
+  Node* offset          = argument(2);
+  Node* fromIndex       = argument(4);
+  Node* toIndex         = argument(5);
+
+  const TypeInstPtr* elem_klass = gvn().type(elementType)->isa_instptr();
+  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
+  BasicType bt = elem_type->basic_type();
+  stubAddr = StubRoutines::select_arraysort_function(bt);
+  if (stubAddr == nullptr) return false;
+
+  const TypeAryPtr* obj_t = _gvn.type(obj)->isa_aryptr();
+  if (obj_t == nullptr || obj_t->elem() == Type::BOTTOM ) {
+    return false; // failed input validation
+  }
+  Node* obj_adr = make_unsafe_address(obj, offset);
+  // Call the stub.
+  make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::array_sort_Type(),
+                    stubAddr, stubName, TypePtr::BOTTOM,
+                    obj_adr, fromIndex, toIndex);
+
+  return true;
+}
+
+
 //------------------------------inline_arraycopy-----------------------
 // public static native void java.lang.System.arraycopy(Object src,  int  srcPos,
 //                                                      Object dest, int destPos,

src/hotspot/share/opto/library_call.hpp

@@ -279,7 +279,8 @@ class LibraryCallKit : public GraphKit {
   JVMState* arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp);
   void arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms_before_guards, int saved_reexecute_sp,
                                       uint new_idx);
-
+  bool inline_arraysort();
+  bool inline_array_partition();
   typedef enum { LS_get_add, LS_get_set, LS_cmp_swap, LS_cmp_swap_weak, LS_cmp_exchange } LoadStoreKind;
   bool inline_unsafe_load_store(BasicType type,  LoadStoreKind kind, AccessKind access_kind);
   bool inline_unsafe_fence(vmIntrinsics::ID id);

src/hotspot/share/opto/runtime.cpp

@@ -857,6 +857,46 @@ const TypeFunc* OptoRuntime::array_fill_Type() {
   return TypeFunc::make(domain, range);
 }
 
+const TypeFunc* OptoRuntime::array_partition_Type() {
+  // create input type (domain)
+  int num_args = 5;
+  int argcnt = num_args;
+  const Type** fields = TypeTuple::fields(argcnt);
+  int argp = TypeFunc::Parms;
+  fields[argp++] = TypePtr::NOTNULL;    // array
+  fields[argp++] = TypeInt::INT;    // low
+  fields[argp++] = TypeInt::INT;    // end
+  fields[argp++] = TypePtr::NOTNULL;    // pivot_indices (int array)
+  fields[argp++] = TypeInt::BOOL;       // isDualPivot
+  assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
+  const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
+
+  // no result type needed
+  fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = nullptr; // void
+  const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
+  return TypeFunc::make(domain, range);
+}
+
+const TypeFunc* OptoRuntime::array_sort_Type() {
+  // create input type (domain)
+  int num_args      = 3;
+  int argcnt = num_args;
+  const Type** fields = TypeTuple::fields(argcnt);
+  int argp = TypeFunc::Parms;
+  fields[argp++] = TypePtr::NOTNULL;    // array
+  fields[argp++] = TypeInt::INT;    // fromIndex
+  fields[argp++] = TypeInt::INT;    // toIndex
+  assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
+  const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
+
+  // no result type needed
+  fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = nullptr; // void
+  const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
+  return TypeFunc::make(domain, range);
+}
+
 // for aescrypt encrypt/decrypt operations, just three pointers returning void (length is constant)
 const TypeFunc* OptoRuntime::aescrypt_block_Type() {
   // create input type (domain)

src/hotspot/share/opto/runtime.hpp

@@ -268,6 +268,8 @@ class OptoRuntime : public AllStatic {
 
   static const TypeFunc* array_fill_Type();
 
+  static const TypeFunc* array_sort_Type();
+  static const TypeFunc* array_partition_Type();
   static const TypeFunc* aescrypt_block_Type();
   static const TypeFunc* cipherBlockChaining_aescrypt_Type();
   static const TypeFunc* electronicCodeBook_aescrypt_Type();

src/hotspot/share/runtime/stubRoutines.cpp

@@ -176,6 +176,15 @@ address StubRoutines::_hf2f = nullptr;
 address StubRoutines::_vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP] = {{nullptr}, {nullptr}};
 address StubRoutines::_vector_d_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP] = {{nullptr}, {nullptr}};
 
+address StubRoutines::_arraysort_int = nullptr;
+address StubRoutines::_arraysort_long = nullptr;
+address StubRoutines::_arraysort_float = nullptr;
+address StubRoutines::_arraysort_double = nullptr;
+address StubRoutines::_array_partition_int  = nullptr;
+address StubRoutines::_array_partition_long = nullptr;
+address StubRoutines::_array_partition_float = nullptr;
+address StubRoutines::_array_partition_double = nullptr;
+
 address StubRoutines::_cont_thaw          = nullptr;
 address StubRoutines::_cont_returnBarrier = nullptr;
 address StubRoutines::_cont_returnBarrierExc = nullptr;
@@ -648,3 +657,27 @@ UnsafeCopyMemoryMark::~UnsafeCopyMemoryMark() {
     }
   }
 }
+
+address StubRoutines::select_arraysort_function(BasicType t) {
+  switch(t) {
+    case T_INT:    return _arraysort_int;
+    case T_LONG:   return _arraysort_long;
+    case T_FLOAT:  return _arraysort_float;
+    case T_DOUBLE: return _arraysort_double;
+  default:
+    ShouldNotReachHere();
+    return nullptr;
+  }
+}
+
+address StubRoutines::select_array_partition_function(BasicType t) {
+  switch(t) {
+    case T_INT:    return _array_partition_int;
+    case T_LONG:   return _array_partition_long;
+    case T_FLOAT:  return _array_partition_float;
+    case T_DOUBLE: return _array_partition_double;
+  default:
+    ShouldNotReachHere();
+    return nullptr;
+  }
+}

src/hotspot/share/runtime/stubRoutines.hpp

@@ -153,6 +153,14 @@ class StubRoutines: AllStatic {
   static BufferBlob* _compiler_stubs_code;                 // code buffer for C2 intrinsics
   static BufferBlob* _final_stubs_code;                    // code buffer for all other routines
 
+  static address _arraysort_int;
+  static address _arraysort_long;
+  static address _arraysort_float;
+  static address _arraysort_double;
+  static address _array_partition_int;
+  static address _array_partition_long;
+  static address _array_partition_float;
+  static address _array_partition_double;
   // Leaf routines which implement arraycopy and their addresses
   // arraycopy operands aligned on element type boundary
   static address _jbyte_arraycopy;
@@ -373,6 +381,8 @@ class StubRoutines: AllStatic {
   static UnsafeArrayCopyStub UnsafeArrayCopy_stub()         { return CAST_TO_FN_PTR(UnsafeArrayCopyStub,  _unsafe_arraycopy); }
 
   static address generic_arraycopy()   { return _generic_arraycopy; }
+  static address select_arraysort_function(BasicType t);
+  static address select_array_partition_function(BasicType t);
 
   static address jbyte_fill()          { return _jbyte_fill; }
   static address jshort_fill()         { return _jshort_fill; }