8340079: Modify rearrange/selectFrom Vector API methods to perform wrapIndexes instead of checkIndexes

src/hotspot/share/adlc/formssel.cpp

@@ -4357,7 +4357,7 @@ bool MatchRule::is_vector() const {
     "RoundDoubleModeV","RotateLeftV" , "RotateRightV", "LoadVector","StoreVector",
     "LoadVectorGather", "StoreVectorScatter", "LoadVectorGatherMasked", "StoreVectorScatterMasked",
     "VectorTest", "VectorLoadMask", "VectorStoreMask", "VectorBlend", "VectorInsert",
-    "VectorRearrange","VectorLoadShuffle", "VectorLoadConst",
+    "VectorRearrange", "VectorLoadShuffle", "VectorLoadConst",
     "VectorCastB2X", "VectorCastS2X", "VectorCastI2X",
     "VectorCastL2X", "VectorCastF2X", "VectorCastD2X", "VectorCastF2HF", "VectorCastHF2F",
     "VectorUCastB2X", "VectorUCastS2X", "VectorUCastI2X",

src/hotspot/share/classfile/vmIntrinsics.hpp

@@ -1007,6 +1007,15 @@ class methodHandle;
                                                "Ljdk/internal/vm/vector/VectorSupport$Vector;")                                                \
    do_name(vector_shuffle_to_vector_name, "shuffleToVector")                                                                                   \
                                                                                                                                                \
+  do_intrinsic(_VectorWrapShuffleIndexes, jdk_internal_vm_vector_VectorSupport, vector_wrap_shuffle_indexes_name,                              \
+                                                                                                         vector_wrap_shuffle_indexes_sig, F_S) \
+   do_signature(vector_wrap_shuffle_indexes_sig, "(Ljava/lang/Class;"                                                                          \
+                                                  "Ljava/lang/Class;"                                                                          \
+                                                  "Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;"                                       \
+                                                  "ILjdk/internal/vm/vector/VectorSupport$WrapShuffleIndexesOperation;)"                       \
+                                                  "Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;")                                      \
+   do_name(vector_wrap_shuffle_indexes_name, "wrapShuffleIndexes")                                                                             \
+                                                                                                                                               \
   do_intrinsic(_VectorLoadOp, jdk_internal_vm_vector_VectorSupport, vector_load_op_name, vector_load_op_sig, F_S)                              \
    do_signature(vector_load_op_sig, "(Ljava/lang/Class;"                                                                                       \
                                      "Ljava/lang/Class;"                                                                                       \
@@ -1128,6 +1137,18 @@ class methodHandle;
                                        "Ljdk/internal/vm/vector/VectorSupport$Vector;")                                                        \
    do_name(vector_rearrange_name, "rearrangeOp")                                                                                               \
                                                                                                                                                \
+  do_intrinsic(_VectorSelectFrom, jdk_internal_vm_vector_VectorSupport, vector_select_from_name, vector_select_from_sig, F_S)                  \
+   do_signature(vector_select_from_sig, "(Ljava/lang/Class;"                                                                                   \
+                                        "Ljava/lang/Class;"                                                                                     \
+                                        "Ljava/lang/Class;"                                                                                     \
+                                        "I"                                                                                                     \
+                                        "Ljdk/internal/vm/vector/VectorSupport$Vector;"                                                         \
+                                        "Ljdk/internal/vm/vector/VectorSupport$Vector;"                                                         \
+                                        "Ljdk/internal/vm/vector/VectorSupport$VectorMask;"                                                     \
+                                        "Ljdk/internal/vm/vector/VectorSupport$VectorSelectFromOp;)"                                            \
+                                        "Ljdk/internal/vm/vector/VectorSupport$Vector;")                                                        \
+   do_name(vector_select_from_name, "selectFromOp")                                                                                              \
+                                                                                                                                               \
   do_intrinsic(_VectorExtract, jdk_internal_vm_vector_VectorSupport, vector_extract_name, vector_extract_sig, F_S)                             \
    do_signature(vector_extract_sig, "(Ljava/lang/Class;"                                                                                       \
                                      "Ljava/lang/Class;"                                                                                       \

src/hotspot/share/opto/c2compiler.cpp

@@ -810,6 +810,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
   case vmIntrinsics::_VectorFromBitsCoerced:
   case vmIntrinsics::_VectorShuffleIota:
   case vmIntrinsics::_VectorShuffleToVector:
+  case vmIntrinsics::_VectorWrapShuffleIndexes:
   case vmIntrinsics::_VectorLoadOp:
   case vmIntrinsics::_VectorLoadMaskedOp:
   case vmIntrinsics::_VectorStoreOp:
@@ -820,6 +821,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
   case vmIntrinsics::_VectorTest:
   case vmIntrinsics::_VectorBlend:
   case vmIntrinsics::_VectorRearrange:
+  case vmIntrinsics::_VectorSelectFrom:
   case vmIntrinsics::_VectorCompare:
   case vmIntrinsics::_VectorBroadcastInt:
   case vmIntrinsics::_VectorConvert:

src/hotspot/share/opto/library_call.cpp

@@ -716,6 +716,8 @@ bool LibraryCallKit::try_to_inline(int predicate) {
     return inline_vector_mask_operation();
   case vmIntrinsics::_VectorShuffleToVector:
     return inline_vector_shuffle_to_vector();
+  case vmIntrinsics::_VectorWrapShuffleIndexes:
+    return inline_vector_wrap_shuffle_indexes();
   case vmIntrinsics::_VectorLoadOp:
     return inline_vector_mem_operation(/*is_store=*/false);
   case vmIntrinsics::_VectorLoadMaskedOp:
@@ -736,6 +738,8 @@ bool LibraryCallKit::try_to_inline(int predicate) {
     return inline_vector_blend();
   case vmIntrinsics::_VectorRearrange:
     return inline_vector_rearrange();
+  case vmIntrinsics::_VectorSelectFrom:
+    return inline_vector_select_from();
   case vmIntrinsics::_VectorCompare:
     return inline_vector_compare();
   case vmIntrinsics::_VectorBroadcastInt:

src/hotspot/share/opto/library_call.hpp

@@ -353,6 +353,7 @@ class LibraryCallKit : public GraphKit {
   bool inline_vector_nary_operation(int n);
   bool inline_vector_frombits_coerced();
   bool inline_vector_shuffle_to_vector();
+  bool inline_vector_wrap_shuffle_indexes();
   bool inline_vector_shuffle_iota();
   Node* partially_wrap_indexes(Node* index_vec, int num_elem, BasicType type_bt);
   bool inline_vector_mask_operation();
@@ -363,6 +364,7 @@ class LibraryCallKit : public GraphKit {
   bool inline_vector_test();
   bool inline_vector_blend();
   bool inline_vector_rearrange();
+  bool inline_vector_select_from();
   bool inline_vector_compare();
   bool inline_vector_broadcast_int();
   bool inline_vector_convert();

src/hotspot/share/opto/vectorIntrinsics.cpp

@@ -757,6 +757,64 @@ bool LibraryCallKit::inline_vector_shuffle_to_vector() {
   return true;
 }
 
+// public static
+// <E,
+//  SH extends VectorShuffle<E>>
+// SH wrapShuffleIndexes(Class<E> eClass, Class<? extends SH> shClass, SH sh, int length,
+//                       ShuffleWrapIndexesOperation<SH> defaultImpl)
+bool LibraryCallKit::inline_vector_wrap_shuffle_indexes() {
+  const TypeInstPtr* elem_klass    = gvn().type(argument(0))->isa_instptr();
+  const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
+  Node*              shuffle       = argument(2);
+  const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
+
+  if (elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr ||
+      !vlen->is_con() || shuffle_klass->const_oop() == nullptr) {
+    // not enough info for intrinsification
+    return false;
+  }
+
+  if (!is_klass_initialized(shuffle_klass)) {
+    log_if_needed("  ** klass argument not initialized");
+    return false;
+  }
+
+  int num_elem = vlen->get_con();
+  if ((num_elem < 4) || !is_power_of_2(num_elem)) {
+    log_if_needed("  ** vlen < 4 or not power of two=%d", num_elem);
+    return false;
+  }
+
+  // Shuffles use byte array based backing storage
+  BasicType shuffle_bt = T_BYTE;
+  if (!arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
+      !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
+    log_if_needed("  ** not supported: op=wrapShuffleIndexes vlen=%d etype=%s",
+                  num_elem, type2name(shuffle_bt));
+    return false;
+  }
+
+  ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
+  const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
+
+  // Unbox shuffle with true flag to indicate its load shuffle to vector
+  // shuffle is a byte array
+  Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, shuffle_bt, num_elem, true);
+
+  const TypeVect* vt  = TypeVect::make(shuffle_bt, num_elem);
+  const Type* shuffle_type_bt = Type::get_const_basic_type(shuffle_bt);
+  Node* mod_mask = gvn().makecon(TypeInt::make(num_elem-1));
+  Node* bcast_mod_mask  = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, shuffle_type_bt));
+  // Wrap the indices greater than lane count.
+  Node* res = gvn().transform(VectorNode::make(Op_AndV, shuffle_vec, bcast_mod_mask, vt));
+
+  // Wrap it up in VectorBox to keep object type information.
+  res = box_vector(res, shuffle_box_type, shuffle_bt, num_elem);
+  set_result(res);
+  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(shuffle_bt))));
+  return true;
+}
+
 // public static
 // <M,
 //  S extends VectorSpecies<E>,
@@ -2044,6 +2102,150 @@ static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr,
   return addr;
 }
 
+//    public static
+//    <V extends Vector<E>,
+//     M  extends VectorMask<E>,
+//     E>
+//    V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
+//                   int length, V v1, V v2, M m,
+//                   VectorSelectFromOp<V, M> defaultImpl)
+bool LibraryCallKit::inline_vector_select_from() {
+  const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
+  const TypeInstPtr* mask_klass    = gvn().type(argument(1))->isa_instptr();
+  const TypeInstPtr* elem_klass    = gvn().type(argument(2))->isa_instptr();
+  const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
+
+  if (vector_klass == nullptr  || elem_klass == nullptr || vlen == nullptr ||
+      vector_klass->const_oop()  == nullptr ||
+      elem_klass->const_oop()    == nullptr ||
+      !vlen->is_con()) {
+    log_if_needed("  ** missing constant: vclass=%s etype=%s vlen=%s",
+                    NodeClassNames[argument(0)->Opcode()],
+                    NodeClassNames[argument(2)->Opcode()],
+                    NodeClassNames[argument(3)->Opcode()]);
+    return false; // not enough info for intrinsification
+  }
+  if (!is_klass_initialized(vector_klass)) {
+    log_if_needed("  ** klass argument not initialized");
+    return false;
+  }
+  ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
+  if (!elem_type->is_primitive_type()) {
+    log_if_needed("  ** not a primitive bt=%d", elem_type->basic_type());
+    return false; // should be primitive type
+  }
+  BasicType elem_bt = elem_type->basic_type();
+  int num_elem = vlen->get_con();
+  if (!is_power_of_2(num_elem)) {
+    log_if_needed("  ** vlen not power of two=%d", num_elem);
+    return false;
+  }
+
+  int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
+  if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)||
+      !arch_supports_vector(Op_AndV, num_elem, T_BYTE, VecMaskNotUsed)              ||
+      !arch_supports_vector(Op_Replicate, num_elem, T_BYTE, VecMaskNotUsed)         ||
+      !arch_supports_vector(cast_vopc, num_elem, T_BYTE, VecMaskNotUsed)) {
+    log_if_needed("  ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
+                    num_elem, type2name(elem_bt));
+    return false; // not supported
+  }
+
+  bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
+  bool use_predicate = is_masked_op;
+  if (is_masked_op &&
+      (mask_klass == nullptr ||
+       mask_klass->const_oop() == nullptr ||
+       !is_klass_initialized(mask_klass))) {
+    log_if_needed("  ** mask_klass argument not initialized");
+    return false; // not supported
+  }
+  VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
+  if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
+    use_predicate = false;
+    if(!is_masked_op ||
+       (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
+        !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)     ||
+        !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
+      log_if_needed("  ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
+                      num_elem, type2name(elem_bt), is_masked_op);
+      return false; // not supported
+    }
+  }
+  ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
+  const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
+
+  // v1 is the index vector
+  Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
+  // v2 is the vector being rearranged
+  Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
+
+  if (v1 == nullptr) {
+    log_if_needed("  ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
+    return false; // operand unboxing failed
+  }
+
+  if (v2 == nullptr) {
+    log_if_needed("  ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
+    return false; // operand unboxing failed
+  }
+
+  Node* mask = nullptr;
+  if (is_masked_op) {
+    ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
+    const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
+    mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
+    if (mask == nullptr) {
+      log_if_needed("  ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
+      return false;
+    }
+  }
+
+  // cast index vector from elem_bt vector to byte vector
+  const Type * byte_bt = Type::get_const_basic_type(T_BYTE);
+  const TypeVect * byte_vt  = TypeVect::make(byte_bt, num_elem);
+  Node* byte_shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, T_BYTE, num_elem));
+
+  // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
+  // this is a simple AND operation as we come here only for power of two vector length
+  Node* mod_val = gvn().makecon(TypeInt::make(num_elem-1));
+  Node* bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, byte_bt));
+  byte_shuffle = gvn().transform(VectorNode::make(Op_AndV, byte_shuffle, bcast_mod, byte_vt));
+
+  // load the shuffle to use in rearrange
+  const TypeVect * shuffle_vt  = TypeVect::make(elem_bt, num_elem);
+  Node* load_shuffle = gvn().transform(new VectorLoadShuffleNode(byte_shuffle, shuffle_vt));
+
+  // and finally rearrange
+  Node* rearrange = new VectorRearrangeNode(v2, load_shuffle);
+  if (is_masked_op) {
+    if (use_predicate) {
+      // masked rearrange is supported so use that directly
+      rearrange->add_req(mask);
+      rearrange->add_flag(Node::Flag_is_predicated_vector);
+    } else {
+      // masked rearrange is not supported so emulate usig blend
+      const TypeVect* vt = v1->bottom_type()->is_vect();
+      rearrange = gvn().transform(rearrange);
+
+      // create a zero vector with each lane element set as zero
+      Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
+      Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
+
+      // For each lane for which mask is set, blend in the rearranged lane into zero vector
+      rearrange = new VectorBlendNode(zerovec, rearrange, mask);
+    }
+  }
+  rearrange = gvn().transform(rearrange);
+
+  // box the result
+  Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
+  set_result(box);
+
+  C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
+  return true;
+}
+
 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
   assert(UseVectorStubs, "sanity");
   assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");

src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java

@@ -263,6 +263,20 @@ V shuffleToVector(Class<? extends Vector<E>> vClass, Class<E> eClass, Class<? ex
       return defaultImpl.apply(sh);
     }
 
+    public interface WrapShuffleIndexesOperation<SH extends VectorShuffle<?>> {
+        SH apply(SH sh);
+    }
+
+    @IntrinsicCandidate
+    public static
+    <E,
+     SH extends VectorShuffle<E>>
+    SH wrapShuffleIndexes(Class<E> eClass, Class<? extends SH> shClass, SH sh, int length,
+                          WrapShuffleIndexesOperation<SH> defaultImpl) {
+       assert isNonCapturingLambda(defaultImpl) : defaultImpl;
+       return defaultImpl.apply(sh);
+    }
+
     /* ============================================================================ */
     public interface IndexOperation<V extends Vector<?>,
                                     S extends VectorSpecies<?>> {
@@ -605,6 +619,23 @@ V rearrangeOp(Class<? extends V> vClass, Class<SH> shClass, Class<M> mClass, Cla
         return defaultImpl.apply(v, sh, m);
     }
 
+    public interface VectorSelectFromOp<V extends Vector<?>,
+                                        M extends VectorMask<?>> {
+        V apply(V v1, V v2, M m);
+    }
+
+    @IntrinsicCandidate
+    public static
+    <V extends Vector<E>,
+     M  extends VectorMask<E>,
+     E>
+    V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
+                   int length, V v1, V v2, M m,
+                   VectorSelectFromOp<V, M> defaultImpl) {
+        assert isNonCapturingLambda(defaultImpl) : defaultImpl;
+        return defaultImpl.apply(v1, v2, m);
+    }
+
     /* ============================================================================ */
 
     public interface VectorBlendOp<V extends Vector<?>,

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java

@@ -133,7 +133,7 @@ public final VectorShuffle<E> checkIndexes() {
     }
 
     @ForceInline
-    public final VectorShuffle<E> wrapIndexes() {
+    public final VectorShuffle<E> wrapIndexesTemplate() {
         Vector<E> shufvec = this.toVector();
         VectorMask<E> vecmask = shufvec.compare(VectorOperators.LT, vspecies().zero());
         if (vecmask.anyTrue()) {