@@ -757,6 +757,64 @@ bool LibraryCallKit::inline_vector_shuffle_to_vector() {
757757 return true ;
758758}
759759
760+ // public static
761+ // <E,
762+ // SH extends VectorShuffle<E>>
763+ // SH wrapShuffleIndexes(Class<E> eClass, Class<? extends SH> shClass, SH sh, int length,
764+ // ShuffleWrapIndexesOperation<SH> defaultImpl)
765+ bool LibraryCallKit::inline_vector_wrap_shuffle_indexes () {
766+ const TypeInstPtr* elem_klass = gvn ().type (argument (0 ))->isa_instptr ();
767+ const TypeInstPtr* shuffle_klass = gvn ().type (argument (1 ))->isa_instptr ();
768+ Node* shuffle = argument (2 );
769+ const TypeInt* vlen = gvn ().type (argument (3 ))->isa_int ();
770+
771+ if (elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top () || vlen == nullptr ||
772+ !vlen->is_con () || shuffle_klass->const_oop () == nullptr ) {
773+ // not enough info for intrinsification
774+ return false ;
775+ }
776+
777+ if (!is_klass_initialized (shuffle_klass)) {
778+ log_if_needed (" ** klass argument not initialized"
779+ return false ;
780+ }
781+
782+ int num_elem = vlen->get_con ();
783+ if ((num_elem < 4 ) || !is_power_of_2 (num_elem)) {
784+ log_if_needed (" ** vlen < 4 or not power of two=%d"
785+ return false ;
786+ }
787+
788+ // Shuffles use byte array based backing storage
789+ BasicType shuffle_bt = T_BYTE;
790+ if (!arch_supports_vector (Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
791+ !arch_supports_vector (Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
792+ log_if_needed (" ** not supported: op=wrapShuffleIndexes vlen=%d etype=%s"
793+ num_elem, type2name (shuffle_bt));
794+ return false ;
795+ }
796+
797+ ciKlass* sbox_klass = shuffle_klass->const_oop ()->as_instance ()->java_lang_Class_klass ();
798+ const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact (TypePtr::NotNull, sbox_klass);
799+
800+ // Unbox shuffle with true flag to indicate its load shuffle to vector
801+ // shuffle is a byte array
802+ Node* shuffle_vec = unbox_vector (shuffle, shuffle_box_type, shuffle_bt, num_elem, true );
803+
804+ const TypeVect* vt = TypeVect::make (shuffle_bt, num_elem);
805+ const Type* shuffle_type_bt = Type::get_const_basic_type (shuffle_bt);
806+ Node* mod_mask = gvn ().makecon (TypeInt::make (num_elem-1 ));
807+ Node* bcast_mod_mask = gvn ().transform (VectorNode::scalar2vector (mod_mask, num_elem, shuffle_type_bt));
808+ // Wrap the indices greater than lane count.
809+ Node* res = gvn ().transform (VectorNode::make (Op_AndV, shuffle_vec, bcast_mod_mask, vt));
810+
811+ // Wrap it up in VectorBox to keep object type information.
812+ res = box_vector (res, shuffle_box_type, shuffle_bt, num_elem);
813+ set_result (res);
814+ C->set_max_vector_size (MAX2 (C->max_vector_size (), (uint)(num_elem * type2aelembytes (shuffle_bt))));
815+ return true ;
816+ }
817+
760818// public static
761819// <M,
762820// S extends VectorSpecies<E>,
@@ -2044,6 +2102,150 @@ static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr,
20442102 return addr;
20452103}
20462104
2105+ // public static
2106+ // <V extends Vector<E>,
2107+ // M extends VectorMask<E>,
2108+ // E>
2109+ // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
2110+ // int length, V v1, V v2, M m,
2111+ // VectorSelectFromOp<V, M> defaultImpl)
2112+ bool LibraryCallKit::inline_vector_select_from () {
2113+ const TypeInstPtr* vector_klass = gvn ().type (argument (0 ))->isa_instptr ();
2114+ const TypeInstPtr* mask_klass = gvn ().type (argument (1 ))->isa_instptr ();
2115+ const TypeInstPtr* elem_klass = gvn ().type (argument (2 ))->isa_instptr ();
2116+ const TypeInt* vlen = gvn ().type (argument (3 ))->isa_int ();
2117+
2118+ if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2119+ vector_klass->const_oop () == nullptr ||
2120+ elem_klass->const_oop () == nullptr ||
2121+ !vlen->is_con ()) {
2122+ log_if_needed (" ** missing constant: vclass=%s etype=%s vlen=%s"
2123+ NodeClassNames[argument (0 )->Opcode ()],
2124+ NodeClassNames[argument (2 )->Opcode ()],
2125+ NodeClassNames[argument (3 )->Opcode ()]);
2126+ return false ; // not enough info for intrinsification
2127+ }
2128+ if (!is_klass_initialized (vector_klass)) {
2129+ log_if_needed (" ** klass argument not initialized"
2130+ return false ;
2131+ }
2132+ ciType* elem_type = elem_klass->const_oop ()->as_instance ()->java_mirror_type ();
2133+ if (!elem_type->is_primitive_type ()) {
2134+ log_if_needed (" ** not a primitive bt=%d" basic_type ());
2135+ return false ; // should be primitive type
2136+ }
2137+ BasicType elem_bt = elem_type->basic_type ();
2138+ int num_elem = vlen->get_con ();
2139+ if (!is_power_of_2 (num_elem)) {
2140+ log_if_needed (" ** vlen not power of two=%d"
2141+ return false ;
2142+ }
2143+
2144+ int cast_vopc = VectorCastNode::opcode (-1 , elem_bt); // from vector of type elem_bt
2145+ if (!arch_supports_vector (Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)||
2146+ !arch_supports_vector (Op_AndV, num_elem, T_BYTE, VecMaskNotUsed) ||
2147+ !arch_supports_vector (Op_Replicate, num_elem, T_BYTE, VecMaskNotUsed) ||
2148+ !arch_supports_vector (cast_vopc, num_elem, T_BYTE, VecMaskNotUsed)) {
2149+ log_if_needed (" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no"
2150+ num_elem, type2name (elem_bt));
2151+ return false ; // not supported
2152+ }
2153+
2154+ bool is_masked_op = argument (6 )->bottom_type () != TypePtr::NULL_PTR;
2155+ bool use_predicate = is_masked_op;
2156+ if (is_masked_op &&
2157+ (mask_klass == nullptr ||
2158+ mask_klass->const_oop () == nullptr ||
2159+ !is_klass_initialized (mask_klass))) {
2160+ log_if_needed (" ** mask_klass argument not initialized"
2161+ return false ; // not supported
2162+ }
2163+ VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2164+ if (!arch_supports_vector (Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2165+ use_predicate = false ;
2166+ if (!is_masked_op ||
2167+ (!arch_supports_vector (Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2168+ !arch_supports_vector (Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2169+ !arch_supports_vector (Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
2170+ log_if_needed (" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d"
2171+ num_elem, type2name (elem_bt), is_masked_op);
2172+ return false ; // not supported
2173+ }
2174+ }
2175+ ciKlass* vbox_klass = vector_klass->const_oop ()->as_instance ()->java_lang_Class_klass ();
2176+ const TypeInstPtr* vbox_type = TypeInstPtr::make_exact (TypePtr::NotNull, vbox_klass);
2177+
2178+ // v1 is the index vector
2179+ Node* v1 = unbox_vector (argument (4 ), vbox_type, elem_bt, num_elem);
2180+ // v2 is the vector being rearranged
2181+ Node* v2 = unbox_vector (argument (5 ), vbox_type, elem_bt, num_elem);
2182+
2183+ if (v1 == nullptr ) {
2184+ log_if_needed (" ** unbox failed v1=%s" argument (4 )->Opcode ()]);
2185+ return false ; // operand unboxing failed
2186+ }
2187+
2188+ if (v2 == nullptr ) {
2189+ log_if_needed (" ** unbox failed v2=%s" argument (5 )->Opcode ()]);
2190+ return false ; // operand unboxing failed
2191+ }
2192+
2193+ Node* mask = nullptr ;
2194+ if (is_masked_op) {
2195+ ciKlass* mbox_klass = mask_klass->const_oop ()->as_instance ()->java_lang_Class_klass ();
2196+ const TypeInstPtr* mbox_type = TypeInstPtr::make_exact (TypePtr::NotNull, mbox_klass);
2197+ mask = unbox_vector (argument (6 ), mbox_type, elem_bt, num_elem);
2198+ if (mask == nullptr ) {
2199+ log_if_needed (" ** unbox failed mask=%s" argument (6 )->Opcode ()]);
2200+ return false ;
2201+ }
2202+ }
2203+
2204+ // cast index vector from elem_bt vector to byte vector
2205+ const Type * byte_bt = Type::get_const_basic_type (T_BYTE);
2206+ const TypeVect * byte_vt = TypeVect::make (byte_bt, num_elem);
2207+ Node* byte_shuffle = gvn ().transform (VectorCastNode::make (cast_vopc, v1, T_BYTE, num_elem));
2208+
2209+ // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2210+ // this is a simple AND operation as we come here only for power of two vector length
2211+ Node* mod_val = gvn ().makecon (TypeInt::make (num_elem-1 ));
2212+ Node* bcast_mod = gvn ().transform (VectorNode::scalar2vector (mod_val, num_elem, byte_bt));
2213+ byte_shuffle = gvn ().transform (VectorNode::make (Op_AndV, byte_shuffle, bcast_mod, byte_vt));
2214+
2215+ // load the shuffle to use in rearrange
2216+ const TypeVect * shuffle_vt = TypeVect::make (elem_bt, num_elem);
2217+ Node* load_shuffle = gvn ().transform (new VectorLoadShuffleNode (byte_shuffle, shuffle_vt));
2218+
2219+ // and finally rearrange
2220+ Node* rearrange = new VectorRearrangeNode (v2, load_shuffle);
2221+ if (is_masked_op) {
2222+ if (use_predicate) {
2223+ // masked rearrange is supported so use that directly
2224+ rearrange->add_req (mask);
2225+ rearrange->add_flag (Node::Flag_is_predicated_vector);
2226+ } else {
2227+ // masked rearrange is not supported so emulate usig blend
2228+ const TypeVect* vt = v1->bottom_type ()->is_vect ();
2229+ rearrange = gvn ().transform (rearrange);
2230+
2231+ // create a zero vector with each lane element set as zero
2232+ Node* zero = gvn ().makecon (Type::get_zero_type (elem_bt));
2233+ Node* zerovec = gvn ().transform (VectorNode::scalar2vector (zero, num_elem, Type::get_const_basic_type (elem_bt)));
2234+
2235+ // For each lane for which mask is set, blend in the rearranged lane into zero vector
2236+ rearrange = new VectorBlendNode (zerovec, rearrange, mask);
2237+ }
2238+ }
2239+ rearrange = gvn ().transform (rearrange);
2240+
2241+ // box the result
2242+ Node* box = box_vector (rearrange, vbox_type, elem_bt, num_elem);
2243+ set_result (box);
2244+
2245+ C->set_max_vector_size (MAX2 (C->max_vector_size (), (uint)(num_elem * type2aelembytes (elem_bt))));
2246+ return true ;
2247+ }
2248+
20472249Node* LibraryCallKit::gen_call_to_svml (int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
20482250 assert (UseVectorStubs, " sanity"
20492251 assert (vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, " need valid op id"
0 commit comments