From ec8383f951db26a7ab96b3729d2af03327ec3900 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Sun, 19 Mar 2023 05:23:25 +0800 Subject: [PATCH 01/13] draft --- src/hotspot/cpu/x86/x86.ad | 74 ++---- src/hotspot/share/classfile/vmIntrinsics.hpp | 9 - src/hotspot/share/opto/c2compiler.cpp | 1 - src/hotspot/share/opto/library_call.cpp | 2 - src/hotspot/share/opto/library_call.hpp | 1 - src/hotspot/share/opto/matcher.hpp | 2 + src/hotspot/share/opto/vector.cpp | 10 - src/hotspot/share/opto/vectorIntrinsics.cpp | 82 +------ src/hotspot/share/opto/vectornode.cpp | 10 +- src/hotspot/share/opto/vectornode.hpp | 5 +- src/hotspot/share/prims/vectorSupport.cpp | 8 +- .../jdk/internal/vm/vector/VectorSupport.java | 17 -- .../jdk/incubator/vector/AbstractShuffle.java | 159 +++---------- .../jdk/incubator/vector/AbstractVector.java | 3 - .../jdk/incubator/vector/Byte128Vector.java | 157 +++++++++--- .../jdk/incubator/vector/Byte256Vector.java | 157 +++++++++--- .../jdk/incubator/vector/Byte512Vector.java | 157 +++++++++--- .../jdk/incubator/vector/Byte64Vector.java | 157 +++++++++--- .../jdk/incubator/vector/ByteMaxVector.java | 157 +++++++++--- .../jdk/incubator/vector/ByteVector.java | 10 +- .../jdk/incubator/vector/Double128Vector.java | 158 ++++++++---- .../jdk/incubator/vector/Double256Vector.java | 158 ++++++++---- .../jdk/incubator/vector/Double512Vector.java | 158 ++++++++---- .../jdk/incubator/vector/Double64Vector.java | 152 ++++++++---- .../jdk/incubator/vector/DoubleMaxVector.java | 158 ++++++++---- .../jdk/incubator/vector/DoubleVector.java | 25 +- .../jdk/incubator/vector/Float128Vector.java | 152 ++++++++---- .../jdk/incubator/vector/Float256Vector.java | 152 ++++++++---- .../jdk/incubator/vector/Float512Vector.java | 152 ++++++++---- .../jdk/incubator/vector/Float64Vector.java | 152 ++++++++---- .../jdk/incubator/vector/FloatMaxVector.java | 152 ++++++++---- .../jdk/incubator/vector/FloatVector.java | 25 +- .../jdk/incubator/vector/Int128Vector.java | 153 ++++++++---- .../jdk/incubator/vector/Int256Vector.java | 153 ++++++++---- .../jdk/incubator/vector/Int512Vector.java | 153 ++++++++---- .../jdk/incubator/vector/Int64Vector.java | 153 ++++++++---- .../jdk/incubator/vector/IntMaxVector.java | 153 ++++++++---- .../jdk/incubator/vector/IntVector.java | 34 ++- .../jdk/incubator/vector/Long128Vector.java | 159 +++++++++---- .../jdk/incubator/vector/Long256Vector.java | 159 +++++++++---- .../jdk/incubator/vector/Long512Vector.java | 159 +++++++++---- .../jdk/incubator/vector/Long64Vector.java | 153 ++++++++---- .../jdk/incubator/vector/LongMaxVector.java | 159 +++++++++---- .../jdk/incubator/vector/LongVector.java | 34 ++- .../jdk/incubator/vector/Short128Vector.java | 155 ++++++++---- .../jdk/incubator/vector/Short256Vector.java | 155 ++++++++---- .../jdk/incubator/vector/Short512Vector.java | 155 ++++++++---- .../jdk/incubator/vector/Short64Vector.java | 155 ++++++++---- .../jdk/incubator/vector/ShortMaxVector.java | 155 ++++++++---- .../jdk/incubator/vector/ShortVector.java | 12 +- .../jdk/incubator/vector/VectorShape.java | 1 + .../jdk/incubator/vector/VectorShuffle.java | 6 +- .../incubator/vector/X-Vector.java.template | 38 ++- .../vector/X-VectorBits.java.template | 225 ++++++++++++++---- 54 files changed, 3798 insertions(+), 1653 deletions(-) diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 80f0c0f4b4927..35fad2cd9659b 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2171,6 +2171,18 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } +// Do Vector::rearrange needs preparation of the shuffle argument +const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + if (elem_bt == T_BYTE || elem_bt == T_INT || elem_bt == T_FLOAT) { + return false; + } else if (elem_bt == T_SHORT) { + return !VM_Version::supports_avx512bw(); + } + + // T_LONG || T_DOUBLE + return !VM_Version::supports_evex(); +} + MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) { assert(Matcher::is_generic_vector(generic_opnd), "not generic"); bool legacy = (generic_opnd->opcode() == LEGVEC); @@ -8400,17 +8412,6 @@ instruct VectorPopulateLIndex(vec dst, rRegL src1, immI_1 src2, vec vtmp) %{ //-------------------------------- Rearrange ---------------------------------- // LoadShuffle/Rearrange for Byte - -instruct loadShuffleB(vec dst) %{ - predicate(Matcher::vector_element_basic_type(n) == T_BYTE); - match(Set dst (VectorLoadShuffle dst)); - format %{ "vector_load_shuffle $dst, $dst" %} - ins_encode %{ - // empty - %} - ins_pipe( pipe_slow ); -%} - instruct rearrangeB(vec dst, vec shuffle) %{ predicate(Matcher::vector_element_basic_type(n) == T_BYTE && Matcher::vector_length(n) < 32); @@ -8488,7 +8489,7 @@ instruct loadShuffleS(vec dst, vec src, vec vtmp) %{ if (UseAVX == 0) { assert(vlen_in_bytes <= 16, "required"); // Multiply each shuffle by two to get byte index - __ pmovzxbw($vtmp$$XMMRegister, $src$$XMMRegister); + __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister); __ psllw($vtmp$$XMMRegister, 1); // Duplicate to create 2 copies of byte index @@ -8503,8 +8504,7 @@ instruct loadShuffleS(vec dst, vec src, vec vtmp) %{ assert(UseAVX > 1 || vlen_in_bytes <= 16, "required"); int vlen_enc = vector_length_encoding(this); // Multiply each shuffle by two to get byte index - __ vpmovzxbw($vtmp$$XMMRegister, $src$$XMMRegister, vlen_enc); - __ vpsllw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 1, vlen_enc); + __ vpsllw($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc); // Duplicate to create 2 copies of byte index __ vpsllw($dst$$XMMRegister, $vtmp$$XMMRegister, 8, vlen_enc); @@ -8551,21 +8551,6 @@ instruct rearrangeS_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVe ins_pipe( pipe_slow ); %} -instruct loadShuffleS_evex(vec dst, vec src) %{ - predicate(Matcher::vector_element_basic_type(n) == T_SHORT && - VM_Version::supports_avx512bw()); - match(Set dst (VectorLoadShuffle src)); - format %{ "vector_load_shuffle $dst, $src" %} - ins_encode %{ - int vlen_enc = vector_length_encoding(this); - if (!VM_Version::supports_avx512vl()) { - vlen_enc = Assembler::AVX_512bit; - } - __ vpmovzxbw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); - %} - ins_pipe( pipe_slow ); -%} - instruct rearrangeS_evex(vec dst, vec src, vec shuffle) %{ predicate(Matcher::vector_element_basic_type(n) == T_SHORT && VM_Version::supports_avx512bw()); @@ -8596,7 +8581,7 @@ instruct loadShuffleI(vec dst, vec src, vec vtmp) %{ // only byte shuffle instruction available on these platforms // Duplicate and multiply each shuffle by 4 - __ pmovzxbd($vtmp$$XMMRegister, $src$$XMMRegister); + __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister); __ pshuflw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0); __ pshufhw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0); __ psllw($vtmp$$XMMRegister, 2); @@ -8625,18 +8610,6 @@ instruct rearrangeI(vec dst, vec shuffle) %{ ins_pipe( pipe_slow ); %} -instruct loadShuffleI_avx(vec dst, vec src) %{ - predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) && - UseAVX >= 2); - match(Set dst (VectorLoadShuffle src)); - format %{ "vector_load_shuffle $dst, $src" %} - ins_encode %{ - int vlen_enc = vector_length_encoding(this); - __ vpmovzxbd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); - %} - ins_pipe( pipe_slow ); -%} - instruct rearrangeI_avx(vec dst, vec src, vec shuffle) %{ predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) && UseAVX >= 2); @@ -8668,8 +8641,7 @@ instruct loadShuffleL(vec dst, vec src, vec vtmp) %{ // only double word shuffle instruction available on these platforms // Multiply each shuffle by two to get double word index - __ vpmovzxbq($vtmp$$XMMRegister, $src$$XMMRegister, vlen_enc); - __ vpsllq($vtmp$$XMMRegister, $vtmp$$XMMRegister, 1, vlen_enc); + __ vpsllq($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc); // Duplicate each double word shuffle __ vpsllq($dst$$XMMRegister, $vtmp$$XMMRegister, 32, vlen_enc); @@ -8695,20 +8667,6 @@ instruct rearrangeL(vec dst, vec src, vec shuffle) %{ ins_pipe( pipe_slow ); %} -instruct loadShuffleL_evex(vec dst, vec src) %{ - predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE - (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl())); - match(Set dst (VectorLoadShuffle src)); - format %{ "vector_load_shuffle $dst, $src" %} - ins_encode %{ - assert(UseAVX > 2, "required"); - - int vlen_enc = vector_length_encoding(this); - __ vpmovzxbq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); - %} - ins_pipe( pipe_slow ); -%} - instruct rearrangeL_evex(vec dst, vec src, vec shuffle) %{ predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl())); diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 6d1ad75ae688c..50ce3493e5acb 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -967,15 +967,6 @@ class methodHandle; "Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;") \ do_name(vector_shuffle_step_iota_name, "shuffleIota") \ \ - do_intrinsic(_VectorShuffleToVector, jdk_internal_vm_vector_VectorSupport, vector_shuffle_to_vector_name, vector_shuffle_to_vector_sig, F_S) \ - do_signature(vector_shuffle_to_vector_sig, "(Ljava/lang/Class;" \ - "Ljava/lang/Class;" \ - "Ljava/lang/Class;" \ - "Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;" \ - "ILjdk/internal/vm/vector/VectorSupport$ShuffleToVectorOperation;)" \ - "Ljdk/internal/vm/vector/VectorSupport$Vector;") \ - do_name(vector_shuffle_to_vector_name, "shuffleToVector") \ - \ do_intrinsic(_VectorLoadOp, jdk_internal_vm_vector_VectorSupport, vector_load_op_name, vector_load_op_sig, F_S) \ do_signature(vector_load_op_sig, "(Ljava/lang/Class;" \ "Ljava/lang/Class;" \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index a6fa097bdee97..a655b89eeb431 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -764,7 +764,6 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt case vmIntrinsics::_VectorTernaryOp: case vmIntrinsics::_VectorFromBitsCoerced: case vmIntrinsics::_VectorShuffleIota: - case vmIntrinsics::_VectorShuffleToVector: case vmIntrinsics::_VectorLoadOp: case vmIntrinsics::_VectorLoadMaskedOp: case vmIntrinsics::_VectorStoreOp: diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index e82ab937ba775..2098107807418 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -689,8 +689,6 @@ bool LibraryCallKit::try_to_inline(int predicate) { return inline_vector_shuffle_iota(); case vmIntrinsics::_VectorMaskOp: return inline_vector_mask_operation(); - case vmIntrinsics::_VectorShuffleToVector: - return inline_vector_shuffle_to_vector(); case vmIntrinsics::_VectorLoadOp: return inline_vector_mem_operation(/*is_store=*/false); case vmIntrinsics::_VectorLoadMaskedOp: diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 97feb431dbf4f..e43b29ec5e2ab 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -340,7 +340,6 @@ class LibraryCallKit : public GraphKit { // Vector API support bool inline_vector_nary_operation(int n); bool inline_vector_frombits_coerced(); - bool inline_vector_shuffle_to_vector(); bool inline_vector_shuffle_iota(); bool inline_vector_mask_operation(); bool inline_vector_mem_operation(bool is_store); diff --git a/src/hotspot/share/opto/matcher.hpp b/src/hotspot/share/opto/matcher.hpp index 2de901c61352b..1f57cea3201de 100644 --- a/src/hotspot/share/opto/matcher.hpp +++ b/src/hotspot/share/opto/matcher.hpp @@ -338,6 +338,8 @@ class Matcher : public PhaseTransform { static const bool vector_needs_partial_operations(Node* node, const TypeVect* vt); + static const bool vector_needs_load_shuffle(BasicType elem_bt, int vlen); + static const RegMask* predicate_reg_mask(void); static const TypeVectMask* predicate_reg_type(const Type* elemTy, int length); diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index 12b5060b60c9b..a4c916289a102 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -36,11 +36,6 @@ static bool is_vector_mask(ciKlass* klass) { return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); } -static bool is_vector_shuffle(ciKlass* klass) { - return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); -} - - void PhaseVector::optimize_vector_boxes() { Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); @@ -438,8 +433,6 @@ void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) { if (is_vector_mask(from_kls)) { bt = T_BOOLEAN; - } else if (is_vector_shuffle(from_kls)) { - bt = T_BYTE; } ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(), @@ -484,9 +477,6 @@ void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) { if (is_vector_mask(from_kls)) { vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem))); - } else if (is_vector_shuffle(from_kls) && !vec_unbox->is_shuffle_to_vector()) { - assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect shuffle type consistency"); - vec_val_load = gvn.transform(new VectorLoadShuffleNode(vec_val_load, TypeVect::make(masktype, num_elem))); } gvn.hash_delete(vec_unbox); diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index 717b426402074..2683fbe8ca9cf 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -63,10 +63,6 @@ static bool is_vector_mask(ciKlass* klass) { return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); } -static bool is_vector_shuffle(ciKlass* klass) { - return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); -} - bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt, VectorMaskUseType mask_use_type, bool has_scalar_args) { bool is_supported = true; @@ -739,71 +735,6 @@ bool LibraryCallKit::inline_vector_mask_operation() { return true; } -// public static -// , -// E> -// V shuffleToVector(Class> vclass, Class elementType, -// Class shuffleClass, Sh s, int length, -// ShuffleToVectorOperation defaultImpl) -bool LibraryCallKit::inline_vector_shuffle_to_vector() { - const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr(); - const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr(); - const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr(); - Node* shuffle = argument(3); - const TypeInt* vlen = gvn().type(argument(4))->isa_int(); - - if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) { - return false; // dead code - } - if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) { - return false; // not enough info for intrinsification - } - if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) { - if (C->print_intrinsics()) { - tty->print_cr(" ** klass argument not initialized"); - } - return false; - } - - int num_elem = vlen->get_con(); - ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); - BasicType elem_bt = elem_type->basic_type(); - - if (num_elem < 4) { - return false; - } - - int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE - // Make sure that cast is implemented to particular type/size combination. - if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) { - if (C->print_intrinsics()) { - tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s", - cast_vopc, num_elem, type2name(elem_bt)); - } - return false; - } - - ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); - const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); - - // Unbox shuffle with true flag to indicate its load shuffle to vector - // shuffle is a byte array - Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true); - - // cast byte to target element type - shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem)); - - ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass(); - const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass); - - // Box vector - Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem); - set_result(res); - C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); - return true; -} - // public static // , @@ -2061,8 +1992,9 @@ bool LibraryCallKit::inline_vector_rearrange() { BasicType elem_bt = elem_type->basic_type(); BasicType shuffle_bt = elem_bt; int num_elem = vlen->get_con(); + bool need_load_shuffle = Matcher::vector_needs_load_shuffle(shuffle_bt, num_elem); - if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) { + if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) { if (C->print_intrinsics()) { tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no", num_elem, type2name(elem_bt)); @@ -2102,6 +2034,7 @@ bool LibraryCallKit::inline_vector_rearrange() { Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem); + const TypeVect* vt = TypeVect::make(elem_bt, num_elem); if (v1 == NULL || shuffle == NULL) { return false; // operand unboxing failed @@ -2121,13 +2054,16 @@ bool LibraryCallKit::inline_vector_rearrange() { } } + if (need_load_shuffle) { + shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, vt)); + } + Node* rearrange = new VectorRearrangeNode(v1, shuffle); if (is_masked_op) { if (use_predicate) { rearrange->add_req(mask); rearrange->add_flag(Node::Flag_is_predicated_vector); } else { - const TypeVect* vt = v1->bottom_type()->is_vect(); rearrange = gvn().transform(rearrange); Node* zero = gvn().makecon(Type::get_zero_type(elem_bt)); Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt))); @@ -2421,9 +2357,7 @@ bool LibraryCallKit::inline_vector_convert() { ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass(); ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass(); - if (is_vector_shuffle(vbox_klass_from)) { - return false; // vector shuffles aren't supported - } + bool is_mask = is_vector_mask(vbox_klass_from); ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type(); diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 81b20c63ab6c6..2d254261ac2b7 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -1643,19 +1643,13 @@ Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (in_vt->length() == out_vt->length()) { Node* value = vbox->in(VectorBoxNode::Value); - bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); - bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); + bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); if (is_vector_mask) { // VectorUnbox (VectorBox vmask) ==> VectorMaskCast vmask const TypeVect* vmask_type = TypeVect::makemask(out_vt->element_basic_type(), out_vt->length()); return new VectorMaskCastNode(value, vmask_type); - } else if (is_vector_shuffle) { - if (!is_shuffle_to_vector()) { - // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle - return new VectorLoadShuffleNode(value, out_vt); - } } else { - // Vector type mismatch is only supported for masks and shuffles, but sometimes it happens in pathological cases. + // Vector type mismatch is only supported for masks, but sometimes it happens in pathological cases. } } else { // Vector length mismatch. diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index d92b0c9c33132..d3e5cc64578ab 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1472,11 +1472,8 @@ class VectorRearrangeNode : public VectorNode { class VectorLoadShuffleNode : public VectorNode { public: VectorLoadShuffleNode(Node* in, const TypeVect* vt) - : VectorNode(in, vt) { - assert(in->bottom_type()->is_vect()->element_basic_type() == T_BYTE, "must be BYTE"); - } + : VectorNode(in, vt) {} - int GetOutShuffleSize() const { return type2aelembytes(vect_type()->element_basic_type()); } virtual int Opcode() const; }; diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp index 4f2e78739ec9c..54fb1cf4d49b8 100644 --- a/src/hotspot/share/prims/vectorSupport.cpp +++ b/src/hotspot/share/prims/vectorSupport.cpp @@ -73,10 +73,6 @@ bool VectorSupport::is_vector_mask(Klass* klass) { return klass->is_subclass_of(vmClasses::vector_VectorMask_klass()); } -bool VectorSupport::is_vector_shuffle(Klass* klass) { - return klass->is_subclass_of(vmClasses::vector_VectorShuffle_klass()); -} - BasicType VectorSupport::klass2bt(InstanceKlass* ik) { assert(ik->is_subclass_of(vmClasses::vector_VectorPayload_klass()), "%s not a VectorPayload", ik->name()->as_C_string()); fieldDescriptor fd; // find_field initializes fd if found @@ -87,9 +83,7 @@ BasicType VectorSupport::klass2bt(InstanceKlass* ik) { assert(fd.is_static(), ""); assert(fd.offset() > 0, ""); - if (is_vector_shuffle(ik)) { - return T_BYTE; - } else if (is_vector_mask(ik)) { + if (is_vector_mask(ik)) { return T_BOOLEAN; } else { // vector and mask oop value = ik->java_mirror()->obj_field(fd.offset()); diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java index a38b72a3b3556..b71faa120e632 100644 --- a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java +++ b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java @@ -246,23 +246,6 @@ SH shuffleIota(Class eClass, Class shClass, S s, return defaultImpl.apply(length, start, step, s); } - public interface ShuffleToVectorOperation, - SH extends VectorShuffle> { - V apply(SH sh); - } - - @IntrinsicCandidate - public static - , - SH extends VectorShuffle, - E> - V shuffleToVector(Class> vClass, Class eClass, Class shClass, SH sh, - int length, - ShuffleToVectorOperation defaultImpl) { - assert isNonCapturingLambda(defaultImpl) : defaultImpl; - return defaultImpl.apply(sh); - } - /* ============================================================================ */ public interface IndexOperation, S extends VectorSpecies> { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java index d648fd86484e0..41f536d39c1bf 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java @@ -24,55 +24,14 @@ */ package jdk.incubator.vector; -import java.util.function.IntUnaryOperator; import jdk.internal.vm.annotation.ForceInline; abstract class AbstractShuffle extends VectorShuffle { - static final IntUnaryOperator IDENTITY = i -> i; - // Internal representation allows for a maximum index of 256 // Values are clipped to [-VLENGTH..VLENGTH-1]. - AbstractShuffle(int length, byte[] reorder) { - super(reorder); - assert(length == reorder.length); - assert(indexesInRange(reorder)); - } - - AbstractShuffle(int length, int[] reorder) { - this(length, reorder, 0); - } - - AbstractShuffle(int length, int[] reorder, int offset) { - super(prepare(length, reorder, offset)); - } - - AbstractShuffle(int length, IntUnaryOperator f) { - super(prepare(length, f)); - } - - private static byte[] prepare(int length, int[] reorder, int offset) { - byte[] a = new byte[length]; - for (int i = 0; i < length; i++) { - int si = reorder[offset + i]; - si = partiallyWrapIndex(si, length); - a[i] = (byte) si; - } - return a; - } - - private static byte[] prepare(int length, IntUnaryOperator f) { - byte[] a = new byte[length]; - for (int i = 0; i < a.length; i++) { - int si = f.applyAsInt(i); - si = partiallyWrapIndex(si, length); - a[i] = (byte) si; - } - return a; - } - - byte[] reorder() { - return (byte[])getPayload(); + AbstractShuffle(Object indices) { + super(indices); } /*package-private*/ @@ -84,89 +43,66 @@ public final VectorSpecies vectorSpecies() { return vspecies(); } + /*package-private*/ + abstract Vector toBitsVector(); + @Override @ForceInline - public void intoArray(int[] a, int offset) { - byte[] reorder = reorder(); - int vlen = reorder.length; - for (int i = 0; i < vlen; i++) { - int sourceIndex = reorder[i]; - assert(sourceIndex >= -vlen && sourceIndex < vlen); - a[offset + i] = sourceIndex; - } + public final Vector toVector() { + return toBitsVector().castShape(vspecies(), 0); } @Override @ForceInline - public int[] toArray() { - byte[] reorder = reorder(); - int[] a = new int[reorder.length]; - intoArray(a, 0); - return a; + public final int[] toArray() { + int[] res = new int[length()]; + intoArray(res, 0); + return res; } - /*package-private*/ + @Override @ForceInline - final - AbstractVector - toVectorTemplate() { - // Note that the values produced by laneSource - // are already clipped. At this point we convert - // them from internal ints (or bytes) into the ETYPE. - // FIXME: Use a conversion intrinsic for this operation. - // https://bugs.openjdk.org/browse/JDK-8225740 - return (AbstractVector) vspecies().fromIntValues(toArray()); + @SuppressWarnings("unchecked") + public final VectorShuffle cast(VectorSpecies s) { + AbstractSpecies species = (AbstractSpecies) s; + if (length() != species.laneCount()) { + throw new IllegalArgumentException("VectorShuffle length and species length differ"); + } + Vector v = toBitsVector(); + if (species.asIntegral() == species) { + return v.castShape(species, 0).toShuffle(); + } else if (species.elementType() == float.class) { + return (VectorShuffle)v.castShape(species.asIntegral(), 0) + .reinterpretAsInts() + .toFPShuffle(); + } else { + return (VectorShuffle)v.castShape(species.asIntegral(), 0) + .reinterpretAsLongs() + .toFPShuffle(); + } } + @Override @ForceInline public final VectorShuffle checkIndexes() { if (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK == 0) { return this; } - Vector shufvec = this.toVector(); - VectorMask vecmask = shufvec.compare(VectorOperators.LT, vspecies().zero()); + Vector shufvec = this.toBitsVector(); + VectorMask vecmask = shufvec.compare(VectorOperators.LT, 0); if (vecmask.anyTrue()) { - byte[] reorder = reorder(); - throw checkIndexFailed(reorder[vecmask.firstTrue()], length()); + int[] indices = toArray(); + throw checkIndexFailed(indices[vecmask.firstTrue()], length()); } return this; } - @ForceInline - public final VectorShuffle wrapIndexes() { - Vector shufvec = this.toVector(); - VectorMask vecmask = shufvec.compare(VectorOperators.LT, vspecies().zero()); - if (vecmask.anyTrue()) { - // FIXME: vectorize this - byte[] reorder = reorder(); - return wrapAndRebuild(reorder); - } - return this; - } - - @ForceInline - public final VectorShuffle wrapAndRebuild(byte[] oldReorder) { - int length = oldReorder.length; - byte[] reorder = new byte[length]; - for (int i = 0; i < length; i++) { - int si = oldReorder[i]; - // FIXME: This does not work unless it's a power of 2. - if ((length & (length - 1)) == 0) { - si += si & length; // power-of-two optimization - } else if (si < 0) { - // non-POT code requires a conditional add - si += length; - } - assert(si >= 0 && si < length); - reorder[i] = (byte) si; - } - return vspecies().dummyVector().shuffleFromBytes(reorder); - } - + @Override @ForceInline public final VectorMask laneIsValid() { - Vector shufvec = this.toVector(); - return shufvec.compare(VectorOperators.GE, vspecies().zero()); + Vector shufvec = this.toVector(); + return shufvec.compare(VectorOperators.GE, 0) + .cast(vspecies()); } @Override @@ -220,21 +156,4 @@ private static IndexOutOfBoundsException checkIndexFailed(int index, int laneCou String msg = "required an index in [0.."+max+"] but found "+index; return new IndexOutOfBoundsException(msg); } - - static boolean indexesInRange(byte[] reorder) { - int length = reorder.length; - for (byte si : reorder) { - if (si >= length || si < -length) { - boolean assertsEnabled = false; - assert(assertsEnabled = true); - if (assertsEnabled) { - String msg = ("index "+si+"out of range ["+length+"] in "+ - java.util.Arrays.toString(reorder)); - throw new AssertionError(msg); - } - return false; - } - } - return true; - } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java index aac198d6b63d6..408c67ec9ec57 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java @@ -192,9 +192,6 @@ ByteVector asByteVectorRawTemplate() { abstract AbstractShuffle iotaShuffle(int start, int step, boolean wrap); - /*do not alias this byte array*/ - abstract AbstractShuffle shuffleFromBytes(byte[] reorder); - abstract AbstractShuffle shuffleFromArray(int[] indexes, int i); abstract AbstractShuffle shuffleFromOp(IntUnaryOperator fn); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java index 12a6c80f21e5b..06d8bfb5d8bad 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java @@ -143,22 +143,28 @@ Byte128Mask maskFromArray(boolean[] bits) { @ForceInline Byte128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Byte128Shuffle)VectorSupport.shuffleIota(ETYPE, Byte128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Byte128Shuffle)VectorSupport.shuffleIota(ETYPE, Byte128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Byte128Shuffle shuffleFromBytes(byte[] reorder) { return new Byte128Shuffle(reorder); } + Byte128Vector iota = Byte128Shuffle.IOTA.toBitsVector(); + ByteVector.ByteSpecies species = Byte128Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Byte128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Byte128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Byte128Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte128Shuffle(indexes, i); } + Byte128Shuffle shuffleFromArray(int[] indices, int i) { return new Byte128Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,10 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Byte128Mask.class, (Byte128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Byte128Shuffle.class); // specialize + public Byte128Shuffle toShuffle() { + return (Byte128Shuffle) super.toShuffleTemplate(Byte128Shuffle.class); // specialize } // Specialized unary testing @@ -809,26 +816,30 @@ static Byte128Mask maskAll(boolean bit) { // Shuffle static final class Byte128Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM - Byte128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Byte128Shuffle(byte[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Byte128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Byte128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Byte128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Byte128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Byte128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + byte[] indices() { + return (byte[])getPayload(); } @Override + @ForceInline public ByteSpecies vspecies() { return VSPECIES; } @@ -843,33 +854,101 @@ public ByteSpecies vspecies() { @Override @ForceInline - public Byte128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte128Shuffle.class, this, VLENGTH, - (s) -> ((Byte128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Byte128Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Byte128Shuffle.class, byte.class, VLENGTH, + Byte128Vector.class, byte.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Byte128Vector toBitsVectorHelper(Byte128Shuffle s) { + return (Byte128Vector) Byte128Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_128; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); } @ForceInline @Override public Byte128Shuffle rearrange(VectorShuffle shuffle) { - Byte128Shuffle s = (Byte128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Byte128Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Byte128Shuffle wrapIndexes() { + Byte128Vector v = toBitsVector(); + ByteVector.ByteSpecies species = Byte128Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static byte[] prepare(int[] indices, int offset) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static byte[] prepare(IntUnaryOperator f) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static boolean indicesInRange(byte[] indices) { + int length = indices.length; + for (byte si : indices) { + if (si >= (byte)length || si < (byte)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Byte128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java index e205071680378..8b982c562ddd5 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java @@ -143,22 +143,28 @@ Byte256Mask maskFromArray(boolean[] bits) { @ForceInline Byte256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Byte256Shuffle)VectorSupport.shuffleIota(ETYPE, Byte256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Byte256Shuffle)VectorSupport.shuffleIota(ETYPE, Byte256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Byte256Shuffle shuffleFromBytes(byte[] reorder) { return new Byte256Shuffle(reorder); } + Byte256Vector iota = Byte256Shuffle.IOTA.toBitsVector(); + ByteVector.ByteSpecies species = Byte256Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Byte256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Byte256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Byte256Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte256Shuffle(indexes, i); } + Byte256Shuffle shuffleFromArray(int[] indices, int i) { return new Byte256Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,10 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Byte256Mask.class, (Byte256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Byte256Shuffle.class); // specialize + public Byte256Shuffle toShuffle() { + return (Byte256Shuffle) super.toShuffleTemplate(Byte256Shuffle.class); // specialize } // Specialized unary testing @@ -841,26 +848,30 @@ static Byte256Mask maskAll(boolean bit) { // Shuffle static final class Byte256Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM - Byte256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Byte256Shuffle(byte[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Byte256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Byte256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Byte256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Byte256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Byte256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + byte[] indices() { + return (byte[])getPayload(); } @Override + @ForceInline public ByteSpecies vspecies() { return VSPECIES; } @@ -875,33 +886,101 @@ public ByteSpecies vspecies() { @Override @ForceInline - public Byte256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte256Shuffle.class, this, VLENGTH, - (s) -> ((Byte256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Byte256Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Byte256Shuffle.class, byte.class, VLENGTH, + Byte256Vector.class, byte.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Byte256Vector toBitsVectorHelper(Byte256Shuffle s) { + return (Byte256Vector) Byte256Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_256; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); } @ForceInline @Override public Byte256Shuffle rearrange(VectorShuffle shuffle) { - Byte256Shuffle s = (Byte256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Byte256Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Byte256Shuffle wrapIndexes() { + Byte256Vector v = toBitsVector(); + ByteVector.ByteSpecies species = Byte256Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static byte[] prepare(int[] indices, int offset) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static byte[] prepare(IntUnaryOperator f) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static boolean indicesInRange(byte[] indices) { + int length = indices.length; + for (byte si : indices) { + if (si >= (byte)length || si < (byte)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Byte256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java index a8a5875ea7b71..e25e20ca5d2bc 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java @@ -143,22 +143,28 @@ Byte512Mask maskFromArray(boolean[] bits) { @ForceInline Byte512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Byte512Shuffle)VectorSupport.shuffleIota(ETYPE, Byte512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Byte512Shuffle)VectorSupport.shuffleIota(ETYPE, Byte512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Byte512Shuffle shuffleFromBytes(byte[] reorder) { return new Byte512Shuffle(reorder); } + Byte512Vector iota = Byte512Shuffle.IOTA.toBitsVector(); + ByteVector.ByteSpecies species = Byte512Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Byte512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Byte512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Byte512Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte512Shuffle(indexes, i); } + Byte512Shuffle shuffleFromArray(int[] indices, int i) { return new Byte512Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,10 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Byte512Mask.class, (Byte512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Byte512Shuffle.class); // specialize + public Byte512Shuffle toShuffle() { + return (Byte512Shuffle) super.toShuffleTemplate(Byte512Shuffle.class); // specialize } // Specialized unary testing @@ -905,26 +912,30 @@ static Byte512Mask maskAll(boolean bit) { // Shuffle static final class Byte512Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM - Byte512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Byte512Shuffle(byte[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Byte512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Byte512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Byte512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Byte512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Byte512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + byte[] indices() { + return (byte[])getPayload(); } @Override + @ForceInline public ByteSpecies vspecies() { return VSPECIES; } @@ -939,33 +950,101 @@ public ByteSpecies vspecies() { @Override @ForceInline - public Byte512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte512Shuffle.class, this, VLENGTH, - (s) -> ((Byte512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Byte512Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Byte512Shuffle.class, byte.class, VLENGTH, + Byte512Vector.class, byte.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Byte512Vector toBitsVectorHelper(Byte512Shuffle s) { + return (Byte512Vector) Byte512Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_512; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); } @ForceInline @Override public Byte512Shuffle rearrange(VectorShuffle shuffle) { - Byte512Shuffle s = (Byte512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Byte512Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Byte512Shuffle wrapIndexes() { + Byte512Vector v = toBitsVector(); + ByteVector.ByteSpecies species = Byte512Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static byte[] prepare(int[] indices, int offset) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static byte[] prepare(IntUnaryOperator f) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static boolean indicesInRange(byte[] indices) { + int length = indices.length; + for (byte si : indices) { + if (si >= (byte)length || si < (byte)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Byte512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java index db3e925294495..3922279009d4c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java @@ -143,22 +143,28 @@ Byte64Mask maskFromArray(boolean[] bits) { @ForceInline Byte64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Byte64Shuffle)VectorSupport.shuffleIota(ETYPE, Byte64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Byte64Shuffle)VectorSupport.shuffleIota(ETYPE, Byte64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Byte64Shuffle shuffleFromBytes(byte[] reorder) { return new Byte64Shuffle(reorder); } + Byte64Vector iota = Byte64Shuffle.IOTA.toBitsVector(); + ByteVector.ByteSpecies species = Byte64Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Byte64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Byte64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Byte64Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte64Shuffle(indexes, i); } + Byte64Shuffle shuffleFromArray(int[] indices, int i) { return new Byte64Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,10 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Byte64Mask.class, (Byte64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Byte64Shuffle.class); // specialize + public Byte64Shuffle toShuffle() { + return (Byte64Shuffle) super.toShuffleTemplate(Byte64Shuffle.class); // specialize } // Specialized unary testing @@ -793,26 +800,30 @@ static Byte64Mask maskAll(boolean bit) { // Shuffle static final class Byte64Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM - Byte64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Byte64Shuffle(byte[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Byte64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Byte64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Byte64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Byte64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Byte64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + byte[] indices() { + return (byte[])getPayload(); } @Override + @ForceInline public ByteSpecies vspecies() { return VSPECIES; } @@ -827,33 +838,101 @@ public ByteSpecies vspecies() { @Override @ForceInline - public Byte64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte64Shuffle.class, this, VLENGTH, - (s) -> ((Byte64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Byte64Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Byte64Shuffle.class, byte.class, VLENGTH, + Byte64Vector.class, byte.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Byte64Vector toBitsVectorHelper(Byte64Shuffle s) { + return (Byte64Vector) Byte64Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_64; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); } @ForceInline @Override public Byte64Shuffle rearrange(VectorShuffle shuffle) { - Byte64Shuffle s = (Byte64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Byte64Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Byte64Shuffle wrapIndexes() { + Byte64Vector v = toBitsVector(); + ByteVector.ByteSpecies species = Byte64Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static byte[] prepare(int[] indices, int offset) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static byte[] prepare(IntUnaryOperator f) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static boolean indicesInRange(byte[] indices) { + int length = indices.length; + for (byte si : indices) { + if (si >= (byte)length || si < (byte)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Byte64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java index 0afdeab13dc40..56b7740d0344b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java @@ -143,22 +143,28 @@ ByteMaxMask maskFromArray(boolean[] bits) { @ForceInline ByteMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (ByteMaxShuffle)VectorSupport.shuffleIota(ETYPE, ByteMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (ByteMaxShuffle)VectorSupport.shuffleIota(ETYPE, ByteMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - ByteMaxShuffle shuffleFromBytes(byte[] reorder) { return new ByteMaxShuffle(reorder); } + ByteMaxVector iota = ByteMaxShuffle.IOTA.toBitsVector(); + ByteVector.ByteSpecies species = ByteMaxVector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + ByteMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + ByteMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - ByteMaxShuffle shuffleFromArray(int[] indexes, int i) { return new ByteMaxShuffle(indexes, i); } + ByteMaxShuffle shuffleFromArray(int[] indices, int i) { return new ByteMaxShuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,10 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, ByteMaxMask.class, (ByteMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(ByteMaxShuffle.class); // specialize + public ByteMaxShuffle toShuffle() { + return (ByteMaxShuffle) super.toShuffleTemplate(ByteMaxShuffle.class); // specialize } // Specialized unary testing @@ -779,26 +786,30 @@ static ByteMaxMask maskAll(boolean bit) { // Shuffle static final class ByteMaxShuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM - ByteMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + ByteMaxShuffle(byte[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public ByteMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + ByteMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public ByteMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + ByteMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public ByteMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + byte[] indices() { + return (byte[])getPayload(); } @Override + @ForceInline public ByteSpecies vspecies() { return VSPECIES; } @@ -813,33 +824,101 @@ public ByteSpecies vspecies() { @Override @ForceInline - public ByteMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, ByteMaxShuffle.class, this, VLENGTH, - (s) -> ((ByteMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + ByteMaxVector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + ByteMaxShuffle.class, byte.class, VLENGTH, + ByteMaxVector.class, byte.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static ByteMaxVector toBitsVectorHelper(ByteMaxShuffle s) { + return (ByteMaxVector) ByteMaxVector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_MAX; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); } @ForceInline @Override public ByteMaxShuffle rearrange(VectorShuffle shuffle) { - ByteMaxShuffle s = (ByteMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (ByteMaxShuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public ByteMaxShuffle wrapIndexes() { + ByteMaxVector v = toBitsVector(); + ByteVector.ByteSpecies species = ByteMaxVector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static byte[] prepare(int[] indices, int offset) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static byte[] prepare(IntUnaryOperator f) { + byte[] a = new byte[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (byte)si; + } + return a; + } + + private static boolean indicesInRange(byte[] indices) { + int length = indices.length; + for (byte si : indices) { + if (si >= (byte)length || si < (byte)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new ByteMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java index 0ae566fda7f2f..aaad8ac3391f3 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1071,7 +1071,7 @@ private static VectorBroadcastIntOp> broadcastIntOp // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,byte,byte,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,byte,VectorMask) @@ -2492,16 +2492,16 @@ VectorShuffle toShuffle0(ByteSpecies dsp) { /*package-private*/ @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { + final VectorShuffle toShuffleTemplate(Class shuffleType) { ByteSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), byte.class, length(), shuffleType, byte.class, length(), this, vsp, ByteVector::toShuffle0); } + /** * {@inheritDoc} * @since 19 diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java index 2c5706fe2f059..6c4336709ab0d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java @@ -143,22 +143,28 @@ Double128Mask maskFromArray(boolean[] bits) { @ForceInline Double128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Double128Shuffle)VectorSupport.shuffleIota(ETYPE, Double128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Double128Shuffle)VectorSupport.shuffleIota(ETYPE, Double128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Double128Shuffle shuffleFromBytes(byte[] reorder) { return new Double128Shuffle(reorder); } + Long128Vector iota = Double128Shuffle.IOTA.toBitsVector(); + LongVector.LongSpecies species = Long128Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Long128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Long128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toFPShuffle(); + } @Override @ForceInline - Double128Shuffle shuffleFromArray(int[] indexes, int i) { return new Double128Shuffle(indexes, i); } + Double128Shuffle shuffleFromArray(int[] indices, int i) { return new Double128Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +350,12 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Double128Mask.class, (Double128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Double128Shuffle.class); // specialize + public Double128Shuffle toShuffle() { + return (Double128Shuffle) castShape(Long128Vector.VSPECIES, 0) + .reinterpretAsLongs() + .toFPShuffle(); } // Specialized unary testing @@ -770,26 +779,30 @@ static Double128Mask maskAll(boolean bit) { // Shuffle static final class Double128Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = double.class; // used by the JVM - Double128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Double128Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Double128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Double128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Double128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Double128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Double128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public DoubleSpecies vspecies() { return VSPECIES; } @@ -797,40 +810,103 @@ public DoubleSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Double128Shuffle IOTA = new Double128Shuffle(IDENTITY); @Override @ForceInline - public Double128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double128Shuffle.class, this, VLENGTH, - (s) -> ((Double128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long128Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Double128Shuffle.class, double.class, VLENGTH, + Long128Vector.class, long.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Long128Vector toBitsVectorHelper(Double128Shuffle s) { + return (Long128Vector) Long128Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); } @ForceInline @Override public Double128Shuffle rearrange(VectorShuffle shuffle) { - Double128Shuffle s = (Double128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return toBitsVector() + .rearrange(shuffle.cast(Long128Vector.VSPECIES)) + .toFPShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Double128Shuffle wrapIndexes() { + Long128Vector v = toBitsVector(); + LongVector.LongSpecies species = Long128Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toFPShuffle(); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Double128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java index 3f4ee1bdc041b..edab5d2135f4f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java @@ -143,22 +143,28 @@ Double256Mask maskFromArray(boolean[] bits) { @ForceInline Double256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Double256Shuffle)VectorSupport.shuffleIota(ETYPE, Double256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Double256Shuffle)VectorSupport.shuffleIota(ETYPE, Double256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Double256Shuffle shuffleFromBytes(byte[] reorder) { return new Double256Shuffle(reorder); } + Long256Vector iota = Double256Shuffle.IOTA.toBitsVector(); + LongVector.LongSpecies species = Long256Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Long256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Long256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toFPShuffle(); + } @Override @ForceInline - Double256Shuffle shuffleFromArray(int[] indexes, int i) { return new Double256Shuffle(indexes, i); } + Double256Shuffle shuffleFromArray(int[] indices, int i) { return new Double256Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +350,12 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Double256Mask.class, (Double256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Double256Shuffle.class); // specialize + public Double256Shuffle toShuffle() { + return (Double256Shuffle) castShape(Long256Vector.VSPECIES, 0) + .reinterpretAsLongs() + .toFPShuffle(); } // Specialized unary testing @@ -774,26 +783,30 @@ static Double256Mask maskAll(boolean bit) { // Shuffle static final class Double256Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = double.class; // used by the JVM - Double256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Double256Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Double256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Double256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Double256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Double256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Double256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public DoubleSpecies vspecies() { return VSPECIES; } @@ -801,40 +814,103 @@ public DoubleSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Double256Shuffle IOTA = new Double256Shuffle(IDENTITY); @Override @ForceInline - public Double256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double256Shuffle.class, this, VLENGTH, - (s) -> ((Double256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long256Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Double256Shuffle.class, double.class, VLENGTH, + Long256Vector.class, long.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Long256Vector toBitsVectorHelper(Double256Shuffle s) { + return (Long256Vector) Long256Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); } @ForceInline @Override public Double256Shuffle rearrange(VectorShuffle shuffle) { - Double256Shuffle s = (Double256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return toBitsVector() + .rearrange(shuffle.cast(Long256Vector.VSPECIES)) + .toFPShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Double256Shuffle wrapIndexes() { + Long256Vector v = toBitsVector(); + LongVector.LongSpecies species = Long256Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toFPShuffle(); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Double256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java index 3f8211eef27d9..356ad265fbc2f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java @@ -143,22 +143,28 @@ Double512Mask maskFromArray(boolean[] bits) { @ForceInline Double512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Double512Shuffle)VectorSupport.shuffleIota(ETYPE, Double512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Double512Shuffle)VectorSupport.shuffleIota(ETYPE, Double512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Double512Shuffle shuffleFromBytes(byte[] reorder) { return new Double512Shuffle(reorder); } + Long512Vector iota = Double512Shuffle.IOTA.toBitsVector(); + LongVector.LongSpecies species = Long512Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Long512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Long512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toFPShuffle(); + } @Override @ForceInline - Double512Shuffle shuffleFromArray(int[] indexes, int i) { return new Double512Shuffle(indexes, i); } + Double512Shuffle shuffleFromArray(int[] indices, int i) { return new Double512Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +350,12 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Double512Mask.class, (Double512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Double512Shuffle.class); // specialize + public Double512Shuffle toShuffle() { + return (Double512Shuffle) castShape(Long512Vector.VSPECIES, 0) + .reinterpretAsLongs() + .toFPShuffle(); } // Specialized unary testing @@ -782,26 +791,30 @@ static Double512Mask maskAll(boolean bit) { // Shuffle static final class Double512Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = double.class; // used by the JVM - Double512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Double512Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Double512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Double512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Double512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Double512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Double512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public DoubleSpecies vspecies() { return VSPECIES; } @@ -809,40 +822,103 @@ public DoubleSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Double512Shuffle IOTA = new Double512Shuffle(IDENTITY); @Override @ForceInline - public Double512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double512Shuffle.class, this, VLENGTH, - (s) -> ((Double512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long512Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Double512Shuffle.class, double.class, VLENGTH, + Long512Vector.class, long.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Long512Vector toBitsVectorHelper(Double512Shuffle s) { + return (Long512Vector) Long512Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); } @ForceInline @Override public Double512Shuffle rearrange(VectorShuffle shuffle) { - Double512Shuffle s = (Double512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return toBitsVector() + .rearrange(shuffle.cast(Long512Vector.VSPECIES)) + .toFPShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Double512Shuffle wrapIndexes() { + Long512Vector v = toBitsVector(); + LongVector.LongSpecies species = Long512Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toFPShuffle(); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Double512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java index 9900c97f912d4..1f4ba3cc81200 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java @@ -143,22 +143,28 @@ Double64Mask maskFromArray(boolean[] bits) { @ForceInline Double64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Double64Shuffle)VectorSupport.shuffleIota(ETYPE, Double64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Double64Shuffle)VectorSupport.shuffleIota(ETYPE, Double64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Double64Shuffle shuffleFromBytes(byte[] reorder) { return new Double64Shuffle(reorder); } + Long64Vector iota = Double64Shuffle.IOTA.toBitsVector(); + LongVector.LongSpecies species = Long64Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Long64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Long64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toFPShuffle(); + } @Override @ForceInline - Double64Shuffle shuffleFromArray(int[] indexes, int i) { return new Double64Shuffle(indexes, i); } + Double64Shuffle shuffleFromArray(int[] indices, int i) { return new Double64Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +350,12 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Double64Mask.class, (Double64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Double64Shuffle.class); // specialize + public Double64Shuffle toShuffle() { + return (Double64Shuffle) castShape(Long64Vector.VSPECIES, 0) + .reinterpretAsLongs() + .toFPShuffle(); } // Specialized unary testing @@ -768,26 +777,30 @@ static Double64Mask maskAll(boolean bit) { // Shuffle static final class Double64Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = double.class; // used by the JVM - Double64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Double64Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Double64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Double64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Double64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Double64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Double64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public DoubleSpecies vspecies() { return VSPECIES; } @@ -795,40 +808,97 @@ public DoubleSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Double64Shuffle IOTA = new Double64Shuffle(IDENTITY); @Override @ForceInline - public Double64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double64Shuffle.class, this, VLENGTH, - (s) -> ((Double64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long64Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Double64Shuffle.class, double.class, VLENGTH, + Long64Vector.class, long.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Long64Vector toBitsVectorHelper(Double64Shuffle s) { + return (Long64Vector) Long64Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + a[offset] = laneSource(0); } @ForceInline @Override public Double64Shuffle rearrange(VectorShuffle shuffle) { - Double64Shuffle s = (Double64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return toBitsVector() + .rearrange(shuffle.cast(Long64Vector.VSPECIES)) + .toFPShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Double64Shuffle wrapIndexes() { + Long64Vector v = toBitsVector(); + LongVector.LongSpecies species = Long64Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toFPShuffle(); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Double64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java index 32ff760fb3693..1806b62c10e22 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java @@ -143,22 +143,28 @@ DoubleMaxMask maskFromArray(boolean[] bits) { @ForceInline DoubleMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (DoubleMaxShuffle)VectorSupport.shuffleIota(ETYPE, DoubleMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (DoubleMaxShuffle)VectorSupport.shuffleIota(ETYPE, DoubleMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - DoubleMaxShuffle shuffleFromBytes(byte[] reorder) { return new DoubleMaxShuffle(reorder); } + LongMaxVector iota = DoubleMaxShuffle.IOTA.toBitsVector(); + LongVector.LongSpecies species = LongMaxVector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + LongMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + LongMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toFPShuffle(); + } @Override @ForceInline - DoubleMaxShuffle shuffleFromArray(int[] indexes, int i) { return new DoubleMaxShuffle(indexes, i); } + DoubleMaxShuffle shuffleFromArray(int[] indices, int i) { return new DoubleMaxShuffle(indices, i); } @Override @ForceInline @@ -344,9 +350,12 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, DoubleMaxMask.class, (DoubleMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(DoubleMaxShuffle.class); // specialize + public DoubleMaxShuffle toShuffle() { + return (DoubleMaxShuffle) castShape(LongMaxVector.VSPECIES, 0) + .reinterpretAsLongs() + .toFPShuffle(); } // Specialized unary testing @@ -767,26 +776,30 @@ static DoubleMaxMask maskAll(boolean bit) { // Shuffle static final class DoubleMaxShuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = double.class; // used by the JVM - DoubleMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + DoubleMaxShuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public DoubleMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + DoubleMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public DoubleMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + DoubleMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public DoubleMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public DoubleSpecies vspecies() { return VSPECIES; } @@ -794,40 +807,103 @@ public DoubleSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final DoubleMaxShuffle IOTA = new DoubleMaxShuffle(IDENTITY); @Override @ForceInline - public DoubleMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, DoubleMaxShuffle.class, this, VLENGTH, - (s) -> ((DoubleMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + LongMaxVector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + DoubleMaxShuffle.class, double.class, VLENGTH, + LongMaxVector.class, long.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static LongMaxVector toBitsVectorHelper(DoubleMaxShuffle s) { + return (LongMaxVector) LongMaxVector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); } @ForceInline @Override public DoubleMaxShuffle rearrange(VectorShuffle shuffle) { - DoubleMaxShuffle s = (DoubleMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return toBitsVector() + .rearrange(shuffle.cast(LongMaxVector.VSPECIES)) + .toFPShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public DoubleMaxShuffle wrapIndexes() { + LongMaxVector v = toBitsVector(); + LongVector.LongSpecies species = LongMaxVector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toFPShuffle(); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new DoubleMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index 64bd795f817d0..a1f72a0e6264a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -953,7 +953,7 @@ DoubleVector lanewise(VectorOperators.Binary op, // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,double,double,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,double,VectorMask) @@ -2321,28 +2321,7 @@ DoubleVector rearrangeTemplate(Class shuffletype, return r1.blend(r0, valid); } - @ForceInline - private final - VectorShuffle toShuffle0(DoubleSpecies dsp) { - double[] a = toArray(); - int[] sa = new int[a.length]; - for (int i = 0; i < a.length; i++) { - sa[i] = (int) a[i]; - } - return VectorShuffle.fromArray(dsp, sa, 0); - } - /*package-private*/ - @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { - DoubleSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, - getClass(), double.class, length(), - shuffleType, byte.class, length(), - this, vsp, - DoubleVector::toShuffle0); - } /** * {@inheritDoc} diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java index 5414be75607bf..2360773c4ee69 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java @@ -143,22 +143,28 @@ Float128Mask maskFromArray(boolean[] bits) { @ForceInline Float128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Float128Shuffle)VectorSupport.shuffleIota(ETYPE, Float128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Float128Shuffle)VectorSupport.shuffleIota(ETYPE, Float128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Float128Shuffle shuffleFromBytes(byte[] reorder) { return new Float128Shuffle(reorder); } + Int128Vector iota = Float128Shuffle.IOTA.toBitsVector(); + IntVector.IntSpecies species = Int128Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Int128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Int128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toFPShuffle(); + } @Override @ForceInline - Float128Shuffle shuffleFromArray(int[] indexes, int i) { return new Float128Shuffle(indexes, i); } + Float128Shuffle shuffleFromArray(int[] indices, int i) { return new Float128Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +350,12 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Float128Mask.class, (Float128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Float128Shuffle.class); // specialize + public Float128Shuffle toShuffle() { + return (Float128Shuffle) castShape(Int128Vector.VSPECIES, 0) + .reinterpretAsInts() + .toFPShuffle(); } // Specialized unary testing @@ -774,26 +783,30 @@ static Float128Mask maskAll(boolean bit) { // Shuffle static final class Float128Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = float.class; // used by the JVM - Float128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Float128Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Float128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Float128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Float128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Float128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Float128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public FloatSpecies vspecies() { return VSPECIES; } @@ -801,40 +814,97 @@ public FloatSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Float128Shuffle IOTA = new Float128Shuffle(IDENTITY); @Override @ForceInline - public Float128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float128Shuffle.class, this, VLENGTH, - (s) -> ((Float128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int128Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Float128Shuffle.class, float.class, VLENGTH, + Int128Vector.class, int.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Int128Vector toBitsVectorHelper(Float128Shuffle s) { + return (Int128Vector) Int128Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); } @ForceInline @Override public Float128Shuffle rearrange(VectorShuffle shuffle) { - Float128Shuffle s = (Float128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return toBitsVector() + .rearrange(shuffle.cast(Int128Vector.VSPECIES)) + .toFPShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Float128Shuffle wrapIndexes() { + Int128Vector v = toBitsVector(); + IntVector.IntSpecies species = Int128Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toFPShuffle(); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Float128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java index 9194fbcde0edb..b17d2301e497a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java @@ -143,22 +143,28 @@ Float256Mask maskFromArray(boolean[] bits) { @ForceInline Float256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Float256Shuffle)VectorSupport.shuffleIota(ETYPE, Float256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Float256Shuffle)VectorSupport.shuffleIota(ETYPE, Float256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Float256Shuffle shuffleFromBytes(byte[] reorder) { return new Float256Shuffle(reorder); } + Int256Vector iota = Float256Shuffle.IOTA.toBitsVector(); + IntVector.IntSpecies species = Int256Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Int256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Int256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toFPShuffle(); + } @Override @ForceInline - Float256Shuffle shuffleFromArray(int[] indexes, int i) { return new Float256Shuffle(indexes, i); } + Float256Shuffle shuffleFromArray(int[] indices, int i) { return new Float256Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +350,12 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Float256Mask.class, (Float256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Float256Shuffle.class); // specialize + public Float256Shuffle toShuffle() { + return (Float256Shuffle) castShape(Int256Vector.VSPECIES, 0) + .reinterpretAsInts() + .toFPShuffle(); } // Specialized unary testing @@ -782,26 +791,30 @@ static Float256Mask maskAll(boolean bit) { // Shuffle static final class Float256Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = float.class; // used by the JVM - Float256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Float256Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Float256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Float256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Float256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Float256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Float256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public FloatSpecies vspecies() { return VSPECIES; } @@ -809,40 +822,97 @@ public FloatSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Float256Shuffle IOTA = new Float256Shuffle(IDENTITY); @Override @ForceInline - public Float256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float256Shuffle.class, this, VLENGTH, - (s) -> ((Float256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int256Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Float256Shuffle.class, float.class, VLENGTH, + Int256Vector.class, int.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Int256Vector toBitsVectorHelper(Float256Shuffle s) { + return (Int256Vector) Int256Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); } @ForceInline @Override public Float256Shuffle rearrange(VectorShuffle shuffle) { - Float256Shuffle s = (Float256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return toBitsVector() + .rearrange(shuffle.cast(Int256Vector.VSPECIES)) + .toFPShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Float256Shuffle wrapIndexes() { + Int256Vector v = toBitsVector(); + IntVector.IntSpecies species = Int256Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toFPShuffle(); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Float256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java index 2e43f153b0c55..cad27f4747746 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java @@ -143,22 +143,28 @@ Float512Mask maskFromArray(boolean[] bits) { @ForceInline Float512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Float512Shuffle)VectorSupport.shuffleIota(ETYPE, Float512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Float512Shuffle)VectorSupport.shuffleIota(ETYPE, Float512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Float512Shuffle shuffleFromBytes(byte[] reorder) { return new Float512Shuffle(reorder); } + Int512Vector iota = Float512Shuffle.IOTA.toBitsVector(); + IntVector.IntSpecies species = Int512Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Int512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Int512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toFPShuffle(); + } @Override @ForceInline - Float512Shuffle shuffleFromArray(int[] indexes, int i) { return new Float512Shuffle(indexes, i); } + Float512Shuffle shuffleFromArray(int[] indices, int i) { return new Float512Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +350,12 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Float512Mask.class, (Float512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Float512Shuffle.class); // specialize + public Float512Shuffle toShuffle() { + return (Float512Shuffle) castShape(Int512Vector.VSPECIES, 0) + .reinterpretAsInts() + .toFPShuffle(); } // Specialized unary testing @@ -798,26 +807,30 @@ static Float512Mask maskAll(boolean bit) { // Shuffle static final class Float512Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = float.class; // used by the JVM - Float512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Float512Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Float512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Float512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Float512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Float512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Float512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public FloatSpecies vspecies() { return VSPECIES; } @@ -825,40 +838,97 @@ public FloatSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Float512Shuffle IOTA = new Float512Shuffle(IDENTITY); @Override @ForceInline - public Float512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float512Shuffle.class, this, VLENGTH, - (s) -> ((Float512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int512Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Float512Shuffle.class, float.class, VLENGTH, + Int512Vector.class, int.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Int512Vector toBitsVectorHelper(Float512Shuffle s) { + return (Int512Vector) Int512Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); } @ForceInline @Override public Float512Shuffle rearrange(VectorShuffle shuffle) { - Float512Shuffle s = (Float512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return toBitsVector() + .rearrange(shuffle.cast(Int512Vector.VSPECIES)) + .toFPShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Float512Shuffle wrapIndexes() { + Int512Vector v = toBitsVector(); + IntVector.IntSpecies species = Int512Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toFPShuffle(); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Float512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java index ccd75e18e062a..f51a9071b8adb 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java @@ -143,22 +143,28 @@ Float64Mask maskFromArray(boolean[] bits) { @ForceInline Float64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Float64Shuffle)VectorSupport.shuffleIota(ETYPE, Float64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Float64Shuffle)VectorSupport.shuffleIota(ETYPE, Float64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Float64Shuffle shuffleFromBytes(byte[] reorder) { return new Float64Shuffle(reorder); } + Int64Vector iota = Float64Shuffle.IOTA.toBitsVector(); + IntVector.IntSpecies species = Int64Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Int64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Int64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toFPShuffle(); + } @Override @ForceInline - Float64Shuffle shuffleFromArray(int[] indexes, int i) { return new Float64Shuffle(indexes, i); } + Float64Shuffle shuffleFromArray(int[] indices, int i) { return new Float64Shuffle(indices, i); } @Override @ForceInline @@ -344,9 +350,12 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Float64Mask.class, (Float64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Float64Shuffle.class); // specialize + public Float64Shuffle toShuffle() { + return (Float64Shuffle) castShape(Int64Vector.VSPECIES, 0) + .reinterpretAsInts() + .toFPShuffle(); } // Specialized unary testing @@ -770,26 +779,30 @@ static Float64Mask maskAll(boolean bit) { // Shuffle static final class Float64Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = float.class; // used by the JVM - Float64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Float64Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Float64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Float64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Float64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Float64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Float64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public FloatSpecies vspecies() { return VSPECIES; } @@ -797,40 +810,97 @@ public FloatSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Float64Shuffle IOTA = new Float64Shuffle(IDENTITY); @Override @ForceInline - public Float64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float64Shuffle.class, this, VLENGTH, - (s) -> ((Float64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int64Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Float64Shuffle.class, float.class, VLENGTH, + Int64Vector.class, int.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Int64Vector toBitsVectorHelper(Float64Shuffle s) { + return (Int64Vector) Int64Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); } @ForceInline @Override public Float64Shuffle rearrange(VectorShuffle shuffle) { - Float64Shuffle s = (Float64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return toBitsVector() + .rearrange(shuffle.cast(Int64Vector.VSPECIES)) + .toFPShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Float64Shuffle wrapIndexes() { + Int64Vector v = toBitsVector(); + IntVector.IntSpecies species = Int64Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toFPShuffle(); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Float64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java index e5b4651531b45..0cb3bede649b4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java @@ -143,22 +143,28 @@ FloatMaxMask maskFromArray(boolean[] bits) { @ForceInline FloatMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (FloatMaxShuffle)VectorSupport.shuffleIota(ETYPE, FloatMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (FloatMaxShuffle)VectorSupport.shuffleIota(ETYPE, FloatMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - FloatMaxShuffle shuffleFromBytes(byte[] reorder) { return new FloatMaxShuffle(reorder); } + IntMaxVector iota = FloatMaxShuffle.IOTA.toBitsVector(); + IntVector.IntSpecies species = IntMaxVector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + IntMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + IntMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toFPShuffle(); + } @Override @ForceInline - FloatMaxShuffle shuffleFromArray(int[] indexes, int i) { return new FloatMaxShuffle(indexes, i); } + FloatMaxShuffle shuffleFromArray(int[] indices, int i) { return new FloatMaxShuffle(indices, i); } @Override @ForceInline @@ -344,9 +350,12 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, FloatMaxMask.class, (FloatMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(FloatMaxShuffle.class); // specialize + public FloatMaxShuffle toShuffle() { + return (FloatMaxShuffle) castShape(IntMaxVector.VSPECIES, 0) + .reinterpretAsInts() + .toFPShuffle(); } // Specialized unary testing @@ -767,26 +776,30 @@ static FloatMaxMask maskAll(boolean bit) { // Shuffle static final class FloatMaxShuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = float.class; // used by the JVM - FloatMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + FloatMaxShuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public FloatMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + FloatMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public FloatMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + FloatMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public FloatMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public FloatSpecies vspecies() { return VSPECIES; } @@ -794,40 +807,97 @@ public FloatSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final FloatMaxShuffle IOTA = new FloatMaxShuffle(IDENTITY); @Override @ForceInline - public FloatMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, FloatMaxShuffle.class, this, VLENGTH, - (s) -> ((FloatMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + IntMaxVector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + FloatMaxShuffle.class, float.class, VLENGTH, + IntMaxVector.class, int.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static IntMaxVector toBitsVectorHelper(FloatMaxShuffle s) { + return (IntMaxVector) IntMaxVector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); } @ForceInline @Override public FloatMaxShuffle rearrange(VectorShuffle shuffle) { - FloatMaxShuffle s = (FloatMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return toBitsVector() + .rearrange(shuffle.cast(IntMaxVector.VSPECIES)) + .toFPShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public FloatMaxShuffle wrapIndexes() { + IntMaxVector v = toBitsVector(); + IntVector.IntSpecies species = IntMaxVector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toFPShuffle(); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new FloatMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index 30ed43e302977..7beca7842c980 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -953,7 +953,7 @@ FloatVector lanewise(VectorOperators.Binary op, // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,float,float,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,float,VectorMask) @@ -2333,28 +2333,7 @@ FloatVector rearrangeTemplate(Class shuffletype, return r1.blend(r0, valid); } - @ForceInline - private final - VectorShuffle toShuffle0(FloatSpecies dsp) { - float[] a = toArray(); - int[] sa = new int[a.length]; - for (int i = 0; i < a.length; i++) { - sa[i] = (int) a[i]; - } - return VectorShuffle.fromArray(dsp, sa, 0); - } - /*package-private*/ - @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { - FloatSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, - getClass(), float.class, length(), - shuffleType, byte.class, length(), - this, vsp, - FloatVector::toShuffle0); - } /** * {@inheritDoc} diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java index 158686576f514..d6dc1db9c4388 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java @@ -143,22 +143,28 @@ Int128Mask maskFromArray(boolean[] bits) { @ForceInline Int128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Int128Shuffle)VectorSupport.shuffleIota(ETYPE, Int128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Int128Shuffle)VectorSupport.shuffleIota(ETYPE, Int128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Int128Shuffle shuffleFromBytes(byte[] reorder) { return new Int128Shuffle(reorder); } + Int128Vector iota = Int128Shuffle.IOTA.toBitsVector(); + IntVector.IntSpecies species = Int128Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Int128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Int128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Int128Shuffle shuffleFromArray(int[] indexes, int i) { return new Int128Shuffle(indexes, i); } + Int128Shuffle shuffleFromArray(int[] indices, int i) { return new Int128Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,15 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Int128Mask.class, (Int128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Int128Shuffle.class); // specialize + public Int128Shuffle toShuffle() { + return (Int128Shuffle) super.toShuffleTemplate(Int128Shuffle.class); // specialize + } + @Override + @ForceInline + Float128Vector.Float128Shuffle toFPShuffle() { + return (Float128Vector.Float128Shuffle) super.toFPShuffleTemplate(Float128Vector.Float128Shuffle.class, Float128Vector.VSPECIES); } // Specialized unary testing @@ -785,26 +797,30 @@ static Int128Mask maskAll(boolean bit) { // Shuffle static final class Int128Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM - Int128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Int128Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Int128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Int128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Int128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Int128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Int128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public IntSpecies vspecies() { return VSPECIES; } @@ -812,40 +828,95 @@ public IntSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Int128Shuffle IOTA = new Int128Shuffle(IDENTITY); @Override @ForceInline - public Int128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int128Shuffle.class, this, VLENGTH, - (s) -> ((Int128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int128Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Int128Shuffle.class, int.class, VLENGTH, + Int128Vector.class, int.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Int128Vector toBitsVectorHelper(Int128Shuffle s) { + return (Int128Vector) Int128Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); } @ForceInline @Override public Int128Shuffle rearrange(VectorShuffle shuffle) { - Int128Shuffle s = (Int128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Int128Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Int128Shuffle wrapIndexes() { + Int128Vector v = toBitsVector(); + IntVector.IntSpecies species = Int128Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Int128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java index 4f251e43f1f48..f93415c0d644c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java @@ -143,22 +143,28 @@ Int256Mask maskFromArray(boolean[] bits) { @ForceInline Int256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Int256Shuffle)VectorSupport.shuffleIota(ETYPE, Int256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Int256Shuffle)VectorSupport.shuffleIota(ETYPE, Int256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Int256Shuffle shuffleFromBytes(byte[] reorder) { return new Int256Shuffle(reorder); } + Int256Vector iota = Int256Shuffle.IOTA.toBitsVector(); + IntVector.IntSpecies species = Int256Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Int256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Int256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Int256Shuffle shuffleFromArray(int[] indexes, int i) { return new Int256Shuffle(indexes, i); } + Int256Shuffle shuffleFromArray(int[] indices, int i) { return new Int256Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,15 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Int256Mask.class, (Int256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Int256Shuffle.class); // specialize + public Int256Shuffle toShuffle() { + return (Int256Shuffle) super.toShuffleTemplate(Int256Shuffle.class); // specialize + } + @Override + @ForceInline + Float256Vector.Float256Shuffle toFPShuffle() { + return (Float256Vector.Float256Shuffle) super.toFPShuffleTemplate(Float256Vector.Float256Shuffle.class, Float256Vector.VSPECIES); } // Specialized unary testing @@ -793,26 +805,30 @@ static Int256Mask maskAll(boolean bit) { // Shuffle static final class Int256Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM - Int256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Int256Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Int256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Int256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Int256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Int256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Int256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public IntSpecies vspecies() { return VSPECIES; } @@ -820,40 +836,95 @@ public IntSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Int256Shuffle IOTA = new Int256Shuffle(IDENTITY); @Override @ForceInline - public Int256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int256Shuffle.class, this, VLENGTH, - (s) -> ((Int256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int256Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Int256Shuffle.class, int.class, VLENGTH, + Int256Vector.class, int.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Int256Vector toBitsVectorHelper(Int256Shuffle s) { + return (Int256Vector) Int256Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); } @ForceInline @Override public Int256Shuffle rearrange(VectorShuffle shuffle) { - Int256Shuffle s = (Int256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Int256Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Int256Shuffle wrapIndexes() { + Int256Vector v = toBitsVector(); + IntVector.IntSpecies species = Int256Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Int256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java index 4867c377a8a97..153ba84c63768 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java @@ -143,22 +143,28 @@ Int512Mask maskFromArray(boolean[] bits) { @ForceInline Int512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Int512Shuffle)VectorSupport.shuffleIota(ETYPE, Int512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Int512Shuffle)VectorSupport.shuffleIota(ETYPE, Int512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Int512Shuffle shuffleFromBytes(byte[] reorder) { return new Int512Shuffle(reorder); } + Int512Vector iota = Int512Shuffle.IOTA.toBitsVector(); + IntVector.IntSpecies species = Int512Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Int512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Int512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Int512Shuffle shuffleFromArray(int[] indexes, int i) { return new Int512Shuffle(indexes, i); } + Int512Shuffle shuffleFromArray(int[] indices, int i) { return new Int512Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,15 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Int512Mask.class, (Int512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Int512Shuffle.class); // specialize + public Int512Shuffle toShuffle() { + return (Int512Shuffle) super.toShuffleTemplate(Int512Shuffle.class); // specialize + } + @Override + @ForceInline + Float512Vector.Float512Shuffle toFPShuffle() { + return (Float512Vector.Float512Shuffle) super.toFPShuffleTemplate(Float512Vector.Float512Shuffle.class, Float512Vector.VSPECIES); } // Specialized unary testing @@ -809,26 +821,30 @@ static Int512Mask maskAll(boolean bit) { // Shuffle static final class Int512Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM - Int512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Int512Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Int512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Int512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Int512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Int512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Int512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public IntSpecies vspecies() { return VSPECIES; } @@ -836,40 +852,95 @@ public IntSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Int512Shuffle IOTA = new Int512Shuffle(IDENTITY); @Override @ForceInline - public Int512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int512Shuffle.class, this, VLENGTH, - (s) -> ((Int512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int512Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Int512Shuffle.class, int.class, VLENGTH, + Int512Vector.class, int.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Int512Vector toBitsVectorHelper(Int512Shuffle s) { + return (Int512Vector) Int512Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); } @ForceInline @Override public Int512Shuffle rearrange(VectorShuffle shuffle) { - Int512Shuffle s = (Int512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Int512Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Int512Shuffle wrapIndexes() { + Int512Vector v = toBitsVector(); + IntVector.IntSpecies species = Int512Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Int512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java index 2f44bfecf8f61..55ccb40ad44d6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java @@ -143,22 +143,28 @@ Int64Mask maskFromArray(boolean[] bits) { @ForceInline Int64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Int64Shuffle)VectorSupport.shuffleIota(ETYPE, Int64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Int64Shuffle)VectorSupport.shuffleIota(ETYPE, Int64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Int64Shuffle shuffleFromBytes(byte[] reorder) { return new Int64Shuffle(reorder); } + Int64Vector iota = Int64Shuffle.IOTA.toBitsVector(); + IntVector.IntSpecies species = Int64Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Int64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Int64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Int64Shuffle shuffleFromArray(int[] indexes, int i) { return new Int64Shuffle(indexes, i); } + Int64Shuffle shuffleFromArray(int[] indices, int i) { return new Int64Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,15 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Int64Mask.class, (Int64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Int64Shuffle.class); // specialize + public Int64Shuffle toShuffle() { + return (Int64Shuffle) super.toShuffleTemplate(Int64Shuffle.class); // specialize + } + @Override + @ForceInline + Float64Vector.Float64Shuffle toFPShuffle() { + return (Float64Vector.Float64Shuffle) super.toFPShuffleTemplate(Float64Vector.Float64Shuffle.class, Float64Vector.VSPECIES); } // Specialized unary testing @@ -781,26 +793,30 @@ static Int64Mask maskAll(boolean bit) { // Shuffle static final class Int64Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM - Int64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Int64Shuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Int64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Int64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Int64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Int64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Int64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public IntSpecies vspecies() { return VSPECIES; } @@ -808,40 +824,95 @@ public IntSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final Int64Shuffle IOTA = new Int64Shuffle(IDENTITY); @Override @ForceInline - public Int64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int64Shuffle.class, this, VLENGTH, - (s) -> ((Int64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Int64Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Int64Shuffle.class, int.class, VLENGTH, + Int64Vector.class, int.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Int64Vector toBitsVectorHelper(Int64Shuffle s) { + return (Int64Vector) Int64Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); } @ForceInline @Override public Int64Shuffle rearrange(VectorShuffle shuffle) { - Int64Shuffle s = (Int64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Int64Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Int64Shuffle wrapIndexes() { + Int64Vector v = toBitsVector(); + IntVector.IntSpecies species = Int64Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Int64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java index b79e73f52d6d1..7afa8f5b52402 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java @@ -143,22 +143,28 @@ IntMaxMask maskFromArray(boolean[] bits) { @ForceInline IntMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (IntMaxShuffle)VectorSupport.shuffleIota(ETYPE, IntMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (IntMaxShuffle)VectorSupport.shuffleIota(ETYPE, IntMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - IntMaxShuffle shuffleFromBytes(byte[] reorder) { return new IntMaxShuffle(reorder); } + IntMaxVector iota = IntMaxShuffle.IOTA.toBitsVector(); + IntVector.IntSpecies species = IntMaxVector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + IntMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + IntMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - IntMaxShuffle shuffleFromArray(int[] indexes, int i) { return new IntMaxShuffle(indexes, i); } + IntMaxShuffle shuffleFromArray(int[] indices, int i) { return new IntMaxShuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,15 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, IntMaxMask.class, (IntMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(IntMaxShuffle.class); // specialize + public IntMaxShuffle toShuffle() { + return (IntMaxShuffle) super.toShuffleTemplate(IntMaxShuffle.class); // specialize + } + @Override + @ForceInline + FloatMaxVector.FloatMaxShuffle toFPShuffle() { + return (FloatMaxVector.FloatMaxShuffle) super.toFPShuffleTemplate(FloatMaxVector.FloatMaxShuffle.class, FloatMaxVector.VSPECIES); } // Specialized unary testing @@ -790,26 +802,30 @@ static boolean[] maskLowerHalf() { // Shuffle static final class IntMaxShuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM - IntMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + IntMaxShuffle(int[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public IntMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + IntMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public IntMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + IntMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public IntMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + int[] indices() { + return (int[])getPayload(); } @Override + @ForceInline public IntSpecies vspecies() { return VSPECIES; } @@ -817,40 +833,95 @@ public IntSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Integer.MAX_VALUE); + assert(Integer.MIN_VALUE <= -VLENGTH); } static final IntMaxShuffle IOTA = new IntMaxShuffle(IDENTITY); @Override @ForceInline - public IntMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, IntMaxShuffle.class, this, VLENGTH, - (s) -> ((IntMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + IntMaxVector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + IntMaxShuffle.class, int.class, VLENGTH, + IntMaxVector.class, int.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static IntMaxVector toBitsVectorHelper(IntMaxShuffle s) { + return (IntMaxVector) IntMaxVector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + toBitsVector().intoArray(a, offset); } @ForceInline @Override public IntMaxShuffle rearrange(VectorShuffle shuffle) { - IntMaxShuffle s = (IntMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (IntMaxShuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public IntMaxShuffle wrapIndexes() { + IntMaxVector v = toBitsVector(); + IntVector.IntSpecies species = IntMaxVector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static int[] prepare(int[] indices, int offset) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static int[] prepare(IntUnaryOperator f) { + int[] a = new int[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (int)si; + } + return a; + } + + private static boolean indicesInRange(int[] indices) { + int length = indices.length; + for (int si : indices) { + if (si >= (int)length || si < (int)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new IntMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java index 87787b9a61199..1a6ac824ee34c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1074,7 +1074,7 @@ private static VectorBroadcastIntOp> broadcastInt // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,int,int,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,int,VectorMask) @@ -2477,16 +2477,38 @@ VectorShuffle toShuffle0(IntSpecies dsp) { /*package-private*/ @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { + final VectorShuffle toShuffleTemplate(Class shuffleType) { IntSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), int.class, length(), - shuffleType, byte.class, length(), + shuffleType, int.class, length(), this, vsp, IntVector::toShuffle0); } + abstract VectorShuffle toFPShuffle(); + + @ForceInline + private final + VectorShuffle toFPShuffle0(VectorSpecies dsp) { + int[] a = toArray(); + int[] sa = new int[a.length]; + for (int i = 0; i < a.length; i++) { + sa[i] = (int) a[i]; + } + return VectorShuffle.fromArray(dsp, sa, 0); + } + + @ForceInline + final + VectorShuffle toFPShuffleTemplate(Class shuffleType, FloatVector.FloatSpecies dsp) { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + getClass(), int.class, length(), + shuffleType, float.class, length(), + this, dsp, + IntVector::toFPShuffle0); + } + /** * {@inheritDoc} * @since 19 diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java index 9b024f6e55cba..7ee5067a60b2b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java @@ -138,22 +138,28 @@ Long128Mask maskFromArray(boolean[] bits) { @ForceInline Long128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Long128Shuffle)VectorSupport.shuffleIota(ETYPE, Long128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Long128Shuffle)VectorSupport.shuffleIota(ETYPE, Long128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Long128Shuffle shuffleFromBytes(byte[] reorder) { return new Long128Shuffle(reorder); } + Long128Vector iota = Long128Shuffle.IOTA.toBitsVector(); + LongVector.LongSpecies species = Long128Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Long128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Long128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Long128Shuffle shuffleFromArray(int[] indexes, int i) { return new Long128Shuffle(indexes, i); } + Long128Shuffle shuffleFromArray(int[] indices, int i) { return new Long128Shuffle(indices, i); } @Override @ForceInline @@ -352,9 +358,15 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Long128Mask.class, (Long128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Long128Shuffle.class); // specialize + public Long128Shuffle toShuffle() { + return (Long128Shuffle) super.toShuffleTemplate(Long128Shuffle.class); // specialize + } + @Override + @ForceInline + Double128Vector.Double128Shuffle toFPShuffle() { + return (Double128Vector.Double128Shuffle) super.toFPShuffleTemplate(Double128Vector.Double128Shuffle.class, Double128Vector.VSPECIES); } // Specialized unary testing @@ -771,26 +783,30 @@ static Long128Mask maskAll(boolean bit) { // Shuffle static final class Long128Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM - Long128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Long128Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Long128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Long128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Long128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Long128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Long128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public LongSpecies vspecies() { return VSPECIES; } @@ -798,40 +814,101 @@ public LongSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Long128Shuffle IOTA = new Long128Shuffle(IDENTITY); @Override @ForceInline - public Long128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long128Shuffle.class, this, VLENGTH, - (s) -> ((Long128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long128Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Long128Shuffle.class, long.class, VLENGTH, + Long128Vector.class, long.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Long128Vector toBitsVectorHelper(Long128Shuffle s) { + return (Long128Vector) Long128Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); } @ForceInline @Override public Long128Shuffle rearrange(VectorShuffle shuffle) { - Long128Shuffle s = (Long128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Long128Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Long128Shuffle wrapIndexes() { + Long128Vector v = toBitsVector(); + LongVector.LongSpecies species = Long128Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Long128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java index 8a1844af3702b..4fd4fa7a836f4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java @@ -138,22 +138,28 @@ Long256Mask maskFromArray(boolean[] bits) { @ForceInline Long256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Long256Shuffle)VectorSupport.shuffleIota(ETYPE, Long256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Long256Shuffle)VectorSupport.shuffleIota(ETYPE, Long256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Long256Shuffle shuffleFromBytes(byte[] reorder) { return new Long256Shuffle(reorder); } + Long256Vector iota = Long256Shuffle.IOTA.toBitsVector(); + LongVector.LongSpecies species = Long256Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Long256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Long256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Long256Shuffle shuffleFromArray(int[] indexes, int i) { return new Long256Shuffle(indexes, i); } + Long256Shuffle shuffleFromArray(int[] indices, int i) { return new Long256Shuffle(indices, i); } @Override @ForceInline @@ -352,9 +358,15 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Long256Mask.class, (Long256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Long256Shuffle.class); // specialize + public Long256Shuffle toShuffle() { + return (Long256Shuffle) super.toShuffleTemplate(Long256Shuffle.class); // specialize + } + @Override + @ForceInline + Double256Vector.Double256Shuffle toFPShuffle() { + return (Double256Vector.Double256Shuffle) super.toFPShuffleTemplate(Double256Vector.Double256Shuffle.class, Double256Vector.VSPECIES); } // Specialized unary testing @@ -775,26 +787,30 @@ static Long256Mask maskAll(boolean bit) { // Shuffle static final class Long256Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM - Long256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Long256Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Long256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Long256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Long256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Long256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Long256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public LongSpecies vspecies() { return VSPECIES; } @@ -802,40 +818,101 @@ public LongSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Long256Shuffle IOTA = new Long256Shuffle(IDENTITY); @Override @ForceInline - public Long256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long256Shuffle.class, this, VLENGTH, - (s) -> ((Long256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long256Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Long256Shuffle.class, long.class, VLENGTH, + Long256Vector.class, long.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Long256Vector toBitsVectorHelper(Long256Shuffle s) { + return (Long256Vector) Long256Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); } @ForceInline @Override public Long256Shuffle rearrange(VectorShuffle shuffle) { - Long256Shuffle s = (Long256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Long256Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Long256Shuffle wrapIndexes() { + Long256Vector v = toBitsVector(); + LongVector.LongSpecies species = Long256Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Long256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java index baec813f58eb3..f869fa28d8658 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java @@ -138,22 +138,28 @@ Long512Mask maskFromArray(boolean[] bits) { @ForceInline Long512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Long512Shuffle)VectorSupport.shuffleIota(ETYPE, Long512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Long512Shuffle)VectorSupport.shuffleIota(ETYPE, Long512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Long512Shuffle shuffleFromBytes(byte[] reorder) { return new Long512Shuffle(reorder); } + Long512Vector iota = Long512Shuffle.IOTA.toBitsVector(); + LongVector.LongSpecies species = Long512Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Long512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Long512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Long512Shuffle shuffleFromArray(int[] indexes, int i) { return new Long512Shuffle(indexes, i); } + Long512Shuffle shuffleFromArray(int[] indices, int i) { return new Long512Shuffle(indices, i); } @Override @ForceInline @@ -352,9 +358,15 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Long512Mask.class, (Long512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Long512Shuffle.class); // specialize + public Long512Shuffle toShuffle() { + return (Long512Shuffle) super.toShuffleTemplate(Long512Shuffle.class); // specialize + } + @Override + @ForceInline + Double512Vector.Double512Shuffle toFPShuffle() { + return (Double512Vector.Double512Shuffle) super.toFPShuffleTemplate(Double512Vector.Double512Shuffle.class, Double512Vector.VSPECIES); } // Specialized unary testing @@ -783,26 +795,30 @@ static Long512Mask maskAll(boolean bit) { // Shuffle static final class Long512Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM - Long512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Long512Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Long512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Long512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Long512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Long512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Long512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public LongSpecies vspecies() { return VSPECIES; } @@ -810,40 +826,101 @@ public LongSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Long512Shuffle IOTA = new Long512Shuffle(IDENTITY); @Override @ForceInline - public Long512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long512Shuffle.class, this, VLENGTH, - (s) -> ((Long512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long512Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Long512Shuffle.class, long.class, VLENGTH, + Long512Vector.class, long.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Long512Vector toBitsVectorHelper(Long512Shuffle s) { + return (Long512Vector) Long512Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); } @ForceInline @Override public Long512Shuffle rearrange(VectorShuffle shuffle) { - Long512Shuffle s = (Long512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Long512Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Long512Shuffle wrapIndexes() { + Long512Vector v = toBitsVector(); + LongVector.LongSpecies species = Long512Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Long512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java index 1d4110050f2d2..d0c283bb09048 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java @@ -138,22 +138,28 @@ Long64Mask maskFromArray(boolean[] bits) { @ForceInline Long64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Long64Shuffle)VectorSupport.shuffleIota(ETYPE, Long64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Long64Shuffle)VectorSupport.shuffleIota(ETYPE, Long64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Long64Shuffle shuffleFromBytes(byte[] reorder) { return new Long64Shuffle(reorder); } + Long64Vector iota = Long64Shuffle.IOTA.toBitsVector(); + LongVector.LongSpecies species = Long64Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Long64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Long64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Long64Shuffle shuffleFromArray(int[] indexes, int i) { return new Long64Shuffle(indexes, i); } + Long64Shuffle shuffleFromArray(int[] indices, int i) { return new Long64Shuffle(indices, i); } @Override @ForceInline @@ -352,9 +358,15 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Long64Mask.class, (Long64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Long64Shuffle.class); // specialize + public Long64Shuffle toShuffle() { + return (Long64Shuffle) super.toShuffleTemplate(Long64Shuffle.class); // specialize + } + @Override + @ForceInline + Double64Vector.Double64Shuffle toFPShuffle() { + return (Double64Vector.Double64Shuffle) super.toFPShuffleTemplate(Double64Vector.Double64Shuffle.class, Double64Vector.VSPECIES); } // Specialized unary testing @@ -769,26 +781,30 @@ static Long64Mask maskAll(boolean bit) { // Shuffle static final class Long64Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM - Long64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Long64Shuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Long64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Long64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Long64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Long64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Long64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public LongSpecies vspecies() { return VSPECIES; } @@ -796,40 +812,95 @@ public LongSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final Long64Shuffle IOTA = new Long64Shuffle(IDENTITY); @Override @ForceInline - public Long64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long64Shuffle.class, this, VLENGTH, - (s) -> ((Long64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Long64Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Long64Shuffle.class, long.class, VLENGTH, + Long64Vector.class, long.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Long64Vector toBitsVectorHelper(Long64Shuffle s) { + return (Long64Vector) Long64Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + a[offset] = laneSource(0); } @ForceInline @Override public Long64Shuffle rearrange(VectorShuffle shuffle) { - Long64Shuffle s = (Long64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Long64Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Long64Shuffle wrapIndexes() { + Long64Vector v = toBitsVector(); + LongVector.LongSpecies species = Long64Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Long64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java index ecc2b393061d4..9c6fc55166fee 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java @@ -138,22 +138,28 @@ LongMaxMask maskFromArray(boolean[] bits) { @ForceInline LongMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (LongMaxShuffle)VectorSupport.shuffleIota(ETYPE, LongMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (LongMaxShuffle)VectorSupport.shuffleIota(ETYPE, LongMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - LongMaxShuffle shuffleFromBytes(byte[] reorder) { return new LongMaxShuffle(reorder); } + LongMaxVector iota = LongMaxShuffle.IOTA.toBitsVector(); + LongVector.LongSpecies species = LongMaxVector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + LongMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + LongMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - LongMaxShuffle shuffleFromArray(int[] indexes, int i) { return new LongMaxShuffle(indexes, i); } + LongMaxShuffle shuffleFromArray(int[] indices, int i) { return new LongMaxShuffle(indices, i); } @Override @ForceInline @@ -352,9 +358,15 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, LongMaxMask.class, (LongMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(LongMaxShuffle.class); // specialize + public LongMaxShuffle toShuffle() { + return (LongMaxShuffle) super.toShuffleTemplate(LongMaxShuffle.class); // specialize + } + @Override + @ForceInline + DoubleMaxVector.DoubleMaxShuffle toFPShuffle() { + return (DoubleMaxVector.DoubleMaxShuffle) super.toFPShuffleTemplate(DoubleMaxVector.DoubleMaxShuffle.class, DoubleMaxVector.VSPECIES); } // Specialized unary testing @@ -769,26 +781,30 @@ static LongMaxMask maskAll(boolean bit) { // Shuffle static final class LongMaxShuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM - LongMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + LongMaxShuffle(long[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public LongMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + LongMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public LongMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + LongMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public LongMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + long[] indices() { + return (long[])getPayload(); } @Override + @ForceInline public LongSpecies vspecies() { return VSPECIES; } @@ -796,40 +812,101 @@ public LongSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Long.MAX_VALUE); + assert(Long.MIN_VALUE <= -VLENGTH); } static final LongMaxShuffle IOTA = new LongMaxShuffle(IDENTITY); @Override @ForceInline - public LongMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, LongMaxShuffle.class, this, VLENGTH, - (s) -> ((LongMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + LongMaxVector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + LongMaxShuffle.class, long.class, VLENGTH, + LongMaxVector.class, long.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static LongMaxVector toBitsVectorHelper(LongMaxShuffle s) { + return (LongMaxVector) LongMaxVector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); } @ForceInline @Override public LongMaxShuffle rearrange(VectorShuffle shuffle) { - LongMaxShuffle s = (LongMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (LongMaxShuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public LongMaxShuffle wrapIndexes() { + LongMaxVector v = toBitsVector(); + LongVector.LongSpecies species = LongMaxVector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static long[] prepare(int[] indices, int offset) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static long[] prepare(IntUnaryOperator f) { + long[] a = new long[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (long)si; + } + return a; + } + + private static boolean indicesInRange(long[] indices) { + int length = indices.length; + for (long si : indices) { + if (si >= (long)length || si < (long)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new LongMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java index 446d214a21bff..3620217fd532f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -987,7 +987,7 @@ private static VectorBroadcastIntOp> broadcastIntOp // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,long,long,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,long,VectorMask) @@ -2343,16 +2343,38 @@ VectorShuffle toShuffle0(LongSpecies dsp) { /*package-private*/ @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { + final VectorShuffle toShuffleTemplate(Class shuffleType) { LongSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), long.class, length(), - shuffleType, byte.class, length(), + shuffleType, long.class, length(), this, vsp, LongVector::toShuffle0); } + abstract VectorShuffle toFPShuffle(); + + @ForceInline + private final + VectorShuffle toFPShuffle0(VectorSpecies dsp) { + long[] a = toArray(); + int[] sa = new int[a.length]; + for (int i = 0; i < a.length; i++) { + sa[i] = (int) a[i]; + } + return VectorShuffle.fromArray(dsp, sa, 0); + } + + @ForceInline + final + VectorShuffle toFPShuffleTemplate(Class shuffleType, DoubleVector.DoubleSpecies dsp) { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + getClass(), long.class, length(), + shuffleType, double.class, length(), + this, dsp, + LongVector::toFPShuffle0); + } + /** * {@inheritDoc} * @since 19 diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java index 1f587ca54a5f9..34f5c5146f8f9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java @@ -143,22 +143,28 @@ Short128Mask maskFromArray(boolean[] bits) { @ForceInline Short128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Short128Shuffle)VectorSupport.shuffleIota(ETYPE, Short128Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Short128Shuffle)VectorSupport.shuffleIota(ETYPE, Short128Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Short128Shuffle shuffleFromBytes(byte[] reorder) { return new Short128Shuffle(reorder); } + Short128Vector iota = Short128Shuffle.IOTA.toBitsVector(); + ShortVector.ShortSpecies species = Short128Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Short128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Short128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Short128Shuffle shuffleFromArray(int[] indexes, int i) { return new Short128Shuffle(indexes, i); } + Short128Shuffle shuffleFromArray(int[] indices, int i) { return new Short128Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,10 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Short128Mask.class, (Short128Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Short128Shuffle.class); // specialize + public Short128Shuffle toShuffle() { + return (Short128Shuffle) super.toShuffleTemplate(Short128Shuffle.class); // specialize } // Specialized unary testing @@ -793,26 +800,30 @@ static Short128Mask maskAll(boolean bit) { // Shuffle static final class Short128Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM - Short128Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Short128Shuffle(short[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Short128Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Short128Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Short128Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Short128Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Short128Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + short[] indices() { + return (short[])getPayload(); } @Override + @ForceInline public ShortSpecies vspecies() { return VSPECIES; } @@ -820,40 +831,102 @@ public ShortSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Short.MAX_VALUE); + assert(Short.MIN_VALUE <= -VLENGTH); } static final Short128Shuffle IOTA = new Short128Shuffle(IDENTITY); @Override @ForceInline - public Short128Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short128Shuffle.class, this, VLENGTH, - (s) -> ((Short128Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Short128Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Short128Shuffle.class, short.class, VLENGTH, + Short128Vector.class, short.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Short128Vector toBitsVectorHelper(Short128Shuffle s) { + return (Short128Vector) Short128Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_128; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); } @ForceInline @Override public Short128Shuffle rearrange(VectorShuffle shuffle) { - Short128Shuffle s = (Short128Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Short128Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Short128Shuffle wrapIndexes() { + Short128Vector v = toBitsVector(); + ShortVector.ShortSpecies species = Short128Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static short[] prepare(int[] indices, int offset) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static short[] prepare(IntUnaryOperator f) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static boolean indicesInRange(short[] indices) { + int length = indices.length; + for (short si : indices) { + if (si >= (short)length || si < (short)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Short128Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java index 5e259b70b591d..8ec44e70f8338 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java @@ -143,22 +143,28 @@ Short256Mask maskFromArray(boolean[] bits) { @ForceInline Short256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Short256Shuffle)VectorSupport.shuffleIota(ETYPE, Short256Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Short256Shuffle)VectorSupport.shuffleIota(ETYPE, Short256Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Short256Shuffle shuffleFromBytes(byte[] reorder) { return new Short256Shuffle(reorder); } + Short256Vector iota = Short256Shuffle.IOTA.toBitsVector(); + ShortVector.ShortSpecies species = Short256Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Short256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Short256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Short256Shuffle shuffleFromArray(int[] indexes, int i) { return new Short256Shuffle(indexes, i); } + Short256Shuffle shuffleFromArray(int[] indices, int i) { return new Short256Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,10 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Short256Mask.class, (Short256Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Short256Shuffle.class); // specialize + public Short256Shuffle toShuffle() { + return (Short256Shuffle) super.toShuffleTemplate(Short256Shuffle.class); // specialize } // Specialized unary testing @@ -809,26 +816,30 @@ static Short256Mask maskAll(boolean bit) { // Shuffle static final class Short256Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM - Short256Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Short256Shuffle(short[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Short256Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Short256Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Short256Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Short256Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Short256Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + short[] indices() { + return (short[])getPayload(); } @Override + @ForceInline public ShortSpecies vspecies() { return VSPECIES; } @@ -836,40 +847,102 @@ public ShortSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Short.MAX_VALUE); + assert(Short.MIN_VALUE <= -VLENGTH); } static final Short256Shuffle IOTA = new Short256Shuffle(IDENTITY); @Override @ForceInline - public Short256Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short256Shuffle.class, this, VLENGTH, - (s) -> ((Short256Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Short256Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Short256Shuffle.class, short.class, VLENGTH, + Short256Vector.class, short.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Short256Vector toBitsVectorHelper(Short256Shuffle s) { + return (Short256Vector) Short256Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_256; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); } @ForceInline @Override public Short256Shuffle rearrange(VectorShuffle shuffle) { - Short256Shuffle s = (Short256Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Short256Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Short256Shuffle wrapIndexes() { + Short256Vector v = toBitsVector(); + ShortVector.ShortSpecies species = Short256Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static short[] prepare(int[] indices, int offset) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static short[] prepare(IntUnaryOperator f) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static boolean indicesInRange(short[] indices) { + int length = indices.length; + for (short si : indices) { + if (si >= (short)length || si < (short)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Short256Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java index a6cbfb4f8e489..b6d1e7558e2c2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java @@ -143,22 +143,28 @@ Short512Mask maskFromArray(boolean[] bits) { @ForceInline Short512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Short512Shuffle)VectorSupport.shuffleIota(ETYPE, Short512Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Short512Shuffle)VectorSupport.shuffleIota(ETYPE, Short512Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Short512Shuffle shuffleFromBytes(byte[] reorder) { return new Short512Shuffle(reorder); } + Short512Vector iota = Short512Shuffle.IOTA.toBitsVector(); + ShortVector.ShortSpecies species = Short512Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Short512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Short512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Short512Shuffle shuffleFromArray(int[] indexes, int i) { return new Short512Shuffle(indexes, i); } + Short512Shuffle shuffleFromArray(int[] indices, int i) { return new Short512Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,10 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Short512Mask.class, (Short512Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Short512Shuffle.class); // specialize + public Short512Shuffle toShuffle() { + return (Short512Shuffle) super.toShuffleTemplate(Short512Shuffle.class); // specialize } // Specialized unary testing @@ -841,26 +848,30 @@ static Short512Mask maskAll(boolean bit) { // Shuffle static final class Short512Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM - Short512Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Short512Shuffle(short[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Short512Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Short512Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Short512Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Short512Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Short512Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + short[] indices() { + return (short[])getPayload(); } @Override + @ForceInline public ShortSpecies vspecies() { return VSPECIES; } @@ -868,40 +879,102 @@ public ShortSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Short.MAX_VALUE); + assert(Short.MIN_VALUE <= -VLENGTH); } static final Short512Shuffle IOTA = new Short512Shuffle(IDENTITY); @Override @ForceInline - public Short512Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short512Shuffle.class, this, VLENGTH, - (s) -> ((Short512Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Short512Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Short512Shuffle.class, short.class, VLENGTH, + Short512Vector.class, short.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Short512Vector toBitsVectorHelper(Short512Shuffle s) { + return (Short512Vector) Short512Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_512; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); } @ForceInline @Override public Short512Shuffle rearrange(VectorShuffle shuffle) { - Short512Shuffle s = (Short512Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Short512Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Short512Shuffle wrapIndexes() { + Short512Vector v = toBitsVector(); + ShortVector.ShortSpecies species = Short512Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static short[] prepare(int[] indices, int offset) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static short[] prepare(IntUnaryOperator f) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static boolean indicesInRange(short[] indices) { + int length = indices.length; + for (short si : indices) { + if (si >= (short)length || si < (short)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Short512Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java index b2a2ec119524d..d5d65703fadee 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java @@ -143,22 +143,28 @@ Short64Mask maskFromArray(boolean[] bits) { @ForceInline Short64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (Short64Shuffle)VectorSupport.shuffleIota(ETYPE, Short64Shuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (Short64Shuffle)VectorSupport.shuffleIota(ETYPE, Short64Shuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - Short64Shuffle shuffleFromBytes(byte[] reorder) { return new Short64Shuffle(reorder); } + Short64Vector iota = Short64Shuffle.IOTA.toBitsVector(); + ShortVector.ShortSpecies species = Short64Vector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + Short64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + Short64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - Short64Shuffle shuffleFromArray(int[] indexes, int i) { return new Short64Shuffle(indexes, i); } + Short64Shuffle shuffleFromArray(int[] indices, int i) { return new Short64Shuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,10 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, Short64Mask.class, (Short64Mask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(Short64Shuffle.class); // specialize + public Short64Shuffle toShuffle() { + return (Short64Shuffle) super.toShuffleTemplate(Short64Shuffle.class); // specialize } // Specialized unary testing @@ -785,26 +792,30 @@ static Short64Mask maskAll(boolean bit) { // Shuffle static final class Short64Shuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM - Short64Shuffle(byte[] reorder) { - super(VLENGTH, reorder); + Short64Shuffle(short[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public Short64Shuffle(int[] reorder) { - super(VLENGTH, reorder); + Short64Shuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public Short64Shuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + Short64Shuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public Short64Shuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + short[] indices() { + return (short[])getPayload(); } @Override + @ForceInline public ShortSpecies vspecies() { return VSPECIES; } @@ -812,40 +823,102 @@ public ShortSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Short.MAX_VALUE); + assert(Short.MIN_VALUE <= -VLENGTH); } static final Short64Shuffle IOTA = new Short64Shuffle(IDENTITY); @Override @ForceInline - public Short64Vector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short64Shuffle.class, this, VLENGTH, - (s) -> ((Short64Vector)(((AbstractShuffle)(s)).toVectorTemplate()))); + Short64Vector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + Short64Shuffle.class, short.class, VLENGTH, + Short64Vector.class, short.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static Short64Vector toBitsVectorHelper(Short64Shuffle s) { + return (Short64Vector) Short64Vector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_64; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); } @ForceInline @Override public Short64Shuffle rearrange(VectorShuffle shuffle) { - Short64Shuffle s = (Short64Shuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (Short64Shuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public Short64Shuffle wrapIndexes() { + Short64Vector v = toBitsVector(); + ShortVector.ShortSpecies species = Short64Vector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static short[] prepare(int[] indices, int offset) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static short[] prepare(IntUnaryOperator f) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static boolean indicesInRange(short[] indices) { + int length = indices.length; + for (short si : indices) { + if (si >= (short)length || si < (short)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new Short64Shuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java index eaf99f61896b5..fb0be69e4d1c3 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java @@ -143,22 +143,28 @@ ShortMaxMask maskFromArray(boolean[] bits) { @ForceInline ShortMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return (ShortMaxShuffle)VectorSupport.shuffleIota(ETYPE, ShortMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return (ShortMaxShuffle)VectorSupport.shuffleIota(ETYPE, ShortMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - ShortMaxShuffle shuffleFromBytes(byte[] reorder) { return new ShortMaxShuffle(reorder); } + ShortMaxVector iota = ShortMaxShuffle.IOTA.toBitsVector(); + ShortVector.ShortSpecies species = ShortMaxVector.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + ShortMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + ShortMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } + return wrapped.toShuffle(); + } @Override @ForceInline - ShortMaxShuffle shuffleFromArray(int[] indexes, int i) { return new ShortMaxShuffle(indexes, i); } + ShortMaxShuffle shuffleFromArray(int[] indices, int i) { return new ShortMaxShuffle(indices, i); } @Override @ForceInline @@ -357,9 +363,10 @@ public final long reduceLanesToLong(VectorOperators.Associative op, return (long) super.reduceLanesTemplate(op, ShortMaxMask.class, (ShortMaxMask) m); // specialized } + @Override @ForceInline - public VectorShuffle toShuffle() { - return super.toShuffleTemplate(ShortMaxShuffle.class); // specialize + public ShortMaxShuffle toShuffle() { + return (ShortMaxShuffle) super.toShuffleTemplate(ShortMaxShuffle.class); // specialize } // Specialized unary testing @@ -779,26 +786,30 @@ static ShortMaxMask maskAll(boolean bit) { // Shuffle static final class ShortMaxShuffle extends AbstractShuffle { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM - ShortMaxShuffle(byte[] reorder) { - super(VLENGTH, reorder); + ShortMaxShuffle(short[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public ShortMaxShuffle(int[] reorder) { - super(VLENGTH, reorder); + ShortMaxShuffle(int[] indices, int i) { + this(prepare(indices, i)); } - public ShortMaxShuffle(int[] reorder, int i) { - super(VLENGTH, reorder, i); + ShortMaxShuffle(IntUnaryOperator fn) { + this(prepare(fn)); } - public ShortMaxShuffle(IntUnaryOperator fn) { - super(VLENGTH, fn); + short[] indices() { + return (short[])getPayload(); } @Override + @ForceInline public ShortSpecies vspecies() { return VSPECIES; } @@ -806,40 +817,102 @@ public ShortSpecies vspecies() { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < Short.MAX_VALUE); + assert(Short.MIN_VALUE <= -VLENGTH); } static final ShortMaxShuffle IOTA = new ShortMaxShuffle(IDENTITY); @Override @ForceInline - public ShortMaxVector toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, ShortMaxShuffle.class, this, VLENGTH, - (s) -> ((ShortMaxVector)(((AbstractShuffle)(s)).toVectorTemplate()))); + ShortMaxVector toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + ShortMaxShuffle.class, short.class, VLENGTH, + ShortMaxVector.class, short.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static ShortMaxVector toBitsVectorHelper(ShortMaxShuffle s) { + return (ShortMaxVector) ShortMaxVector.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { + VectorSpecies species = IntVector.SPECIES_MAX; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); } @ForceInline @Override public ShortMaxShuffle rearrange(VectorShuffle shuffle) { - ShortMaxShuffle s = (ShortMaxShuffle) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index + return (ShortMaxShuffle)toVector().rearrange(shuffle).toShuffle(); + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public ShortMaxShuffle wrapIndexes() { + ShortMaxVector v = toBitsVector(); + ShortVector.ShortSpecies species = ShortMaxVector.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask neg = v.compare(VectorOperators.LT, 0); + Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } + return v.toShuffle(); + } + + private static short[] prepare(int[] indices, int offset) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static short[] prepare(IntUnaryOperator f) { + short[] a = new short[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = (short)si; + } + return a; + } + + private static boolean indicesInRange(short[] indices) { + int length = indices.length; + for (short si : indices) { + if (si >= (short)length || si < (short)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new ShortMaxShuffle(r); + return true; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java index af7ef119a92df..51610789b18a9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1071,7 +1071,7 @@ private static VectorBroadcastIntOp> broadcastInt // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,short,short,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,short,VectorMask) @@ -2493,16 +2493,16 @@ VectorShuffle toShuffle0(ShortSpecies dsp) { /*package-private*/ @ForceInline - final - VectorShuffle toShuffleTemplate(Class shuffleType) { + final VectorShuffle toShuffleTemplate(Class shuffleType) { ShortSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), short.class, length(), - shuffleType, byte.class, length(), + shuffleType, short.class, length(), this, vsp, ShortVector::toShuffle0); } + /** * {@inheritDoc} * @since 19 diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java index 1ffbcef821afb..f51fb766f9fea 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java @@ -124,6 +124,7 @@ VectorSpecies withLanes(Class elementType) { * @throws IllegalArgumentException if no such vector shape exists * @see #vectorBitSize() */ + @ForceInline public static VectorShape forBitSize(int bitSize) { switch (bitSize) { case 64: diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java index b046874c39052..46ad9f357742b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java @@ -133,8 +133,8 @@ */ @SuppressWarnings("exports") public abstract class VectorShuffle extends jdk.internal.vm.vector.VectorSupport.VectorShuffle { - VectorShuffle(byte[] reorder) { - super(reorder); + VectorShuffle(Object indices) { + super(indices); } /** @@ -556,7 +556,7 @@ private static int unzipIndex(int i, int vlen, int part) { * @param i the lane index * @return the {@code int} lane element at lane index {@code i} */ - public int laneSource(int i) { return toArray()[i]; } + public abstract int laneSource(int i); /** * Rearranges the lane elements of this shuffle selecting lane indexes diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template index a4b4750f2523e..6930551fbbc66 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1204,7 +1204,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { // and broadcast, but it would be more surprising not to continue // the obvious pattern started by unary and binary. - /** + /** * {@inheritDoc} * @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask) * @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask) @@ -2856,6 +2856,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { return r1.blend(r0, valid); } +#if[!FP] @ForceInline private final VectorShuffle<$Boxtype$> toShuffle0($Type$Species dsp) { @@ -2869,15 +2870,40 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { /*package-private*/ @ForceInline - final - VectorShuffle<$Boxtype$> toShuffleTemplate(Class shuffleType) { + final VectorShuffle<$Boxtype$> toShuffleTemplate(Class shuffleType) { $Type$Species vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), $type$.class, length(), - shuffleType, byte.class, length(), + shuffleType, $type$.class, length(), this, vsp, $Type$Vector::toShuffle0); } +#end[!FP] + +#if[intOrLong] + abstract VectorShuffle<$Boxfptype$> toFPShuffle(); + + @ForceInline + private final + VectorShuffle<$Boxfptype$> toFPShuffle0(VectorSpecies<$Boxfptype$> dsp) { + $type$[] a = toArray(); + int[] sa = new int[a.length]; + for (int i = 0; i < a.length; i++) { + sa[i] = (int) a[i]; + } + return VectorShuffle.fromArray(dsp, sa, 0); + } + + @ForceInline + final + VectorShuffle<$Boxfptype$> toFPShuffleTemplate(Class shuffleType, $Fptype$Vector.$Fptype$Species dsp) { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + getClass(), $type$.class, length(), + shuffleType, $fptype$.class, length(), + this, dsp, + $Type$Vector::toFPShuffle0); + } +#end[intOrLong] /** * {@inheritDoc} diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template index 14ba8d34161f1..db628b7a6aa14 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template @@ -145,22 +145,33 @@ final class $vectortype$ extends $abstractvectortype$ { @ForceInline $shuffletype$ iotaShuffle(int start, int step, boolean wrap) { - if (wrap) { - return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 1, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l)))); - } else { - return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 0, - (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart))); - } - } + if ((VLENGTH & (VLENGTH - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) + : shuffleFromOp(i -> i * step + start); + } - @Override - @ForceInline - $shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); } + $bitsvectortype$ iota = $shuffletype$.IOTA.toBitsVector(); + $Bitstype$Vector.$Bitstype$Species species = $bitsvectortype$.VSPECIES; + iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) + .lanewise(VectorOperators.ADD, species.broadcast(start)); + $bitsvectortype$ wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + + if (!wrap) { + $bitsvectortype$ wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask<$Boxbitstype$> mask = iota.compare(VectorOperators.EQ, iota); + wrapped = wrappedLower.blend(wrapped, mask); + } +#if[!FP] + return wrapped.toShuffle(); +#end[!FP] +#if[FP] + return wrapped.toFPShuffle(); +#end[FP] + } @Override @ForceInline - $shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); } + $shuffletype$ shuffleFromArray(int[] indices, int i) { return new $shuffletype$(indices, i); } @Override @ForceInline @@ -361,10 +372,29 @@ final class $vectortype$ extends $abstractvectortype$ { return (long) super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m); // specialized } +#if[!FP] + @Override + @ForceInline + public $shuffletype$ toShuffle() { + return ($shuffletype$) super.toShuffleTemplate($shuffletype$.class); // specialize + } +#end[!FP] +#if[FP] + @Override + @ForceInline + public $shuffletype$ toShuffle() { + return ($shuffletype$) castShape($bitsvectortype$.VSPECIES, 0) + .reinterpretAs$Bitstype$s() + .toFPShuffle(); + } +#end[FP] +#if[intOrLong] + @Override @ForceInline - public VectorShuffle<$Boxtype$> toShuffle() { - return super.toShuffleTemplate($shuffletype$.class); // specialize + $fpvectortype$.$Fptype$$bits$Shuffle toFPShuffle() { + return ($fpvectortype$.$Fptype$$bits$Shuffle) super.toFPShuffleTemplate($fpvectortype$.$Fptype$$bits$Shuffle.class, $fpvectortype$.VSPECIES); } +#end[intOrLong] // Specialized unary testing @@ -1067,26 +1097,30 @@ final class $vectortype$ extends $abstractvectortype$ { // Shuffle static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> { + static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM - $shuffletype$(byte[] reorder) { - super(VLENGTH, reorder); + $shuffletype$($bitstype$[] indices) { + super(indices); + assert(VLENGTH == indices.length); + assert(indicesInRange(indices)); } - public $shuffletype$(int[] reorder) { - super(VLENGTH, reorder); + $shuffletype$(int[] indices, int i) { + this(prepare(indices, i)); } - public $shuffletype$(int[] reorder, int i) { - super(VLENGTH, reorder, i); + $shuffletype$(IntUnaryOperator fn) { + this(prepare(fn)); } - public $shuffletype$(IntUnaryOperator fn) { - super(VLENGTH, fn); + $bitstype$[] indices() { + return ($bitstype$[])getPayload(); } @Override + @ForceInline public $Type$Species vspecies() { return VSPECIES; } @@ -1094,40 +1128,149 @@ final class $vectortype$ extends $abstractvectortype$ { static { // There must be enough bits in the shuffle lanes to encode // VLENGTH valid indexes and VLENGTH exceptional ones. - assert(VLENGTH < Byte.MAX_VALUE); - assert(Byte.MIN_VALUE <= -VLENGTH); + assert(VLENGTH < $Boxbitstype$.MAX_VALUE); + assert($Boxbitstype$.MIN_VALUE <= -VLENGTH); } static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY); @Override @ForceInline - public $vectortype$ toVector() { - return VectorSupport.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH, - (s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate()))); + $bitsvectortype$ toBitsVector() { + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + $shuffletype$.class, $type$.class, VLENGTH, + $bitsvectortype$.class, $bitstype$.class, VLENGTH, + this, vspecies().asIntegral(), + (v, s) -> toBitsVectorHelper(v)); + } + + private static $bitsvectortype$ toBitsVectorHelper($shuffletype$ s) { + return ($bitsvectortype$) $bitsvectortype$.VSPECIES.dummyVector() + .vectorFactory(s.indices()); } @Override @ForceInline - public VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) - throw new IllegalArgumentException("VectorShuffle length and species length differ"); - int[] shuffleArray = toArray(); - return s.shuffleFromArray(shuffleArray, 0).check(s); + public int laneSource(int i) { + return (int)toBitsVector().lane(i); + } + + @Override + @ForceInline + public void intoArray(int[] a, int offset) { +#if[byte] + VectorSpecies species = IntVector.SPECIES_$BITS$; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.B2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.B2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); + v.convertShape(VectorOperators.B2I, species, 2) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 2); + v.convertShape(VectorOperators.B2I, species, 3) + .reinterpretAsInts() + .intoArray(a, offset + species.length() * 3); +#end[byte] +#if[short] + VectorSpecies species = IntVector.SPECIES_$BITS$; + Vector v = toBitsVector(); + v.convertShape(VectorOperators.S2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); + v.convertShape(VectorOperators.S2I, species, 1) + .reinterpretAsInts() + .intoArray(a, offset + species.length()); +#end[short] +#if[intOrFloat] + toBitsVector().intoArray(a, offset); +#end[intOrFloat] +#if[longOrDouble] +#if[!1L] + VectorSpecies species = VectorSpecies.of( + int.class, + VectorShape.forBitSize(length() * Integer.SIZE)); + Vector v = toBitsVector(); + v.convertShape(VectorOperators.L2I, species, 0) + .reinterpretAsInts() + .intoArray(a, offset); +#end[!1L] +#if[1L] + a[offset] = laneSource(0); +#end[1L] +#end[longOrDouble] } @ForceInline @Override public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) { - $shuffletype$ s = ($shuffletype$) shuffle; - byte[] reorder1 = reorder(); - byte[] reorder2 = s.reorder(); - byte[] r = new byte[reorder1.length]; - for (int i = 0; i < reorder1.length; i++) { - int ssi = reorder2[i]; - r[i] = reorder1[ssi]; // throws on exceptional index +#if[!FP] + return ($shuffletype$)toVector().rearrange(shuffle).toShuffle(); +#end[!FP] +#if[FP] + return toBitsVector() + .rearrange(shuffle.cast($bitsvectortype$.VSPECIES)) + .toFPShuffle(); +#end[FP] + } + + @ForceInline + @Override + @SuppressWarnings("unchecked") + public $shuffletype$ wrapIndexes() { + $bitsvectortype$ v = toBitsVector(); + $Bitstype$Vector.$Bitstype$Species species = $bitsvectortype$.VSPECIES; + if ((VLENGTH & (VLENGTH - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); + } else { + VectorMask<$Boxbitstype$> neg = v.compare(VectorOperators.LT, 0); + Vector<$Boxbitstype$> adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); + v = v.blend(adjusted, neg); + } +#if[!FP] + return v.toShuffle(); +#end[!FP] +#if[FP] + return v.toFPShuffle(); +#end[FP] + } + + private static $bitstype$[] prepare(int[] indices, int offset) { + $bitstype$[] a = new $bitstype$[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = indices[offset + i]; + si = partiallyWrapIndex(si, VLENGTH); + a[i] = ($bitstype$)si; + } + return a; + } + + private static $bitstype$[] prepare(IntUnaryOperator f) { + $bitstype$[] a = new $bitstype$[VLENGTH]; + for (int i = 0; i < VLENGTH; i++) { + int si = f.applyAsInt(i); + si = partiallyWrapIndex(si, VLENGTH); + a[i] = ($bitstype$)si; + } + return a; + } + + private static boolean indicesInRange($bitstype$[] indices) { + int length = indices.length; + for ($bitstype$ si : indices) { + if (si >= ($bitstype$)length || si < ($bitstype$)(-length)) { + boolean assertsEnabled = false; + assert(assertsEnabled = true); + if (assertsEnabled) { + String msg = ("index "+si+"out of range ["+length+"] in "+ + java.util.Arrays.toString(indices)); + throw new AssertionError(msg); + } + return false; + } } - return new $shuffletype$(r); + return true; } } From 471cf5ab0aab7ca9ede58e71e5f3bbc3673dc9d4 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Sun, 19 Mar 2023 06:10:42 +0800 Subject: [PATCH 02/13] fix mismatched fp vector payload types --- src/hotspot/share/opto/vectorIntrinsics.cpp | 14 +++++++++++--- .../jdk/incubator/vector/Double128Vector.java | 2 +- .../jdk/incubator/vector/Double256Vector.java | 2 +- .../jdk/incubator/vector/Double512Vector.java | 2 +- .../jdk/incubator/vector/Double64Vector.java | 2 +- .../jdk/incubator/vector/DoubleMaxVector.java | 2 +- .../jdk/incubator/vector/Float128Vector.java | 2 +- .../jdk/incubator/vector/Float256Vector.java | 2 +- .../jdk/incubator/vector/Float512Vector.java | 2 +- .../jdk/incubator/vector/Float64Vector.java | 2 +- .../jdk/incubator/vector/FloatMaxVector.java | 2 +- .../classes/jdk/incubator/vector/IntVector.java | 2 +- .../classes/jdk/incubator/vector/LongVector.java | 2 +- .../jdk/incubator/vector/X-Vector.java.template | 2 +- .../incubator/vector/X-VectorBits.java.template | 2 +- 15 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index 2683fbe8ca9cf..77937923650ae 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -1989,15 +1989,22 @@ bool LibraryCallKit::inline_vector_rearrange() { } return false; // should be primitive type } + BasicType elem_bt = elem_type->basic_type(); BasicType shuffle_bt = elem_bt; + if (shuffle_bt == T_FLOAT) { + shuffle_bt = T_INT; + } else if (shuffle_bt == T_DOUBLE) { + shuffle_bt = T_LONG; + } + int num_elem = vlen->get_con(); bool need_load_shuffle = Matcher::vector_needs_load_shuffle(shuffle_bt, num_elem); - if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) { + if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) { if (C->print_intrinsics()) { tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no", - num_elem, type2name(elem_bt)); + num_elem, type2name(shuffle_bt)); } return false; // not supported } @@ -2035,6 +2042,7 @@ bool LibraryCallKit::inline_vector_rearrange() { Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem); Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem); const TypeVect* vt = TypeVect::make(elem_bt, num_elem); + const TypeVect* st = TypeVect::make(shuffle_bt, num_elem); if (v1 == NULL || shuffle == NULL) { return false; // operand unboxing failed @@ -2055,7 +2063,7 @@ bool LibraryCallKit::inline_vector_rearrange() { } if (need_load_shuffle) { - shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, vt)); + shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st)); } Node* rearrange = new VectorRearrangeNode(v1, shuffle); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java index 6c4336709ab0d..0340380749e92 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java @@ -819,7 +819,7 @@ public DoubleSpecies vspecies() { @ForceInline Long128Vector toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Double128Shuffle.class, double.class, VLENGTH, + Double128Shuffle.class, long.class, VLENGTH, Long128Vector.class, long.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java index edab5d2135f4f..c830ab1273e48 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java @@ -823,7 +823,7 @@ public DoubleSpecies vspecies() { @ForceInline Long256Vector toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Double256Shuffle.class, double.class, VLENGTH, + Double256Shuffle.class, long.class, VLENGTH, Long256Vector.class, long.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java index 356ad265fbc2f..2d392b003487f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java @@ -831,7 +831,7 @@ public DoubleSpecies vspecies() { @ForceInline Long512Vector toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Double512Shuffle.class, double.class, VLENGTH, + Double512Shuffle.class, long.class, VLENGTH, Long512Vector.class, long.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java index 1f4ba3cc81200..d6f0297914289 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java @@ -817,7 +817,7 @@ public DoubleSpecies vspecies() { @ForceInline Long64Vector toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Double64Shuffle.class, double.class, VLENGTH, + Double64Shuffle.class, long.class, VLENGTH, Long64Vector.class, long.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java index 1806b62c10e22..4a8f2119fc949 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java @@ -816,7 +816,7 @@ public DoubleSpecies vspecies() { @ForceInline LongMaxVector toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - DoubleMaxShuffle.class, double.class, VLENGTH, + DoubleMaxShuffle.class, long.class, VLENGTH, LongMaxVector.class, long.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java index 2360773c4ee69..485e11680a4ff 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java @@ -823,7 +823,7 @@ public FloatSpecies vspecies() { @ForceInline Int128Vector toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Float128Shuffle.class, float.class, VLENGTH, + Float128Shuffle.class, int.class, VLENGTH, Int128Vector.class, int.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java index b17d2301e497a..0066c7503bd8f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java @@ -831,7 +831,7 @@ public FloatSpecies vspecies() { @ForceInline Int256Vector toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Float256Shuffle.class, float.class, VLENGTH, + Float256Shuffle.class, int.class, VLENGTH, Int256Vector.class, int.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java index cad27f4747746..e68161b9f629d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java @@ -847,7 +847,7 @@ public FloatSpecies vspecies() { @ForceInline Int512Vector toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Float512Shuffle.class, float.class, VLENGTH, + Float512Shuffle.class, int.class, VLENGTH, Int512Vector.class, int.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java index f51a9071b8adb..c93b1314d4047 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java @@ -819,7 +819,7 @@ public FloatSpecies vspecies() { @ForceInline Int64Vector toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Float64Shuffle.class, float.class, VLENGTH, + Float64Shuffle.class, int.class, VLENGTH, Int64Vector.class, int.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java index 0cb3bede649b4..165507c8fdb28 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java @@ -816,7 +816,7 @@ public FloatSpecies vspecies() { @ForceInline IntMaxVector toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - FloatMaxShuffle.class, float.class, VLENGTH, + FloatMaxShuffle.class, int.class, VLENGTH, IntMaxVector.class, int.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java index 1a6ac824ee34c..aceaff4ae8615 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java @@ -2504,7 +2504,7 @@ VectorShuffle toFPShuffle0(VectorSpecies dsp) { VectorShuffle toFPShuffleTemplate(Class shuffleType, FloatVector.FloatSpecies dsp) { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), int.class, length(), - shuffleType, float.class, length(), + shuffleType, int.class, length(), this, dsp, IntVector::toFPShuffle0); } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java index 3620217fd532f..8cf34cca675e8 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java @@ -2370,7 +2370,7 @@ VectorShuffle toFPShuffle0(VectorSpecies dsp) { VectorShuffle toFPShuffleTemplate(Class shuffleType, DoubleVector.DoubleSpecies dsp) { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), long.class, length(), - shuffleType, double.class, length(), + shuffleType, long.class, length(), this, dsp, LongVector::toFPShuffle0); } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template index 6930551fbbc66..ce90fc93845d2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template @@ -2899,7 +2899,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { VectorShuffle<$Boxfptype$> toFPShuffleTemplate(Class shuffleType, $Fptype$Vector.$Fptype$Species dsp) { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), $type$.class, length(), - shuffleType, $fptype$.class, length(), + shuffleType, $type$.class, length(), this, dsp, $Type$Vector::toFPShuffle0); } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template index db628b7a6aa14..fb282512a2df5 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template @@ -1137,7 +1137,7 @@ final class $vectortype$ extends $abstractvectortype$ { @ForceInline $bitsvectortype$ toBitsVector() { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - $shuffletype$.class, $type$.class, VLENGTH, + $shuffletype$.class, $bitstype$.class, VLENGTH, $bitsvectortype$.class, $bitstype$.class, VLENGTH, this, vspecies().asIntegral(), (v, s) -> toBitsVectorHelper(v)); From d7f651c626fa98c4290f7b77a97a909a70d5be33 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Sun, 19 Mar 2023 13:56:58 +0800 Subject: [PATCH 03/13] other architecture --- src/hotspot/cpu/aarch64/aarch64_vector.ad | 40 +++----------------- src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 | 40 +++----------------- src/hotspot/cpu/arm/arm.ad | 6 ++- src/hotspot/cpu/ppc/ppc.ad | 4 ++ src/hotspot/cpu/riscv/riscv.ad | 4 ++ src/hotspot/cpu/s390/s390.ad | 4 ++ 6 files changed, 27 insertions(+), 71 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad index 4c3523b71bfe1..78ffe0445874a 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector.ad +++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad @@ -307,6 +307,10 @@ source %{ } } + const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; + } + // Assert that the given node is not a variable shift. bool assert_not_var_shift(const Node* n) { assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift"); @@ -6053,41 +6057,6 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{ ins_pipe(pipe_slow); %} -// ------------------------------ Vector shuffle ------------------------------- - -instruct loadshuffle(vReg dst, vReg src) %{ - match(Set dst (VectorLoadShuffle src)); - format %{ "loadshuffle $dst, $src" %} - ins_encode %{ - BasicType bt = Matcher::vector_element_basic_type(this); - uint length_in_bytes = Matcher::vector_length_in_bytes(this); - if (bt == T_BYTE) { - if ($dst$$FloatRegister != $src$$FloatRegister) { - if (VM_Version::use_neon_for_vector(length_in_bytes)) { - __ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B, - $src$$FloatRegister, $src$$FloatRegister); - } else { - assert(UseSVE > 0, "must be sve"); - __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister); - } - } - } else { - if (VM_Version::use_neon_for_vector(length_in_bytes)) { - // 4S/8S, 4I, 4F - __ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B); - if (type2aelembytes(bt) == 4) { - __ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H); - } - } else { - assert(UseSVE > 0, "must be sve"); - __ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt), - $src$$FloatRegister, __ B); - } - } - %} - ins_pipe(pipe_slow); -%} - // ------------------------------ Vector rearrange ----------------------------- // Here is an example that rearranges a NEON vector with 4 ints: @@ -6110,6 +6079,7 @@ instruct loadshuffle(vReg dst, vReg src) %{ // need to lookup 2/4 bytes as a group. For VectorRearrange long, we use bsl // to implement rearrange. +// Maybe move the shuffle preparation to VectorLoadShuffle instruct rearrange_HS_neon(vReg dst, vReg src, vReg shuffle, vReg tmp1, vReg tmp2) %{ predicate(UseSVE == 0 && (Matcher::vector_element_basic_type(n) == T_SHORT || diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 index 277c972a25bf0..3d28f0cb704d0 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 +++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 @@ -297,6 +297,10 @@ source %{ } } + const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; + } + // Assert that the given node is not a variable shift. bool assert_not_var_shift(const Node* n) { assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift"); @@ -4406,41 +4410,6 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{ ins_pipe(pipe_slow); %} -// ------------------------------ Vector shuffle ------------------------------- - -instruct loadshuffle(vReg dst, vReg src) %{ - match(Set dst (VectorLoadShuffle src)); - format %{ "loadshuffle $dst, $src" %} - ins_encode %{ - BasicType bt = Matcher::vector_element_basic_type(this); - uint length_in_bytes = Matcher::vector_length_in_bytes(this); - if (bt == T_BYTE) { - if ($dst$$FloatRegister != $src$$FloatRegister) { - if (VM_Version::use_neon_for_vector(length_in_bytes)) { - __ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B, - $src$$FloatRegister, $src$$FloatRegister); - } else { - assert(UseSVE > 0, "must be sve"); - __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister); - } - } - } else { - if (VM_Version::use_neon_for_vector(length_in_bytes)) { - // 4S/8S, 4I, 4F - __ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B); - if (type2aelembytes(bt) == 4) { - __ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H); - } - } else { - assert(UseSVE > 0, "must be sve"); - __ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt), - $src$$FloatRegister, __ B); - } - } - %} - ins_pipe(pipe_slow); -%} - // ------------------------------ Vector rearrange ----------------------------- // Here is an example that rearranges a NEON vector with 4 ints: @@ -4463,6 +4432,7 @@ instruct loadshuffle(vReg dst, vReg src) %{ // need to lookup 2/4 bytes as a group. For VectorRearrange long, we use bsl // to implement rearrange. +// Maybe move the shuffle preparation to VectorLoadShuffle instruct rearrange_HS_neon(vReg dst, vReg src, vReg shuffle, vReg tmp1, vReg tmp2) %{ predicate(UseSVE == 0 && (Matcher::vector_element_basic_type(n) == T_SHORT || diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index d838c5dc69f31..8e56c353e957e 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -1,5 +1,5 @@ // -// Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -1025,6 +1025,10 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } +const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; +} + const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index b03e386fa942d..cce19d5c16ac6 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -2189,6 +2189,10 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } +const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; +} + const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 301dc01f689a8..3141dc7b2ebef 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1876,6 +1876,10 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } +const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; +} + const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index bd4ab45361853..4f8bbbfe1ec12 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1529,6 +1529,10 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } +const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { + return false; +} + const RegMask* Matcher::predicate_reg_mask(void) { return NULL; } From 66c23a1b792811dc3f2aa613089a6cc16c550008 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Sun, 19 Mar 2023 15:02:39 +0800 Subject: [PATCH 04/13] small beautifications --- src/hotspot/share/classfile/vmIntrinsics.hpp | 9 -- src/hotspot/share/opto/c2compiler.cpp | 1 - src/hotspot/share/opto/library_call.cpp | 2 - src/hotspot/share/opto/library_call.hpp | 1 - src/hotspot/share/opto/vectorIntrinsics.cpp | 97 ------------------- .../jdk/internal/vm/vector/VectorSupport.java | 19 ---- .../jdk/incubator/vector/Byte128Vector.java | 3 +- .../jdk/incubator/vector/Byte256Vector.java | 3 +- .../jdk/incubator/vector/Byte512Vector.java | 3 +- .../jdk/incubator/vector/Byte64Vector.java | 3 +- .../jdk/incubator/vector/ByteMaxVector.java | 3 +- .../jdk/incubator/vector/Double128Vector.java | 1 - .../jdk/incubator/vector/Double256Vector.java | 1 - .../jdk/incubator/vector/Double512Vector.java | 1 - .../jdk/incubator/vector/Double64Vector.java | 1 - .../jdk/incubator/vector/DoubleMaxVector.java | 1 - .../jdk/incubator/vector/Float128Vector.java | 1 - .../jdk/incubator/vector/Float256Vector.java | 1 - .../jdk/incubator/vector/Float512Vector.java | 1 - .../jdk/incubator/vector/Float64Vector.java | 1 - .../jdk/incubator/vector/FloatMaxVector.java | 1 - .../jdk/incubator/vector/Int128Vector.java | 3 +- .../jdk/incubator/vector/Int256Vector.java | 3 +- .../jdk/incubator/vector/Int512Vector.java | 3 +- .../jdk/incubator/vector/Int64Vector.java | 3 +- .../jdk/incubator/vector/IntMaxVector.java | 3 +- .../jdk/incubator/vector/Long128Vector.java | 3 +- .../jdk/incubator/vector/Long256Vector.java | 3 +- .../jdk/incubator/vector/Long512Vector.java | 3 +- .../jdk/incubator/vector/Long64Vector.java | 3 +- .../jdk/incubator/vector/LongMaxVector.java | 3 +- .../jdk/incubator/vector/Short128Vector.java | 3 +- .../jdk/incubator/vector/Short256Vector.java | 3 +- .../jdk/incubator/vector/Short512Vector.java | 3 +- .../jdk/incubator/vector/Short64Vector.java | 3 +- .../jdk/incubator/vector/ShortMaxVector.java | 3 +- .../vector/X-VectorBits.java.template | 3 +- 37 files changed, 21 insertions(+), 181 deletions(-) diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 50ce3493e5acb..680957af53b30 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -958,15 +958,6 @@ class methodHandle; "Ljdk/internal/vm/vector/VectorSupport$VectorPayload;") \ do_name(vector_frombits_coerced_name, "fromBitsCoerced") \ \ - do_intrinsic(_VectorShuffleIota, jdk_internal_vm_vector_VectorSupport, vector_shuffle_step_iota_name, vector_shuffle_step_iota_sig, F_S) \ - do_signature(vector_shuffle_step_iota_sig, "(Ljava/lang/Class;" \ - "Ljava/lang/Class;" \ - "Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;" \ - "IIII" \ - "Ljdk/internal/vm/vector/VectorSupport$ShuffleIotaOperation;)" \ - "Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;") \ - do_name(vector_shuffle_step_iota_name, "shuffleIota") \ - \ do_intrinsic(_VectorLoadOp, jdk_internal_vm_vector_VectorSupport, vector_load_op_name, vector_load_op_sig, F_S) \ do_signature(vector_load_op_sig, "(Ljava/lang/Class;" \ "Ljava/lang/Class;" \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index a655b89eeb431..91ac5531356ce 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -763,7 +763,6 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt case vmIntrinsics::_VectorBinaryOp: case vmIntrinsics::_VectorTernaryOp: case vmIntrinsics::_VectorFromBitsCoerced: - case vmIntrinsics::_VectorShuffleIota: case vmIntrinsics::_VectorLoadOp: case vmIntrinsics::_VectorLoadMaskedOp: case vmIntrinsics::_VectorStoreOp: diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 2098107807418..e1bd837d574e7 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -685,8 +685,6 @@ bool LibraryCallKit::try_to_inline(int predicate) { return inline_vector_nary_operation(3); case vmIntrinsics::_VectorFromBitsCoerced: return inline_vector_frombits_coerced(); - case vmIntrinsics::_VectorShuffleIota: - return inline_vector_shuffle_iota(); case vmIntrinsics::_VectorMaskOp: return inline_vector_mask_operation(); case vmIntrinsics::_VectorLoadOp: diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index e43b29ec5e2ab..453cf9d2bfb34 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -340,7 +340,6 @@ class LibraryCallKit : public GraphKit { // Vector API support bool inline_vector_nary_operation(int n); bool inline_vector_frombits_coerced(); - bool inline_vector_shuffle_iota(); bool inline_vector_mask_operation(); bool inline_vector_mem_operation(bool is_store); bool inline_vector_mem_masked_operation(bool is_store); diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index 77937923650ae..099702db14c7c 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -578,103 +578,6 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) { return true; } -// , E> -// Sh ShuffleIota(Class E, Class shuffleClass, Vector.Species s, int length, -// int start, int step, int wrap, ShuffleIotaOperation defaultImpl) -bool LibraryCallKit::inline_vector_shuffle_iota() { - const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr(); - const TypeInt* vlen = gvn().type(argument(3))->isa_int(); - const TypeInt* start_val = gvn().type(argument(4))->isa_int(); - const TypeInt* step_val = gvn().type(argument(5))->isa_int(); - const TypeInt* wrap = gvn().type(argument(6))->isa_int(); - - Node* start = argument(4); - Node* step = argument(5); - - if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) { - return false; // dead code - } - if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) || - shuffle_klass->const_oop() == NULL || !wrap->is_con()) { - return false; // not enough info for intrinsification - } - if (!is_klass_initialized(shuffle_klass)) { - if (C->print_intrinsics()) { - tty->print_cr(" ** klass argument not initialized"); - } - return false; - } - - int do_wrap = wrap->get_con(); - int num_elem = vlen->get_con(); - BasicType elem_bt = T_BYTE; - - if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) { - return false; - } - if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) { - return false; - } - if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) { - return false; - } - if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) { - return false; - } - if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) { - return false; - } - if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) { - return false; - } - - const Type * type_bt = Type::get_const_basic_type(elem_bt); - const TypeVect * vt = TypeVect::make(type_bt, num_elem); - - Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt)); - - if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) { - Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt)); - res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt)); - } else if (step_val->get_con() > 1) { - Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con()))); - Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem); - res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt)); - } - - if (!start_val->is_con() || start_val->get_con() != 0) { - Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt)); - res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt)); - } - - Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1)); - Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt)); - if(do_wrap) { - // Wrap the indices greater than lane count. - res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt)); - } else { - ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ge)); - Node * lane_cnt = gvn().makecon(TypeInt::make(num_elem)); - Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt)); - const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem); - Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vmask_type)); - - // Make the indices greater than lane count as -ve values. This matches the java side implementation. - res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt)); - Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt)); - res = gvn().transform(new VectorBlendNode(biased_val, res, mask)); - } - - ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass(); - const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass); - - // Wrap it up in VectorBox to keep object type information. - res = box_vector(res, shuffle_box_type, elem_bt, num_elem); - set_result(res); - C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); - return true; -} - // // long maskReductionCoerced(int oper, Class maskClass, Class elemClass, // int length, M m, VectorMaskOp defaultImpl) diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java index b71faa120e632..9e04798a89849 100644 --- a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java +++ b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java @@ -227,25 +227,6 @@ M indexPartiallyInUpperRange(Class mClass, Class eClass, return defaultImpl.apply(offset, limit); } - /* ============================================================================ */ - public interface ShuffleIotaOperation, - SH extends VectorShuffle> { - SH apply(int length, int start, int step, S s); - } - - @IntrinsicCandidate - public static - , - SH extends VectorShuffle> - SH shuffleIota(Class eClass, Class shClass, S s, - int length, - int start, int step, int wrap, - ShuffleIotaOperation defaultImpl) { - assert isNonCapturingLambda(defaultImpl) : defaultImpl; - return defaultImpl.apply(length, start, step, s); - } - /* ============================================================================ */ public interface IndexOperation, S extends VectorSpecies> { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java index 06d8bfb5d8bad..2f328fa612f98 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java @@ -895,12 +895,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Byte128Shuffle rearrange(VectorShuffle shuffle) { - return (Byte128Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Byte128Shuffle wrapIndexes() { Byte128Vector v = toBitsVector(); ByteVector.ByteSpecies species = Byte128Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java index 8b982c562ddd5..a4ce6f197ccd7 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java @@ -927,12 +927,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Byte256Shuffle rearrange(VectorShuffle shuffle) { - return (Byte256Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Byte256Shuffle wrapIndexes() { Byte256Vector v = toBitsVector(); ByteVector.ByteSpecies species = Byte256Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java index e25e20ca5d2bc..ad2d1ef46ebc2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java @@ -991,12 +991,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Byte512Shuffle rearrange(VectorShuffle shuffle) { - return (Byte512Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Byte512Shuffle wrapIndexes() { Byte512Vector v = toBitsVector(); ByteVector.ByteSpecies species = Byte512Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java index 3922279009d4c..a0beab4d28850 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java @@ -879,12 +879,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Byte64Shuffle rearrange(VectorShuffle shuffle) { - return (Byte64Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Byte64Shuffle wrapIndexes() { Byte64Vector v = toBitsVector(); ByteVector.ByteSpecies species = Byte64Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java index 56b7740d0344b..0541422d9f70a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java @@ -865,12 +865,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public ByteMaxShuffle rearrange(VectorShuffle shuffle) { - return (ByteMaxShuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public ByteMaxShuffle wrapIndexes() { ByteMaxVector v = toBitsVector(); ByteVector.ByteSpecies species = ByteMaxVector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java index 0340380749e92..04a9953714e91 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java @@ -858,7 +858,6 @@ public Double128Shuffle rearrange(VectorShuffle shuffle) { @ForceInline @Override - @SuppressWarnings("unchecked") public Double128Shuffle wrapIndexes() { Long128Vector v = toBitsVector(); LongVector.LongSpecies species = Long128Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java index c830ab1273e48..97ffe156388ed 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java @@ -862,7 +862,6 @@ public Double256Shuffle rearrange(VectorShuffle shuffle) { @ForceInline @Override - @SuppressWarnings("unchecked") public Double256Shuffle wrapIndexes() { Long256Vector v = toBitsVector(); LongVector.LongSpecies species = Long256Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java index 2d392b003487f..99719a7c05726 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java @@ -870,7 +870,6 @@ public Double512Shuffle rearrange(VectorShuffle shuffle) { @ForceInline @Override - @SuppressWarnings("unchecked") public Double512Shuffle wrapIndexes() { Long512Vector v = toBitsVector(); LongVector.LongSpecies species = Long512Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java index d6f0297914289..88262fdfb8969 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java @@ -850,7 +850,6 @@ public Double64Shuffle rearrange(VectorShuffle shuffle) { @ForceInline @Override - @SuppressWarnings("unchecked") public Double64Shuffle wrapIndexes() { Long64Vector v = toBitsVector(); LongVector.LongSpecies species = Long64Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java index 4a8f2119fc949..df53cdf1d08c1 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java @@ -855,7 +855,6 @@ public DoubleMaxShuffle rearrange(VectorShuffle shuffle) { @ForceInline @Override - @SuppressWarnings("unchecked") public DoubleMaxShuffle wrapIndexes() { LongMaxVector v = toBitsVector(); LongVector.LongSpecies species = LongMaxVector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java index 485e11680a4ff..8b3c20781abdb 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java @@ -856,7 +856,6 @@ public Float128Shuffle rearrange(VectorShuffle shuffle) { @ForceInline @Override - @SuppressWarnings("unchecked") public Float128Shuffle wrapIndexes() { Int128Vector v = toBitsVector(); IntVector.IntSpecies species = Int128Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java index 0066c7503bd8f..a8302e18e1abc 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java @@ -864,7 +864,6 @@ public Float256Shuffle rearrange(VectorShuffle shuffle) { @ForceInline @Override - @SuppressWarnings("unchecked") public Float256Shuffle wrapIndexes() { Int256Vector v = toBitsVector(); IntVector.IntSpecies species = Int256Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java index e68161b9f629d..2c19634be04be 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java @@ -880,7 +880,6 @@ public Float512Shuffle rearrange(VectorShuffle shuffle) { @ForceInline @Override - @SuppressWarnings("unchecked") public Float512Shuffle wrapIndexes() { Int512Vector v = toBitsVector(); IntVector.IntSpecies species = Int512Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java index c93b1314d4047..93816e831bdb2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java @@ -852,7 +852,6 @@ public Float64Shuffle rearrange(VectorShuffle shuffle) { @ForceInline @Override - @SuppressWarnings("unchecked") public Float64Shuffle wrapIndexes() { Int64Vector v = toBitsVector(); IntVector.IntSpecies species = Int64Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java index 165507c8fdb28..d13039a733069 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java @@ -849,7 +849,6 @@ public FloatMaxShuffle rearrange(VectorShuffle shuffle) { @ForceInline @Override - @SuppressWarnings("unchecked") public FloatMaxShuffle wrapIndexes() { IntMaxVector v = toBitsVector(); IntVector.IntSpecies species = IntMaxVector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java index d6dc1db9c4388..8318f5f20687b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java @@ -863,12 +863,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Int128Shuffle rearrange(VectorShuffle shuffle) { - return (Int128Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Int128Shuffle wrapIndexes() { Int128Vector v = toBitsVector(); IntVector.IntSpecies species = Int128Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java index f93415c0d644c..02444367dd8a9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java @@ -871,12 +871,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Int256Shuffle rearrange(VectorShuffle shuffle) { - return (Int256Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Int256Shuffle wrapIndexes() { Int256Vector v = toBitsVector(); IntVector.IntSpecies species = Int256Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java index 153ba84c63768..57b5172dd01a8 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java @@ -887,12 +887,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Int512Shuffle rearrange(VectorShuffle shuffle) { - return (Int512Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Int512Shuffle wrapIndexes() { Int512Vector v = toBitsVector(); IntVector.IntSpecies species = Int512Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java index 55ccb40ad44d6..3f034e5d2499a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java @@ -859,12 +859,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Int64Shuffle rearrange(VectorShuffle shuffle) { - return (Int64Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Int64Shuffle wrapIndexes() { Int64Vector v = toBitsVector(); IntVector.IntSpecies species = Int64Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java index 7afa8f5b52402..10ce47c98614d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java @@ -868,12 +868,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public IntMaxShuffle rearrange(VectorShuffle shuffle) { - return (IntMaxShuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public IntMaxShuffle wrapIndexes() { IntMaxVector v = toBitsVector(); IntVector.IntSpecies species = IntMaxVector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java index 7ee5067a60b2b..613c953b9169a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java @@ -855,12 +855,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Long128Shuffle rearrange(VectorShuffle shuffle) { - return (Long128Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Long128Shuffle wrapIndexes() { Long128Vector v = toBitsVector(); LongVector.LongSpecies species = Long128Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java index 4fd4fa7a836f4..603e907dc568e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java @@ -859,12 +859,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Long256Shuffle rearrange(VectorShuffle shuffle) { - return (Long256Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Long256Shuffle wrapIndexes() { Long256Vector v = toBitsVector(); LongVector.LongSpecies species = Long256Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java index f869fa28d8658..43612dfc5fe75 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java @@ -867,12 +867,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Long512Shuffle rearrange(VectorShuffle shuffle) { - return (Long512Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Long512Shuffle wrapIndexes() { Long512Vector v = toBitsVector(); LongVector.LongSpecies species = Long512Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java index d0c283bb09048..01eac54797333 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java @@ -847,12 +847,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Long64Shuffle rearrange(VectorShuffle shuffle) { - return (Long64Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Long64Shuffle wrapIndexes() { Long64Vector v = toBitsVector(); LongVector.LongSpecies species = Long64Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java index 9c6fc55166fee..8463d36dfa8ef 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java @@ -853,12 +853,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public LongMaxShuffle rearrange(VectorShuffle shuffle) { - return (LongMaxShuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public LongMaxShuffle wrapIndexes() { LongMaxVector v = toBitsVector(); LongVector.LongSpecies species = LongMaxVector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java index 34f5c5146f8f9..bddf49ed7fb33 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java @@ -873,12 +873,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Short128Shuffle rearrange(VectorShuffle shuffle) { - return (Short128Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Short128Shuffle wrapIndexes() { Short128Vector v = toBitsVector(); ShortVector.ShortSpecies species = Short128Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java index 8ec44e70f8338..400fef459f7af 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java @@ -889,12 +889,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Short256Shuffle rearrange(VectorShuffle shuffle) { - return (Short256Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Short256Shuffle wrapIndexes() { Short256Vector v = toBitsVector(); ShortVector.ShortSpecies species = Short256Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java index b6d1e7558e2c2..dca7f7bad4ab4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java @@ -921,12 +921,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Short512Shuffle rearrange(VectorShuffle shuffle) { - return (Short512Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Short512Shuffle wrapIndexes() { Short512Vector v = toBitsVector(); ShortVector.ShortSpecies species = Short512Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java index d5d65703fadee..9f2c8f9a7f4f5 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java @@ -865,12 +865,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public Short64Shuffle rearrange(VectorShuffle shuffle) { - return (Short64Shuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public Short64Shuffle wrapIndexes() { Short64Vector v = toBitsVector(); ShortVector.ShortSpecies species = Short64Vector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java index fb0be69e4d1c3..c7835607586c2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java @@ -859,12 +859,11 @@ public void intoArray(int[] a, int offset) { @ForceInline @Override public ShortMaxShuffle rearrange(VectorShuffle shuffle) { - return (ShortMaxShuffle)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); } @ForceInline @Override - @SuppressWarnings("unchecked") public ShortMaxShuffle wrapIndexes() { ShortMaxVector v = toBitsVector(); ShortVector.ShortSpecies species = ShortMaxVector.VSPECIES; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template index fb282512a2df5..1dbc61bad0410 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template @@ -1206,7 +1206,7 @@ final class $vectortype$ extends $abstractvectortype$ { @Override public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) { #if[!FP] - return ($shuffletype$)toVector().rearrange(shuffle).toShuffle(); + return toBitsVector().rearrange(shuffle).toShuffle(); #end[!FP] #if[FP] return toBitsVector() @@ -1217,7 +1217,6 @@ final class $vectortype$ extends $abstractvectortype$ { @ForceInline @Override - @SuppressWarnings("unchecked") public $shuffletype$ wrapIndexes() { $bitsvectortype$ v = toBitsVector(); $Bitstype$Vector.$Bitstype$Species species = $bitsvectortype$.VSPECIES; From 10d2fd34291b5eaf823d28e376677a8bd11a9fd9 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Sun, 19 Mar 2023 15:11:29 +0800 Subject: [PATCH 05/13] optimise laneIsValid --- .../share/classes/jdk/incubator/vector/AbstractShuffle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java index 41f536d39c1bf..fb7508c340cec 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java @@ -100,7 +100,7 @@ public final VectorShuffle checkIndexes() { @Override @ForceInline public final VectorMask laneIsValid() { - Vector shufvec = this.toVector(); + Vector shufvec = this.toBitsVector(); return shufvec.compare(VectorOperators.GE, 0) .cast(vspecies()); } From 7acf928d939ace11690715234c66faa12a0f2f76 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Sun, 19 Mar 2023 19:05:25 +0800 Subject: [PATCH 06/13] fix internal types, clean up --- src/hotspot/share/opto/graphKit.hpp | 2 +- src/hotspot/share/opto/vectorIntrinsics.cpp | 6 +++--- src/hotspot/share/opto/vectornode.hpp | 6 +----- src/hotspot/share/prims/vectorSupport.hpp | 1 - .../classes/jdk/incubator/vector/AbstractShuffle.java | 5 ++++- .../classes/jdk/incubator/vector/Byte128Vector.java | 7 +++---- .../classes/jdk/incubator/vector/Byte256Vector.java | 7 +++---- .../classes/jdk/incubator/vector/Byte512Vector.java | 7 +++---- .../share/classes/jdk/incubator/vector/Byte64Vector.java | 7 +++---- .../classes/jdk/incubator/vector/ByteMaxVector.java | 7 +++---- .../classes/jdk/incubator/vector/Double128Vector.java | 9 ++++----- .../classes/jdk/incubator/vector/Double256Vector.java | 9 ++++----- .../classes/jdk/incubator/vector/Double512Vector.java | 9 ++++----- .../classes/jdk/incubator/vector/Double64Vector.java | 9 ++++----- .../classes/jdk/incubator/vector/DoubleMaxVector.java | 9 ++++----- .../classes/jdk/incubator/vector/Float128Vector.java | 9 ++++----- .../classes/jdk/incubator/vector/Float256Vector.java | 9 ++++----- .../classes/jdk/incubator/vector/Float512Vector.java | 9 ++++----- .../classes/jdk/incubator/vector/Float64Vector.java | 9 ++++----- .../classes/jdk/incubator/vector/FloatMaxVector.java | 9 ++++----- .../share/classes/jdk/incubator/vector/Int128Vector.java | 7 +++---- .../share/classes/jdk/incubator/vector/Int256Vector.java | 7 +++---- .../share/classes/jdk/incubator/vector/Int512Vector.java | 7 +++---- .../share/classes/jdk/incubator/vector/Int64Vector.java | 7 +++---- .../share/classes/jdk/incubator/vector/IntMaxVector.java | 7 +++---- .../classes/jdk/incubator/vector/Long128Vector.java | 7 +++---- .../classes/jdk/incubator/vector/Long256Vector.java | 7 +++---- .../classes/jdk/incubator/vector/Long512Vector.java | 7 +++---- .../share/classes/jdk/incubator/vector/Long64Vector.java | 7 +++---- .../classes/jdk/incubator/vector/LongMaxVector.java | 7 +++---- .../classes/jdk/incubator/vector/Short128Vector.java | 7 +++---- .../classes/jdk/incubator/vector/Short256Vector.java | 7 +++---- .../classes/jdk/incubator/vector/Short512Vector.java | 7 +++---- .../classes/jdk/incubator/vector/Short64Vector.java | 7 +++---- .../classes/jdk/incubator/vector/ShortMaxVector.java | 7 +++---- .../jdk/incubator/vector/X-VectorBits.java.template | 9 ++++----- 36 files changed, 113 insertions(+), 146 deletions(-) diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index fe731fc35cad6..ca35d1470c239 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -904,7 +904,7 @@ class GraphKit : public Phase { // Vector API support (implemented in vectorIntrinsics.cpp) Node* box_vector(Node* in, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception = false); - Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector = false); + Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem); Node* vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem); }; diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index 6c11fbbed219f..734120104a674 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -156,7 +156,7 @@ Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType return gvn().transform(vbox); } -Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) { +Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) { assert(EnableVectorSupport, ""); const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr(); if (vbox_type->instance_klass() != vbox_type_v->instance_klass()) { @@ -167,7 +167,7 @@ Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType el } assert(check_vbox(vbox_type), ""); const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass())); - Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector)); + Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory())); return unbox; } @@ -615,7 +615,7 @@ bool LibraryCallKit::inline_vector_mask_operation() { const Type* elem_ty = Type::get_const_basic_type(elem_bt); ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass(); const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass); - Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true); + Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem); if (mask_vec == nullptr) { if (C->print_intrinsics()) { tty->print_cr(" ** unbox failed mask=%s", diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index a3553f91cf32a..b3ea6409127d1 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1704,14 +1704,11 @@ class VectorBoxAllocateNode : public CallStaticJavaNode { }; class VectorUnboxNode : public VectorNode { - private: - bool _shuffle_to_vector; protected: uint size_of() const { return sizeof(*this); } public: - VectorUnboxNode(Compile* C, const TypeVect* vec_type, Node* obj, Node* mem, bool shuffle_to_vector) + VectorUnboxNode(Compile* C, const TypeVect* vec_type, Node* obj, Node* mem) : VectorNode(mem, obj, vec_type) { - _shuffle_to_vector = shuffle_to_vector; init_class_id(Class_VectorUnbox); init_flags(Flag_is_macro); C->add_macro_node(this); @@ -1722,7 +1719,6 @@ class VectorUnboxNode : public VectorNode { Node* mem() const { return in(1); } virtual Node* Identity(PhaseGVN* phase); Node* Ideal(PhaseGVN* phase, bool can_reshape); - bool is_shuffle_to_vector() { return _shuffle_to_vector; } }; class RotateRightVNode : public VectorNode { diff --git a/src/hotspot/share/prims/vectorSupport.hpp b/src/hotspot/share/prims/vectorSupport.hpp index 7302e0060648b..90c5bcf7aaf62 100644 --- a/src/hotspot/share/prims/vectorSupport.hpp +++ b/src/hotspot/share/prims/vectorSupport.hpp @@ -147,6 +147,5 @@ class VectorSupport : AllStatic { static bool is_vector(Klass* klass); static bool is_vector_mask(Klass* klass); - static bool is_vector_shuffle(Klass* klass); }; #endif // SHARE_PRIMS_VECTORSUPPORT_HPP diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java index fb7508c340cec..26eddb2d584d0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java @@ -24,10 +24,13 @@ */ package jdk.incubator.vector; +import java.util.function.IntUnaryOperator; import jdk.internal.vm.annotation.ForceInline; abstract class AbstractShuffle extends VectorShuffle { - // Internal representation allows for a maximum index of 256 + static final IntUnaryOperator IDENTITY = i -> i; + + // Internal representation allows for a maximum index of E.MAX_VALUE - 1 // Values are clipped to [-VLENGTH..VLENGTH-1]. AbstractShuffle(Object indices) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java index 2f328fa612f98..c555d684055fc 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java @@ -155,9 +155,9 @@ Byte128Shuffle iotaShuffle(int start, int step, boolean wrap) { Byte128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Byte128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Byte128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -816,7 +816,6 @@ static Byte128Mask maskAll(boolean bit) { // Shuffle static final class Byte128Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java index a4ce6f197ccd7..4686f441ec799 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java @@ -155,9 +155,9 @@ Byte256Shuffle iotaShuffle(int start, int step, boolean wrap) { Byte256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Byte256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Byte256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -848,7 +848,6 @@ static Byte256Mask maskAll(boolean bit) { // Shuffle static final class Byte256Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java index ad2d1ef46ebc2..b99ae65020d75 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java @@ -155,9 +155,9 @@ Byte512Shuffle iotaShuffle(int start, int step, boolean wrap) { Byte512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Byte512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Byte512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -912,7 +912,6 @@ static Byte512Mask maskAll(boolean bit) { // Shuffle static final class Byte512Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java index a0beab4d28850..fff34a98ea99f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java @@ -155,9 +155,9 @@ Byte64Shuffle iotaShuffle(int start, int step, boolean wrap) { Byte64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Byte64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Byte64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -800,7 +800,6 @@ static Byte64Mask maskAll(boolean bit) { // Shuffle static final class Byte64Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java index 0541422d9f70a..e3cbabd18f101 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java @@ -155,9 +155,9 @@ ByteMaxShuffle iotaShuffle(int start, int step, boolean wrap) { ByteMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - ByteMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + ByteMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -786,7 +786,6 @@ static ByteMaxMask maskAll(boolean bit) { // Shuffle static final class ByteMaxShuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = byte.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java index 04a9953714e91..e676cd2e11d49 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java @@ -155,9 +155,9 @@ Double128Shuffle iotaShuffle(int start, int step, boolean wrap) { Long128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Long128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Long128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toFPShuffle(); } @@ -779,9 +779,8 @@ static Double128Mask maskAll(boolean bit) { // Shuffle static final class Double128Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = double.class; // used by the JVM + static final Class ETYPE = long.class; // used by the JVM Double128Shuffle(long[] indices) { super(indices); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java index 97ffe156388ed..e231b8807dee8 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java @@ -155,9 +155,9 @@ Double256Shuffle iotaShuffle(int start, int step, boolean wrap) { Long256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Long256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Long256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toFPShuffle(); } @@ -783,9 +783,8 @@ static Double256Mask maskAll(boolean bit) { // Shuffle static final class Double256Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = double.class; // used by the JVM + static final Class ETYPE = long.class; // used by the JVM Double256Shuffle(long[] indices) { super(indices); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java index 99719a7c05726..60fec60f825ca 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java @@ -155,9 +155,9 @@ Double512Shuffle iotaShuffle(int start, int step, boolean wrap) { Long512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Long512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Long512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toFPShuffle(); } @@ -791,9 +791,8 @@ static Double512Mask maskAll(boolean bit) { // Shuffle static final class Double512Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = double.class; // used by the JVM + static final Class ETYPE = long.class; // used by the JVM Double512Shuffle(long[] indices) { super(indices); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java index 88262fdfb8969..4a203f31c9f57 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java @@ -155,9 +155,9 @@ Double64Shuffle iotaShuffle(int start, int step, boolean wrap) { Long64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Long64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Long64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toFPShuffle(); } @@ -777,9 +777,8 @@ static Double64Mask maskAll(boolean bit) { // Shuffle static final class Double64Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = double.class; // used by the JVM + static final Class ETYPE = long.class; // used by the JVM Double64Shuffle(long[] indices) { super(indices); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java index df53cdf1d08c1..af64b577f73cb 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java @@ -155,9 +155,9 @@ DoubleMaxShuffle iotaShuffle(int start, int step, boolean wrap) { LongMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - LongMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + LongMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toFPShuffle(); } @@ -776,9 +776,8 @@ static DoubleMaxMask maskAll(boolean bit) { // Shuffle static final class DoubleMaxShuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = double.class; // used by the JVM + static final Class ETYPE = long.class; // used by the JVM DoubleMaxShuffle(long[] indices) { super(indices); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java index 8b3c20781abdb..5197500caeec9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java @@ -155,9 +155,9 @@ Float128Shuffle iotaShuffle(int start, int step, boolean wrap) { Int128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Int128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Int128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toFPShuffle(); } @@ -783,9 +783,8 @@ static Float128Mask maskAll(boolean bit) { // Shuffle static final class Float128Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = float.class; // used by the JVM + static final Class ETYPE = int.class; // used by the JVM Float128Shuffle(int[] indices) { super(indices); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java index a8302e18e1abc..0fc7489f35ca2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java @@ -155,9 +155,9 @@ Float256Shuffle iotaShuffle(int start, int step, boolean wrap) { Int256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Int256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Int256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toFPShuffle(); } @@ -791,9 +791,8 @@ static Float256Mask maskAll(boolean bit) { // Shuffle static final class Float256Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = float.class; // used by the JVM + static final Class ETYPE = int.class; // used by the JVM Float256Shuffle(int[] indices) { super(indices); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java index 2c19634be04be..3431622349d0d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java @@ -155,9 +155,9 @@ Float512Shuffle iotaShuffle(int start, int step, boolean wrap) { Int512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Int512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Int512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toFPShuffle(); } @@ -807,9 +807,8 @@ static Float512Mask maskAll(boolean bit) { // Shuffle static final class Float512Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = float.class; // used by the JVM + static final Class ETYPE = int.class; // used by the JVM Float512Shuffle(int[] indices) { super(indices); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java index 93816e831bdb2..390c50a54c39b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java @@ -155,9 +155,9 @@ Float64Shuffle iotaShuffle(int start, int step, boolean wrap) { Int64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Int64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Int64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toFPShuffle(); } @@ -779,9 +779,8 @@ static Float64Mask maskAll(boolean bit) { // Shuffle static final class Float64Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = float.class; // used by the JVM + static final Class ETYPE = int.class; // used by the JVM Float64Shuffle(int[] indices) { super(indices); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java index d13039a733069..3aabf8e34cfed 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java @@ -155,9 +155,9 @@ FloatMaxShuffle iotaShuffle(int start, int step, boolean wrap) { IntMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - IntMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + IntMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toFPShuffle(); } @@ -776,9 +776,8 @@ static FloatMaxMask maskAll(boolean bit) { // Shuffle static final class FloatMaxShuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class ETYPE = float.class; // used by the JVM + static final Class ETYPE = int.class; // used by the JVM FloatMaxShuffle(int[] indices) { super(indices); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java index 8318f5f20687b..e93fb5c65bbc6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java @@ -155,9 +155,9 @@ Int128Shuffle iotaShuffle(int start, int step, boolean wrap) { Int128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Int128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Int128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -797,7 +797,6 @@ static Int128Mask maskAll(boolean bit) { // Shuffle static final class Int128Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java index 02444367dd8a9..220b5179a9f4d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java @@ -155,9 +155,9 @@ Int256Shuffle iotaShuffle(int start, int step, boolean wrap) { Int256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Int256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Int256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -805,7 +805,6 @@ static Int256Mask maskAll(boolean bit) { // Shuffle static final class Int256Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java index 57b5172dd01a8..fd3f9b2c68154 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java @@ -155,9 +155,9 @@ Int512Shuffle iotaShuffle(int start, int step, boolean wrap) { Int512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Int512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Int512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -821,7 +821,6 @@ static Int512Mask maskAll(boolean bit) { // Shuffle static final class Int512Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java index 3f034e5d2499a..384fc1dd9b2c0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java @@ -155,9 +155,9 @@ Int64Shuffle iotaShuffle(int start, int step, boolean wrap) { Int64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Int64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Int64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -793,7 +793,6 @@ static Int64Mask maskAll(boolean bit) { // Shuffle static final class Int64Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java index 10ce47c98614d..35c26e00fbbdf 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java @@ -155,9 +155,9 @@ IntMaxShuffle iotaShuffle(int start, int step, boolean wrap) { IntMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - IntMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + IntMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -802,7 +802,6 @@ static boolean[] maskLowerHalf() { // Shuffle static final class IntMaxShuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = int.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java index 613c953b9169a..51744b8d88c29 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java @@ -150,9 +150,9 @@ Long128Shuffle iotaShuffle(int start, int step, boolean wrap) { Long128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Long128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Long128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -783,7 +783,6 @@ static Long128Mask maskAll(boolean bit) { // Shuffle static final class Long128Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java index 603e907dc568e..a4fb5ff5e8a6d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java @@ -150,9 +150,9 @@ Long256Shuffle iotaShuffle(int start, int step, boolean wrap) { Long256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Long256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Long256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -787,7 +787,6 @@ static Long256Mask maskAll(boolean bit) { // Shuffle static final class Long256Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java index 43612dfc5fe75..8ad3a5a6fe33d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java @@ -150,9 +150,9 @@ Long512Shuffle iotaShuffle(int start, int step, boolean wrap) { Long512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Long512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Long512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -795,7 +795,6 @@ static Long512Mask maskAll(boolean bit) { // Shuffle static final class Long512Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java index 01eac54797333..0ee353d60d62a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java @@ -150,9 +150,9 @@ Long64Shuffle iotaShuffle(int start, int step, boolean wrap) { Long64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Long64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Long64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -781,7 +781,6 @@ static Long64Mask maskAll(boolean bit) { // Shuffle static final class Long64Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java index 8463d36dfa8ef..72bee5c553592 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java @@ -150,9 +150,9 @@ LongMaxShuffle iotaShuffle(int start, int step, boolean wrap) { LongMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - LongMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + LongMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -781,7 +781,6 @@ static LongMaxMask maskAll(boolean bit) { // Shuffle static final class LongMaxShuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = long.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java index bddf49ed7fb33..c7f65e4848666 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java @@ -155,9 +155,9 @@ Short128Shuffle iotaShuffle(int start, int step, boolean wrap) { Short128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Short128Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Short128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -800,7 +800,6 @@ static Short128Mask maskAll(boolean bit) { // Shuffle static final class Short128Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java index 400fef459f7af..9c42fb57b45e9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java @@ -155,9 +155,9 @@ Short256Shuffle iotaShuffle(int start, int step, boolean wrap) { Short256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Short256Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Short256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -816,7 +816,6 @@ static Short256Mask maskAll(boolean bit) { // Shuffle static final class Short256Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java index dca7f7bad4ab4..f1f9fe96314a6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java @@ -155,9 +155,9 @@ Short512Shuffle iotaShuffle(int start, int step, boolean wrap) { Short512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Short512Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Short512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -848,7 +848,6 @@ static Short512Mask maskAll(boolean bit) { // Shuffle static final class Short512Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java index 9f2c8f9a7f4f5..9995202f975a5 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java @@ -155,9 +155,9 @@ Short64Shuffle iotaShuffle(int start, int step, boolean wrap) { Short64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - Short64Vector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + Short64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -792,7 +792,6 @@ static Short64Mask maskAll(boolean bit) { // Shuffle static final class Short64Shuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java index c7835607586c2..6386c3dedc303 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java @@ -155,9 +155,9 @@ ShortMaxShuffle iotaShuffle(int start, int step, boolean wrap) { ShortMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - ShortMaxVector wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + ShortMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } return wrapped.toShuffle(); } @@ -786,7 +786,6 @@ static ShortMaxMask maskAll(boolean bit) { // Shuffle static final class ShortMaxShuffle extends AbstractShuffle { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM static final Class ETYPE = short.class; // used by the JVM diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template index 1dbc61bad0410..0a31e32fb27cf 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template @@ -157,9 +157,9 @@ final class $vectortype$ extends $abstractvectortype$ { $bitsvectortype$ wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); if (!wrap) { - $bitsvectortype$ wrappedLower = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask<$Boxbitstype$> mask = iota.compare(VectorOperators.EQ, iota); - wrapped = wrappedLower.blend(wrapped, mask); + $bitsvectortype$ wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); + VectorMask<$Boxbitstype$> mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); } #if[!FP] return wrapped.toShuffle(); @@ -1097,9 +1097,8 @@ final class $vectortype$ extends $abstractvectortype$ { // Shuffle static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> { - static final IntUnaryOperator IDENTITY = i -> i; static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM - static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM + static final Class<$Boxbitstype$> ETYPE = $bitstype$.class; // used by the JVM $shuffletype$($bitstype$[] indices) { super(indices); From 060554a9fca08091ed530819c82ce2995b5c25e7 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Mon, 20 Mar 2023 03:33:20 +0800 Subject: [PATCH 07/13] fix Matcher::vector_needs_load_shuffle --- src/hotspot/cpu/x86/x86.ad | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index e7c6cb6d26d19..8faf95440b956 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2173,14 +2173,15 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* // Do Vector::rearrange needs preparation of the shuffle argument const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { - if (elem_bt == T_BYTE || elem_bt == T_INT || elem_bt == T_FLOAT) { + if (elem_bt == T_BYTE) { return false; } else if (elem_bt == T_SHORT) { return !VM_Version::supports_avx512bw(); + } else if (elem_bt == T_INT || elem_bt == T_FLOAT) { + return !VM_Version::supports_avx2(); + } else { // T_LONG || T_DOUBLE + return vlen < 8 && !VM_Version::supports_avx512vl(); } - - // T_LONG || T_DOUBLE - return !VM_Version::supports_evex(); } MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) { @@ -8475,7 +8476,7 @@ instruct rearrangeB_evex_vbmi(vec dst, vec src, vec shuffle) %{ instruct loadShuffleS(vec dst, vec src, vec vtmp) %{ predicate(Matcher::vector_element_basic_type(n) == T_SHORT && - Matcher::vector_length(n) <= 16 && !VM_Version::supports_avx512bw()); // NB! aligned with rearrangeS + !VM_Version::supports_avx512bw()); match(Set dst (VectorLoadShuffle src)); effect(TEMP dst, TEMP vtmp); format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %} From 05eb7be0370762c59b1b01a5d3fcf0c666f6ec5a Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Tue, 21 Mar 2023 23:32:27 +0800 Subject: [PATCH 08/13] clean up --- .../jdk/incubator/vector/AbstractShuffle.java | 24 ++++++++ .../jdk/incubator/vector/AbstractVector.java | 22 ++++++- .../jdk/incubator/vector/Byte128Vector.java | 42 ------------- .../jdk/incubator/vector/Byte256Vector.java | 42 ------------- .../jdk/incubator/vector/Byte512Vector.java | 42 ------------- .../jdk/incubator/vector/Byte64Vector.java | 42 ------------- .../jdk/incubator/vector/ByteMaxVector.java | 42 ------------- .../jdk/incubator/vector/Double128Vector.java | 44 -------------- .../jdk/incubator/vector/Double256Vector.java | 44 -------------- .../jdk/incubator/vector/Double512Vector.java | 44 -------------- .../jdk/incubator/vector/Double64Vector.java | 44 -------------- .../jdk/incubator/vector/DoubleMaxVector.java | 44 -------------- .../jdk/incubator/vector/Float128Vector.java | 44 -------------- .../jdk/incubator/vector/Float256Vector.java | 44 -------------- .../jdk/incubator/vector/Float512Vector.java | 44 -------------- .../jdk/incubator/vector/Float64Vector.java | 44 -------------- .../jdk/incubator/vector/FloatMaxVector.java | 44 -------------- .../jdk/incubator/vector/Int128Vector.java | 42 ------------- .../jdk/incubator/vector/Int256Vector.java | 42 ------------- .../jdk/incubator/vector/Int512Vector.java | 42 ------------- .../jdk/incubator/vector/Int64Vector.java | 42 ------------- .../jdk/incubator/vector/IntMaxVector.java | 42 ------------- .../jdk/incubator/vector/IntVector.java | 6 +- .../jdk/incubator/vector/Long128Vector.java | 42 ------------- .../jdk/incubator/vector/Long256Vector.java | 42 ------------- .../jdk/incubator/vector/Long512Vector.java | 42 ------------- .../jdk/incubator/vector/Long64Vector.java | 42 ------------- .../jdk/incubator/vector/LongMaxVector.java | 42 ------------- .../jdk/incubator/vector/LongVector.java | 6 +- .../jdk/incubator/vector/Short128Vector.java | 42 ------------- .../jdk/incubator/vector/Short256Vector.java | 42 ------------- .../jdk/incubator/vector/Short512Vector.java | 42 ------------- .../jdk/incubator/vector/Short64Vector.java | 42 ------------- .../jdk/incubator/vector/ShortMaxVector.java | 42 ------------- .../incubator/vector/X-Vector.java.template | 6 +- .../vector/X-VectorBits.java.template | 59 ------------------- 36 files changed, 54 insertions(+), 1349 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java index 26eddb2d584d0..20f4fa52a0f3e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java @@ -108,6 +108,30 @@ public final VectorMask laneIsValid() { .cast(vspecies()); } + @ForceInline + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + public final VectorShuffle rearrange(VectorShuffle shuffle) { + Vector v = toBitsVector(); + return (VectorShuffle) v.rearrange(shuffle.cast(vspecies().asIntegral())) + .toShuffle() + .cast(vspecies()); + } + + @ForceInline + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + public final VectorShuffle wrapIndexes() { + Vector v = toBitsVector(); + if ((length() & (length() - 1)) == 0) { + v = v.lanewise(VectorOperators.AND, length() - 1); + } else { + v = v.blend(v.lanewise(VectorOperators.ADD, length()), + v.compare(VectorOperators.LT, 0)); + } + return (VectorShuffle) v.toShuffle().cast(vspecies()); + } + @Override @ForceInline @SuppressWarnings("unchecked") diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java index 408c67ec9ec57..c53e68e5a3cf7 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java @@ -190,7 +190,27 @@ ByteVector asByteVectorRawTemplate() { abstract AbstractShuffle iotaShuffle(); - abstract AbstractShuffle iotaShuffle(int start, int step, boolean wrap); + @ForceInline + @SuppressWarnings({"rawtypes", "unchecked"}) + final AbstractShuffle iotaShuffle(int start, int step, boolean wrap) { + if ((length() & (length() - 1)) != 0) { + return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, length()))) + : shuffleFromOp(i -> i * step + start); + } + + AbstractSpecies species = vspecies().asIntegral(); + Vector iota = species.iota(); + iota = iota.lanewise(VectorOperators.MUL, step) + .lanewise(VectorOperators.ADD, start); + Vector wrapped = iota.lanewise(VectorOperators.AND, length() - 1); + + if (!wrap) { + Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, length()); + VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); + wrapped = wrappedEx.blend(wrapped, mask); + } + return (AbstractShuffle) wrapped.toShuffle().cast(vspecies()); + } abstract AbstractShuffle shuffleFromArray(int[] indexes, int i); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java index c555d684055fc..792ecf19f2c2b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java @@ -141,27 +141,6 @@ Byte128Mask maskFromArray(boolean[] bits) { @ForceInline Byte128Shuffle iotaShuffle() { return Byte128Shuffle.IOTA; } - @ForceInline - Byte128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Byte128Vector iota = Byte128Shuffle.IOTA.toBitsVector(); - ByteVector.ByteSpecies species = Byte128Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Byte128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Byte128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Byte128Shuffle shuffleFromArray(int[] indices, int i) { return new Byte128Shuffle(indices, i); } @@ -891,27 +870,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset + species.length() * 3); } - @ForceInline - @Override - public Byte128Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Byte128Shuffle wrapIndexes() { - Byte128Vector v = toBitsVector(); - ByteVector.ByteSpecies species = Byte128Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static byte[] prepare(int[] indices, int offset) { byte[] a = new byte[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java index 4686f441ec799..733e5cc97b75a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java @@ -141,27 +141,6 @@ Byte256Mask maskFromArray(boolean[] bits) { @ForceInline Byte256Shuffle iotaShuffle() { return Byte256Shuffle.IOTA; } - @ForceInline - Byte256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Byte256Vector iota = Byte256Shuffle.IOTA.toBitsVector(); - ByteVector.ByteSpecies species = Byte256Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Byte256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Byte256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Byte256Shuffle shuffleFromArray(int[] indices, int i) { return new Byte256Shuffle(indices, i); } @@ -923,27 +902,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset + species.length() * 3); } - @ForceInline - @Override - public Byte256Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Byte256Shuffle wrapIndexes() { - Byte256Vector v = toBitsVector(); - ByteVector.ByteSpecies species = Byte256Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static byte[] prepare(int[] indices, int offset) { byte[] a = new byte[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java index b99ae65020d75..615cdc324ae6e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java @@ -141,27 +141,6 @@ Byte512Mask maskFromArray(boolean[] bits) { @ForceInline Byte512Shuffle iotaShuffle() { return Byte512Shuffle.IOTA; } - @ForceInline - Byte512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Byte512Vector iota = Byte512Shuffle.IOTA.toBitsVector(); - ByteVector.ByteSpecies species = Byte512Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Byte512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Byte512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Byte512Shuffle shuffleFromArray(int[] indices, int i) { return new Byte512Shuffle(indices, i); } @@ -987,27 +966,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset + species.length() * 3); } - @ForceInline - @Override - public Byte512Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Byte512Shuffle wrapIndexes() { - Byte512Vector v = toBitsVector(); - ByteVector.ByteSpecies species = Byte512Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static byte[] prepare(int[] indices, int offset) { byte[] a = new byte[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java index fff34a98ea99f..43069537615d2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java @@ -141,27 +141,6 @@ Byte64Mask maskFromArray(boolean[] bits) { @ForceInline Byte64Shuffle iotaShuffle() { return Byte64Shuffle.IOTA; } - @ForceInline - Byte64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Byte64Vector iota = Byte64Shuffle.IOTA.toBitsVector(); - ByteVector.ByteSpecies species = Byte64Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Byte64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Byte64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Byte64Shuffle shuffleFromArray(int[] indices, int i) { return new Byte64Shuffle(indices, i); } @@ -875,27 +854,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset + species.length() * 3); } - @ForceInline - @Override - public Byte64Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Byte64Shuffle wrapIndexes() { - Byte64Vector v = toBitsVector(); - ByteVector.ByteSpecies species = Byte64Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static byte[] prepare(int[] indices, int offset) { byte[] a = new byte[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java index e3cbabd18f101..52ecc0dbbabd3 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java @@ -141,27 +141,6 @@ ByteMaxMask maskFromArray(boolean[] bits) { @ForceInline ByteMaxShuffle iotaShuffle() { return ByteMaxShuffle.IOTA; } - @ForceInline - ByteMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - ByteMaxVector iota = ByteMaxShuffle.IOTA.toBitsVector(); - ByteVector.ByteSpecies species = ByteMaxVector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - ByteMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - ByteMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline ByteMaxShuffle shuffleFromArray(int[] indices, int i) { return new ByteMaxShuffle(indices, i); } @@ -861,27 +840,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset + species.length() * 3); } - @ForceInline - @Override - public ByteMaxShuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public ByteMaxShuffle wrapIndexes() { - ByteMaxVector v = toBitsVector(); - ByteVector.ByteSpecies species = ByteMaxVector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static byte[] prepare(int[] indices, int offset) { byte[] a = new byte[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java index e676cd2e11d49..27338df235ed8 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java @@ -141,27 +141,6 @@ Double128Mask maskFromArray(boolean[] bits) { @ForceInline Double128Shuffle iotaShuffle() { return Double128Shuffle.IOTA; } - @ForceInline - Double128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Long128Vector iota = Double128Shuffle.IOTA.toBitsVector(); - LongVector.LongSpecies species = Long128Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Long128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Long128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toFPShuffle(); - } - @Override @ForceInline Double128Shuffle shuffleFromArray(int[] indices, int i) { return new Double128Shuffle(indices, i); } @@ -847,29 +826,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset); } - @ForceInline - @Override - public Double128Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector() - .rearrange(shuffle.cast(Long128Vector.VSPECIES)) - .toFPShuffle(); - } - - @ForceInline - @Override - public Double128Shuffle wrapIndexes() { - Long128Vector v = toBitsVector(); - LongVector.LongSpecies species = Long128Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toFPShuffle(); - } - private static long[] prepare(int[] indices, int offset) { long[] a = new long[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java index e231b8807dee8..5bc912f05a0dd 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java @@ -141,27 +141,6 @@ Double256Mask maskFromArray(boolean[] bits) { @ForceInline Double256Shuffle iotaShuffle() { return Double256Shuffle.IOTA; } - @ForceInline - Double256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Long256Vector iota = Double256Shuffle.IOTA.toBitsVector(); - LongVector.LongSpecies species = Long256Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Long256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Long256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toFPShuffle(); - } - @Override @ForceInline Double256Shuffle shuffleFromArray(int[] indices, int i) { return new Double256Shuffle(indices, i); } @@ -851,29 +830,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset); } - @ForceInline - @Override - public Double256Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector() - .rearrange(shuffle.cast(Long256Vector.VSPECIES)) - .toFPShuffle(); - } - - @ForceInline - @Override - public Double256Shuffle wrapIndexes() { - Long256Vector v = toBitsVector(); - LongVector.LongSpecies species = Long256Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toFPShuffle(); - } - private static long[] prepare(int[] indices, int offset) { long[] a = new long[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java index 60fec60f825ca..adb57ebbb117d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java @@ -141,27 +141,6 @@ Double512Mask maskFromArray(boolean[] bits) { @ForceInline Double512Shuffle iotaShuffle() { return Double512Shuffle.IOTA; } - @ForceInline - Double512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Long512Vector iota = Double512Shuffle.IOTA.toBitsVector(); - LongVector.LongSpecies species = Long512Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Long512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Long512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toFPShuffle(); - } - @Override @ForceInline Double512Shuffle shuffleFromArray(int[] indices, int i) { return new Double512Shuffle(indices, i); } @@ -859,29 +838,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset); } - @ForceInline - @Override - public Double512Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector() - .rearrange(shuffle.cast(Long512Vector.VSPECIES)) - .toFPShuffle(); - } - - @ForceInline - @Override - public Double512Shuffle wrapIndexes() { - Long512Vector v = toBitsVector(); - LongVector.LongSpecies species = Long512Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toFPShuffle(); - } - private static long[] prepare(int[] indices, int offset) { long[] a = new long[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java index 4a203f31c9f57..b80beacf8f641 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java @@ -141,27 +141,6 @@ Double64Mask maskFromArray(boolean[] bits) { @ForceInline Double64Shuffle iotaShuffle() { return Double64Shuffle.IOTA; } - @ForceInline - Double64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Long64Vector iota = Double64Shuffle.IOTA.toBitsVector(); - LongVector.LongSpecies species = Long64Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Long64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Long64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toFPShuffle(); - } - @Override @ForceInline Double64Shuffle shuffleFromArray(int[] indices, int i) { return new Double64Shuffle(indices, i); } @@ -839,29 +818,6 @@ public void intoArray(int[] a, int offset) { a[offset] = laneSource(0); } - @ForceInline - @Override - public Double64Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector() - .rearrange(shuffle.cast(Long64Vector.VSPECIES)) - .toFPShuffle(); - } - - @ForceInline - @Override - public Double64Shuffle wrapIndexes() { - Long64Vector v = toBitsVector(); - LongVector.LongSpecies species = Long64Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toFPShuffle(); - } - private static long[] prepare(int[] indices, int offset) { long[] a = new long[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java index af64b577f73cb..f35a238ac300a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java @@ -141,27 +141,6 @@ DoubleMaxMask maskFromArray(boolean[] bits) { @ForceInline DoubleMaxShuffle iotaShuffle() { return DoubleMaxShuffle.IOTA; } - @ForceInline - DoubleMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - LongMaxVector iota = DoubleMaxShuffle.IOTA.toBitsVector(); - LongVector.LongSpecies species = LongMaxVector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - LongMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - LongMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toFPShuffle(); - } - @Override @ForceInline DoubleMaxShuffle shuffleFromArray(int[] indices, int i) { return new DoubleMaxShuffle(indices, i); } @@ -844,29 +823,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset); } - @ForceInline - @Override - public DoubleMaxShuffle rearrange(VectorShuffle shuffle) { - return toBitsVector() - .rearrange(shuffle.cast(LongMaxVector.VSPECIES)) - .toFPShuffle(); - } - - @ForceInline - @Override - public DoubleMaxShuffle wrapIndexes() { - LongMaxVector v = toBitsVector(); - LongVector.LongSpecies species = LongMaxVector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toFPShuffle(); - } - private static long[] prepare(int[] indices, int offset) { long[] a = new long[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java index 5197500caeec9..8aa95de599ac6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java @@ -141,27 +141,6 @@ Float128Mask maskFromArray(boolean[] bits) { @ForceInline Float128Shuffle iotaShuffle() { return Float128Shuffle.IOTA; } - @ForceInline - Float128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Int128Vector iota = Float128Shuffle.IOTA.toBitsVector(); - IntVector.IntSpecies species = Int128Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Int128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Int128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toFPShuffle(); - } - @Override @ForceInline Float128Shuffle shuffleFromArray(int[] indices, int i) { return new Float128Shuffle(indices, i); } @@ -845,29 +824,6 @@ public void intoArray(int[] a, int offset) { toBitsVector().intoArray(a, offset); } - @ForceInline - @Override - public Float128Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector() - .rearrange(shuffle.cast(Int128Vector.VSPECIES)) - .toFPShuffle(); - } - - @ForceInline - @Override - public Float128Shuffle wrapIndexes() { - Int128Vector v = toBitsVector(); - IntVector.IntSpecies species = Int128Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toFPShuffle(); - } - private static int[] prepare(int[] indices, int offset) { int[] a = new int[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java index 0fc7489f35ca2..7f90068ebee93 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java @@ -141,27 +141,6 @@ Float256Mask maskFromArray(boolean[] bits) { @ForceInline Float256Shuffle iotaShuffle() { return Float256Shuffle.IOTA; } - @ForceInline - Float256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Int256Vector iota = Float256Shuffle.IOTA.toBitsVector(); - IntVector.IntSpecies species = Int256Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Int256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Int256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toFPShuffle(); - } - @Override @ForceInline Float256Shuffle shuffleFromArray(int[] indices, int i) { return new Float256Shuffle(indices, i); } @@ -853,29 +832,6 @@ public void intoArray(int[] a, int offset) { toBitsVector().intoArray(a, offset); } - @ForceInline - @Override - public Float256Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector() - .rearrange(shuffle.cast(Int256Vector.VSPECIES)) - .toFPShuffle(); - } - - @ForceInline - @Override - public Float256Shuffle wrapIndexes() { - Int256Vector v = toBitsVector(); - IntVector.IntSpecies species = Int256Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toFPShuffle(); - } - private static int[] prepare(int[] indices, int offset) { int[] a = new int[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java index 3431622349d0d..2978f48691de7 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java @@ -141,27 +141,6 @@ Float512Mask maskFromArray(boolean[] bits) { @ForceInline Float512Shuffle iotaShuffle() { return Float512Shuffle.IOTA; } - @ForceInline - Float512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Int512Vector iota = Float512Shuffle.IOTA.toBitsVector(); - IntVector.IntSpecies species = Int512Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Int512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Int512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toFPShuffle(); - } - @Override @ForceInline Float512Shuffle shuffleFromArray(int[] indices, int i) { return new Float512Shuffle(indices, i); } @@ -869,29 +848,6 @@ public void intoArray(int[] a, int offset) { toBitsVector().intoArray(a, offset); } - @ForceInline - @Override - public Float512Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector() - .rearrange(shuffle.cast(Int512Vector.VSPECIES)) - .toFPShuffle(); - } - - @ForceInline - @Override - public Float512Shuffle wrapIndexes() { - Int512Vector v = toBitsVector(); - IntVector.IntSpecies species = Int512Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toFPShuffle(); - } - private static int[] prepare(int[] indices, int offset) { int[] a = new int[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java index 390c50a54c39b..3d016c116c6c4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java @@ -141,27 +141,6 @@ Float64Mask maskFromArray(boolean[] bits) { @ForceInline Float64Shuffle iotaShuffle() { return Float64Shuffle.IOTA; } - @ForceInline - Float64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Int64Vector iota = Float64Shuffle.IOTA.toBitsVector(); - IntVector.IntSpecies species = Int64Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Int64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Int64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toFPShuffle(); - } - @Override @ForceInline Float64Shuffle shuffleFromArray(int[] indices, int i) { return new Float64Shuffle(indices, i); } @@ -841,29 +820,6 @@ public void intoArray(int[] a, int offset) { toBitsVector().intoArray(a, offset); } - @ForceInline - @Override - public Float64Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector() - .rearrange(shuffle.cast(Int64Vector.VSPECIES)) - .toFPShuffle(); - } - - @ForceInline - @Override - public Float64Shuffle wrapIndexes() { - Int64Vector v = toBitsVector(); - IntVector.IntSpecies species = Int64Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toFPShuffle(); - } - private static int[] prepare(int[] indices, int offset) { int[] a = new int[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java index 3aabf8e34cfed..785b5e32a08b0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java @@ -141,27 +141,6 @@ FloatMaxMask maskFromArray(boolean[] bits) { @ForceInline FloatMaxShuffle iotaShuffle() { return FloatMaxShuffle.IOTA; } - @ForceInline - FloatMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - IntMaxVector iota = FloatMaxShuffle.IOTA.toBitsVector(); - IntVector.IntSpecies species = IntMaxVector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - IntMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - IntMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toFPShuffle(); - } - @Override @ForceInline FloatMaxShuffle shuffleFromArray(int[] indices, int i) { return new FloatMaxShuffle(indices, i); } @@ -838,29 +817,6 @@ public void intoArray(int[] a, int offset) { toBitsVector().intoArray(a, offset); } - @ForceInline - @Override - public FloatMaxShuffle rearrange(VectorShuffle shuffle) { - return toBitsVector() - .rearrange(shuffle.cast(IntMaxVector.VSPECIES)) - .toFPShuffle(); - } - - @ForceInline - @Override - public FloatMaxShuffle wrapIndexes() { - IntMaxVector v = toBitsVector(); - IntVector.IntSpecies species = IntMaxVector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toFPShuffle(); - } - private static int[] prepare(int[] indices, int offset) { int[] a = new int[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java index e93fb5c65bbc6..afebe84f779db 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java @@ -141,27 +141,6 @@ Int128Mask maskFromArray(boolean[] bits) { @ForceInline Int128Shuffle iotaShuffle() { return Int128Shuffle.IOTA; } - @ForceInline - Int128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Int128Vector iota = Int128Shuffle.IOTA.toBitsVector(); - IntVector.IntSpecies species = Int128Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Int128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Int128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Int128Shuffle shuffleFromArray(int[] indices, int i) { return new Int128Shuffle(indices, i); } @@ -859,27 +838,6 @@ public void intoArray(int[] a, int offset) { toBitsVector().intoArray(a, offset); } - @ForceInline - @Override - public Int128Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Int128Shuffle wrapIndexes() { - Int128Vector v = toBitsVector(); - IntVector.IntSpecies species = Int128Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static int[] prepare(int[] indices, int offset) { int[] a = new int[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java index 220b5179a9f4d..7e5e2b8171c11 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java @@ -141,27 +141,6 @@ Int256Mask maskFromArray(boolean[] bits) { @ForceInline Int256Shuffle iotaShuffle() { return Int256Shuffle.IOTA; } - @ForceInline - Int256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Int256Vector iota = Int256Shuffle.IOTA.toBitsVector(); - IntVector.IntSpecies species = Int256Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Int256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Int256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Int256Shuffle shuffleFromArray(int[] indices, int i) { return new Int256Shuffle(indices, i); } @@ -867,27 +846,6 @@ public void intoArray(int[] a, int offset) { toBitsVector().intoArray(a, offset); } - @ForceInline - @Override - public Int256Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Int256Shuffle wrapIndexes() { - Int256Vector v = toBitsVector(); - IntVector.IntSpecies species = Int256Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static int[] prepare(int[] indices, int offset) { int[] a = new int[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java index fd3f9b2c68154..715ce7e2ab746 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java @@ -141,27 +141,6 @@ Int512Mask maskFromArray(boolean[] bits) { @ForceInline Int512Shuffle iotaShuffle() { return Int512Shuffle.IOTA; } - @ForceInline - Int512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Int512Vector iota = Int512Shuffle.IOTA.toBitsVector(); - IntVector.IntSpecies species = Int512Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Int512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Int512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Int512Shuffle shuffleFromArray(int[] indices, int i) { return new Int512Shuffle(indices, i); } @@ -883,27 +862,6 @@ public void intoArray(int[] a, int offset) { toBitsVector().intoArray(a, offset); } - @ForceInline - @Override - public Int512Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Int512Shuffle wrapIndexes() { - Int512Vector v = toBitsVector(); - IntVector.IntSpecies species = Int512Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static int[] prepare(int[] indices, int offset) { int[] a = new int[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java index 384fc1dd9b2c0..67e538fdea6bf 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java @@ -141,27 +141,6 @@ Int64Mask maskFromArray(boolean[] bits) { @ForceInline Int64Shuffle iotaShuffle() { return Int64Shuffle.IOTA; } - @ForceInline - Int64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Int64Vector iota = Int64Shuffle.IOTA.toBitsVector(); - IntVector.IntSpecies species = Int64Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Int64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Int64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Int64Shuffle shuffleFromArray(int[] indices, int i) { return new Int64Shuffle(indices, i); } @@ -855,27 +834,6 @@ public void intoArray(int[] a, int offset) { toBitsVector().intoArray(a, offset); } - @ForceInline - @Override - public Int64Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Int64Shuffle wrapIndexes() { - Int64Vector v = toBitsVector(); - IntVector.IntSpecies species = Int64Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static int[] prepare(int[] indices, int offset) { int[] a = new int[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java index 35c26e00fbbdf..0f15fc0a58589 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java @@ -141,27 +141,6 @@ IntMaxMask maskFromArray(boolean[] bits) { @ForceInline IntMaxShuffle iotaShuffle() { return IntMaxShuffle.IOTA; } - @ForceInline - IntMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - IntMaxVector iota = IntMaxShuffle.IOTA.toBitsVector(); - IntVector.IntSpecies species = IntMaxVector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - IntMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - IntMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline IntMaxShuffle shuffleFromArray(int[] indices, int i) { return new IntMaxShuffle(indices, i); } @@ -864,27 +843,6 @@ public void intoArray(int[] a, int offset) { toBitsVector().intoArray(a, offset); } - @ForceInline - @Override - public IntMaxShuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public IntMaxShuffle wrapIndexes() { - IntMaxVector v = toBitsVector(); - IntVector.IntSpecies species = IntMaxVector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static int[] prepare(int[] indices, int offset) { int[] a = new int[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java index aceaff4ae8615..6adf9067443aa 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java @@ -2486,11 +2486,11 @@ shuffleType, int.class, length(), IntVector::toShuffle0); } - abstract VectorShuffle toFPShuffle(); + abstract AbstractShuffle toFPShuffle(); @ForceInline private final - VectorShuffle toFPShuffle0(VectorSpecies dsp) { + AbstractShuffle toFPShuffle0(VectorSpecies dsp) { int[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2501,7 +2501,7 @@ VectorShuffle toFPShuffle0(VectorSpecies dsp) { @ForceInline final - VectorShuffle toFPShuffleTemplate(Class shuffleType, FloatVector.FloatSpecies dsp) { + AbstractShuffle toFPShuffleTemplate(Class shuffleType, FloatVector.FloatSpecies dsp) { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), int.class, length(), shuffleType, int.class, length(), diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java index 51744b8d88c29..7424dc6702a28 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java @@ -136,27 +136,6 @@ Long128Mask maskFromArray(boolean[] bits) { @ForceInline Long128Shuffle iotaShuffle() { return Long128Shuffle.IOTA; } - @ForceInline - Long128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Long128Vector iota = Long128Shuffle.IOTA.toBitsVector(); - LongVector.LongSpecies species = Long128Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Long128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Long128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Long128Shuffle shuffleFromArray(int[] indices, int i) { return new Long128Shuffle(indices, i); } @@ -851,27 +830,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset); } - @ForceInline - @Override - public Long128Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Long128Shuffle wrapIndexes() { - Long128Vector v = toBitsVector(); - LongVector.LongSpecies species = Long128Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static long[] prepare(int[] indices, int offset) { long[] a = new long[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java index a4fb5ff5e8a6d..8c16913b8e21c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java @@ -136,27 +136,6 @@ Long256Mask maskFromArray(boolean[] bits) { @ForceInline Long256Shuffle iotaShuffle() { return Long256Shuffle.IOTA; } - @ForceInline - Long256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Long256Vector iota = Long256Shuffle.IOTA.toBitsVector(); - LongVector.LongSpecies species = Long256Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Long256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Long256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Long256Shuffle shuffleFromArray(int[] indices, int i) { return new Long256Shuffle(indices, i); } @@ -855,27 +834,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset); } - @ForceInline - @Override - public Long256Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Long256Shuffle wrapIndexes() { - Long256Vector v = toBitsVector(); - LongVector.LongSpecies species = Long256Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static long[] prepare(int[] indices, int offset) { long[] a = new long[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java index 8ad3a5a6fe33d..141617cc40f3e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java @@ -136,27 +136,6 @@ Long512Mask maskFromArray(boolean[] bits) { @ForceInline Long512Shuffle iotaShuffle() { return Long512Shuffle.IOTA; } - @ForceInline - Long512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Long512Vector iota = Long512Shuffle.IOTA.toBitsVector(); - LongVector.LongSpecies species = Long512Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Long512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Long512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Long512Shuffle shuffleFromArray(int[] indices, int i) { return new Long512Shuffle(indices, i); } @@ -863,27 +842,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset); } - @ForceInline - @Override - public Long512Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Long512Shuffle wrapIndexes() { - Long512Vector v = toBitsVector(); - LongVector.LongSpecies species = Long512Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static long[] prepare(int[] indices, int offset) { long[] a = new long[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java index 0ee353d60d62a..d201e73cb6af0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java @@ -136,27 +136,6 @@ Long64Mask maskFromArray(boolean[] bits) { @ForceInline Long64Shuffle iotaShuffle() { return Long64Shuffle.IOTA; } - @ForceInline - Long64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Long64Vector iota = Long64Shuffle.IOTA.toBitsVector(); - LongVector.LongSpecies species = Long64Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Long64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Long64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Long64Shuffle shuffleFromArray(int[] indices, int i) { return new Long64Shuffle(indices, i); } @@ -843,27 +822,6 @@ public void intoArray(int[] a, int offset) { a[offset] = laneSource(0); } - @ForceInline - @Override - public Long64Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Long64Shuffle wrapIndexes() { - Long64Vector v = toBitsVector(); - LongVector.LongSpecies species = Long64Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static long[] prepare(int[] indices, int offset) { long[] a = new long[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java index 72bee5c553592..1731c38e7d745 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java @@ -136,27 +136,6 @@ LongMaxMask maskFromArray(boolean[] bits) { @ForceInline LongMaxShuffle iotaShuffle() { return LongMaxShuffle.IOTA; } - @ForceInline - LongMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - LongMaxVector iota = LongMaxShuffle.IOTA.toBitsVector(); - LongVector.LongSpecies species = LongMaxVector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - LongMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - LongMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline LongMaxShuffle shuffleFromArray(int[] indices, int i) { return new LongMaxShuffle(indices, i); } @@ -849,27 +828,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset); } - @ForceInline - @Override - public LongMaxShuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public LongMaxShuffle wrapIndexes() { - LongMaxVector v = toBitsVector(); - LongVector.LongSpecies species = LongMaxVector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static long[] prepare(int[] indices, int offset) { long[] a = new long[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java index 8cf34cca675e8..9dbb1bdd3a9fa 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java @@ -2352,11 +2352,11 @@ shuffleType, long.class, length(), LongVector::toShuffle0); } - abstract VectorShuffle toFPShuffle(); + abstract AbstractShuffle toFPShuffle(); @ForceInline private final - VectorShuffle toFPShuffle0(VectorSpecies dsp) { + AbstractShuffle toFPShuffle0(VectorSpecies dsp) { long[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2367,7 +2367,7 @@ VectorShuffle toFPShuffle0(VectorSpecies dsp) { @ForceInline final - VectorShuffle toFPShuffleTemplate(Class shuffleType, DoubleVector.DoubleSpecies dsp) { + AbstractShuffle toFPShuffleTemplate(Class shuffleType, DoubleVector.DoubleSpecies dsp) { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), long.class, length(), shuffleType, long.class, length(), diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java index c7f65e4848666..085275bfaf7db 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java @@ -141,27 +141,6 @@ Short128Mask maskFromArray(boolean[] bits) { @ForceInline Short128Shuffle iotaShuffle() { return Short128Shuffle.IOTA; } - @ForceInline - Short128Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Short128Vector iota = Short128Shuffle.IOTA.toBitsVector(); - ShortVector.ShortSpecies species = Short128Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Short128Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Short128Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Short128Shuffle shuffleFromArray(int[] indices, int i) { return new Short128Shuffle(indices, i); } @@ -869,27 +848,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset + species.length()); } - @ForceInline - @Override - public Short128Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Short128Shuffle wrapIndexes() { - Short128Vector v = toBitsVector(); - ShortVector.ShortSpecies species = Short128Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static short[] prepare(int[] indices, int offset) { short[] a = new short[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java index 9c42fb57b45e9..ed380d4db7bc1 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java @@ -141,27 +141,6 @@ Short256Mask maskFromArray(boolean[] bits) { @ForceInline Short256Shuffle iotaShuffle() { return Short256Shuffle.IOTA; } - @ForceInline - Short256Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Short256Vector iota = Short256Shuffle.IOTA.toBitsVector(); - ShortVector.ShortSpecies species = Short256Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Short256Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Short256Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Short256Shuffle shuffleFromArray(int[] indices, int i) { return new Short256Shuffle(indices, i); } @@ -885,27 +864,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset + species.length()); } - @ForceInline - @Override - public Short256Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Short256Shuffle wrapIndexes() { - Short256Vector v = toBitsVector(); - ShortVector.ShortSpecies species = Short256Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static short[] prepare(int[] indices, int offset) { short[] a = new short[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java index f1f9fe96314a6..2b88778403381 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java @@ -141,27 +141,6 @@ Short512Mask maskFromArray(boolean[] bits) { @ForceInline Short512Shuffle iotaShuffle() { return Short512Shuffle.IOTA; } - @ForceInline - Short512Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Short512Vector iota = Short512Shuffle.IOTA.toBitsVector(); - ShortVector.ShortSpecies species = Short512Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Short512Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Short512Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Short512Shuffle shuffleFromArray(int[] indices, int i) { return new Short512Shuffle(indices, i); } @@ -917,27 +896,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset + species.length()); } - @ForceInline - @Override - public Short512Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Short512Shuffle wrapIndexes() { - Short512Vector v = toBitsVector(); - ShortVector.ShortSpecies species = Short512Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static short[] prepare(int[] indices, int offset) { short[] a = new short[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java index 9995202f975a5..cdc5a500c5f74 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java @@ -141,27 +141,6 @@ Short64Mask maskFromArray(boolean[] bits) { @ForceInline Short64Shuffle iotaShuffle() { return Short64Shuffle.IOTA; } - @ForceInline - Short64Shuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - Short64Vector iota = Short64Shuffle.IOTA.toBitsVector(); - ShortVector.ShortSpecies species = Short64Vector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - Short64Vector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - Short64Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline Short64Shuffle shuffleFromArray(int[] indices, int i) { return new Short64Shuffle(indices, i); } @@ -861,27 +840,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset + species.length()); } - @ForceInline - @Override - public Short64Shuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public Short64Shuffle wrapIndexes() { - Short64Vector v = toBitsVector(); - ShortVector.ShortSpecies species = Short64Vector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static short[] prepare(int[] indices, int offset) { short[] a = new short[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java index 6386c3dedc303..2446e386e7951 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java @@ -141,27 +141,6 @@ ShortMaxMask maskFromArray(boolean[] bits) { @ForceInline ShortMaxShuffle iotaShuffle() { return ShortMaxShuffle.IOTA; } - @ForceInline - ShortMaxShuffle iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - ShortMaxVector iota = ShortMaxShuffle.IOTA.toBitsVector(); - ShortVector.ShortSpecies species = ShortMaxVector.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - ShortMaxVector wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - ShortMaxVector wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } - return wrapped.toShuffle(); - } - @Override @ForceInline ShortMaxShuffle shuffleFromArray(int[] indices, int i) { return new ShortMaxShuffle(indices, i); } @@ -855,27 +834,6 @@ public void intoArray(int[] a, int offset) { .intoArray(a, offset + species.length()); } - @ForceInline - @Override - public ShortMaxShuffle rearrange(VectorShuffle shuffle) { - return toBitsVector().rearrange(shuffle).toShuffle(); - } - - @ForceInline - @Override - public ShortMaxShuffle wrapIndexes() { - ShortMaxVector v = toBitsVector(); - ShortVector.ShortSpecies species = ShortMaxVector.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask neg = v.compare(VectorOperators.LT, 0); - Vector adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } - return v.toShuffle(); - } - private static short[] prepare(int[] indices, int offset) { short[] a = new short[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template index ce90fc93845d2..4f6e7355680b8 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template @@ -2881,11 +2881,11 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { #end[!FP] #if[intOrLong] - abstract VectorShuffle<$Boxfptype$> toFPShuffle(); + abstract AbstractShuffle<$Boxfptype$> toFPShuffle(); @ForceInline private final - VectorShuffle<$Boxfptype$> toFPShuffle0(VectorSpecies<$Boxfptype$> dsp) { + AbstractShuffle<$Boxfptype$> toFPShuffle0(VectorSpecies<$Boxfptype$> dsp) { $type$[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2896,7 +2896,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { @ForceInline final - VectorShuffle<$Boxfptype$> toFPShuffleTemplate(Class shuffleType, $Fptype$Vector.$Fptype$Species dsp) { + AbstractShuffle<$Boxfptype$> toFPShuffleTemplate(Class shuffleType, $Fptype$Vector.$Fptype$Species dsp) { return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, getClass(), $type$.class, length(), shuffleType, $type$.class, length(), diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template index 0a31e32fb27cf..231f05c9faf27 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template @@ -143,32 +143,6 @@ final class $vectortype$ extends $abstractvectortype$ { @ForceInline $shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; } - @ForceInline - $shuffletype$ iotaShuffle(int start, int step, boolean wrap) { - if ((VLENGTH & (VLENGTH - 1)) != 0) { - return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, VLENGTH))) - : shuffleFromOp(i -> i * step + start); - } - - $bitsvectortype$ iota = $shuffletype$.IOTA.toBitsVector(); - $Bitstype$Vector.$Bitstype$Species species = $bitsvectortype$.VSPECIES; - iota = iota.lanewise(VectorOperators.MUL, species.broadcast(step)) - .lanewise(VectorOperators.ADD, species.broadcast(start)); - $bitsvectortype$ wrapped = iota.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - - if (!wrap) { - $bitsvectortype$ wrappedEx = wrapped.lanewise(VectorOperators.SUB, species.broadcast(VLENGTH)); - VectorMask<$Boxbitstype$> mask = wrapped.compare(VectorOperators.EQ, iota); - wrapped = wrappedEx.blend(wrapped, mask); - } -#if[!FP] - return wrapped.toShuffle(); -#end[!FP] -#if[FP] - return wrapped.toFPShuffle(); -#end[FP] - } - @Override @ForceInline $shuffletype$ shuffleFromArray(int[] indices, int i) { return new $shuffletype$(indices, i); } @@ -1201,39 +1175,6 @@ final class $vectortype$ extends $abstractvectortype$ { #end[longOrDouble] } - @ForceInline - @Override - public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) { -#if[!FP] - return toBitsVector().rearrange(shuffle).toShuffle(); -#end[!FP] -#if[FP] - return toBitsVector() - .rearrange(shuffle.cast($bitsvectortype$.VSPECIES)) - .toFPShuffle(); -#end[FP] - } - - @ForceInline - @Override - public $shuffletype$ wrapIndexes() { - $bitsvectortype$ v = toBitsVector(); - $Bitstype$Vector.$Bitstype$Species species = $bitsvectortype$.VSPECIES; - if ((VLENGTH & (VLENGTH - 1)) == 0) { - v = v.lanewise(VectorOperators.AND, species.broadcast(VLENGTH - 1)); - } else { - VectorMask<$Boxbitstype$> neg = v.compare(VectorOperators.LT, 0); - Vector<$Boxbitstype$> adjusted = v.lanewise(VectorOperators.ADD, VLENGTH); - v = v.blend(adjusted, neg); - } -#if[!FP] - return v.toShuffle(); -#end[!FP] -#if[FP] - return v.toFPShuffle(); -#end[FP] - } - private static $bitstype$[] prepare(int[] indices, int offset) { $bitstype$[] a = new $bitstype$[VLENGTH]; for (int i = 0; i < VLENGTH; i++) { From 4caa9d103a6b07e6a48b9da196e5c588f0eb3f6b Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Tue, 21 Mar 2023 23:54:42 +0800 Subject: [PATCH 09/13] missing casts --- .../share/classes/jdk/incubator/vector/IntVector.java | 2 +- .../share/classes/jdk/incubator/vector/LongVector.java | 2 +- .../share/classes/jdk/incubator/vector/X-Vector.java.template | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java index 6adf9067443aa..5840d7e198e27 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java @@ -2496,7 +2496,7 @@ AbstractShuffle toFPShuffle0(VectorSpecies dsp) { for (int i = 0; i < a.length; i++) { sa[i] = (int) a[i]; } - return VectorShuffle.fromArray(dsp, sa, 0); + return (AbstractShuffle) VectorShuffle.fromArray(dsp, sa, 0); } @ForceInline diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java index 9dbb1bdd3a9fa..b7c5e2a4fe62e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java @@ -2362,7 +2362,7 @@ AbstractShuffle toFPShuffle0(VectorSpecies dsp) { for (int i = 0; i < a.length; i++) { sa[i] = (int) a[i]; } - return VectorShuffle.fromArray(dsp, sa, 0); + return (AbstractShuffle) VectorShuffle.fromArray(dsp, sa, 0); } @ForceInline diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template index 4f6e7355680b8..aa2f708fc37b3 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template @@ -2891,7 +2891,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { for (int i = 0; i < a.length; i++) { sa[i] = (int) a[i]; } - return VectorShuffle.fromArray(dsp, sa, 0); + return (AbstractShuffle<$Boxfptype$>) VectorShuffle.fromArray(dsp, sa, 0); } @ForceInline From e0b9ee88a7ef031a15ab38703daf253631437d73 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Wed, 22 Mar 2023 20:36:32 +0800 Subject: [PATCH 10/13] reviews --- src/hotspot/share/prims/vectorSupport.hpp | 2 +- .../jdk/incubator/vector/AbstractShuffle.java | 22 +++++++++---------- .../jdk/incubator/vector/AbstractVector.java | 6 ++--- .../jdk/incubator/vector/VectorShape.java | 2 +- .../jdk/incubator/vector/VectorShuffle.java | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/hotspot/share/prims/vectorSupport.hpp b/src/hotspot/share/prims/vectorSupport.hpp index 90c5bcf7aaf62..c840a66f7ca11 100644 --- a/src/hotspot/share/prims/vectorSupport.hpp +++ b/src/hotspot/share/prims/vectorSupport.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java index 20f4fa52a0f3e..71e6710ea670d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,13 +75,13 @@ public final VectorShuffle cast(VectorSpecies s) { if (species.asIntegral() == species) { return v.castShape(species, 0).toShuffle(); } else if (species.elementType() == float.class) { - return (VectorShuffle)v.castShape(species.asIntegral(), 0) - .reinterpretAsInts() - .toFPShuffle(); + return (VectorShuffle) v.castShape(species.asIntegral(), 0) + .reinterpretAsInts() + .toFPShuffle(); } else { - return (VectorShuffle)v.castShape(species.asIntegral(), 0) - .reinterpretAsLongs() - .toFPShuffle(); + return (VectorShuffle) v.castShape(species.asIntegral(), 0) + .reinterpretAsLongs() + .toFPShuffle(); } } @@ -105,7 +105,7 @@ public final VectorShuffle checkIndexes() { public final VectorMask laneIsValid() { Vector shufvec = this.toBitsVector(); return shufvec.compare(VectorOperators.GE, 0) - .cast(vspecies()); + .cast(vspecies()); } @ForceInline @@ -114,8 +114,8 @@ public final VectorMask laneIsValid() { public final VectorShuffle rearrange(VectorShuffle shuffle) { Vector v = toBitsVector(); return (VectorShuffle) v.rearrange(shuffle.cast(vspecies().asIntegral())) - .toShuffle() - .cast(vspecies()); + .toShuffle() + .cast(vspecies()); } @ForceInline @@ -127,7 +127,7 @@ public final VectorShuffle wrapIndexes() { v = v.lanewise(VectorOperators.AND, length() - 1); } else { v = v.blend(v.lanewise(VectorOperators.ADD, length()), - v.compare(VectorOperators.LT, 0)); + v.compare(VectorOperators.LT, 0)); } return (VectorShuffle) v.toShuffle().cast(vspecies()); } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java index c53e68e5a3cf7..5a91bb5aa9930 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -195,13 +195,13 @@ ByteVector asByteVectorRawTemplate() { final AbstractShuffle iotaShuffle(int start, int step, boolean wrap) { if ((length() & (length() - 1)) != 0) { return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, length()))) - : shuffleFromOp(i -> i * step + start); + : shuffleFromOp(i -> i * step + start); } AbstractSpecies species = vspecies().asIntegral(); Vector iota = species.iota(); iota = iota.lanewise(VectorOperators.MUL, step) - .lanewise(VectorOperators.ADD, start); + .lanewise(VectorOperators.ADD, start); Vector wrapped = iota.lanewise(VectorOperators.AND, length() - 1); if (!wrap) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java index f51fb766f9fea..89e0d38bcb06c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShape.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java index 46ad9f357742b..bc1780a81acc0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorShuffle.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From a4835c002591ae8d8a8cef998ef6c220e0f5631e Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Thu, 30 Mar 2023 22:25:27 +0800 Subject: [PATCH 11/13] move implementations up --- src/hotspot/cpu/x86/x86.ad | 16 +++--- .../jdk/incubator/vector/AbstractShuffle.java | 33 ++++++------ .../jdk/incubator/vector/AbstractSpecies.java | 8 +++ .../jdk/incubator/vector/AbstractVector.java | 30 +++++++++-- .../jdk/incubator/vector/Byte128Vector.java | 18 +++---- .../jdk/incubator/vector/Byte256Vector.java | 18 +++---- .../jdk/incubator/vector/Byte512Vector.java | 18 +++---- .../jdk/incubator/vector/Byte64Vector.java | 18 +++---- .../jdk/incubator/vector/ByteMaxVector.java | 18 +++---- .../jdk/incubator/vector/ByteVector.java | 24 ++++----- .../jdk/incubator/vector/Double128Vector.java | 20 +++----- .../jdk/incubator/vector/Double256Vector.java | 20 +++----- .../jdk/incubator/vector/Double512Vector.java | 20 +++----- .../jdk/incubator/vector/Double64Vector.java | 20 +++----- .../jdk/incubator/vector/DoubleMaxVector.java | 20 +++----- .../jdk/incubator/vector/DoubleVector.java | 19 ++++++- .../jdk/incubator/vector/Float128Vector.java | 20 +++----- .../jdk/incubator/vector/Float256Vector.java | 20 +++----- .../jdk/incubator/vector/Float512Vector.java | 20 +++----- .../jdk/incubator/vector/Float64Vector.java | 20 +++----- .../jdk/incubator/vector/FloatMaxVector.java | 20 +++----- .../jdk/incubator/vector/FloatVector.java | 19 ++++++- .../jdk/incubator/vector/Int128Vector.java | 23 +++------ .../jdk/incubator/vector/Int256Vector.java | 23 +++------ .../jdk/incubator/vector/Int512Vector.java | 23 +++------ .../jdk/incubator/vector/Int64Vector.java | 23 +++------ .../jdk/incubator/vector/IntMaxVector.java | 23 +++------ .../jdk/incubator/vector/IntVector.java | 46 ++++------------- .../jdk/incubator/vector/Long128Vector.java | 23 +++------ .../jdk/incubator/vector/Long256Vector.java | 23 +++------ .../jdk/incubator/vector/Long512Vector.java | 23 +++------ .../jdk/incubator/vector/Long64Vector.java | 23 +++------ .../jdk/incubator/vector/LongMaxVector.java | 23 +++------ .../jdk/incubator/vector/LongVector.java | 46 ++++------------- .../jdk/incubator/vector/Short128Vector.java | 18 +++---- .../jdk/incubator/vector/Short256Vector.java | 18 +++---- .../jdk/incubator/vector/Short512Vector.java | 18 +++---- .../jdk/incubator/vector/Short64Vector.java | 18 +++---- .../jdk/incubator/vector/ShortMaxVector.java | 18 +++---- .../jdk/incubator/vector/ShortVector.java | 24 ++++----- .../jdk/incubator/vector/VectorSpecies.java | 6 +++ .../incubator/vector/X-Vector.java.template | 50 ++++--------------- .../vector/X-VectorBits.java.template | 36 +++---------- 43 files changed, 393 insertions(+), 574 deletions(-) diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index b9b7388791850..67a540a0ba33e 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2173,14 +2173,14 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* // Do Vector::rearrange needs preparation of the shuffle argument const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { - if (elem_bt == T_BYTE) { - return false; - } else if (elem_bt == T_SHORT) { - return !VM_Version::supports_avx512bw(); - } else if (elem_bt == T_INT || elem_bt == T_FLOAT) { - return !VM_Version::supports_avx2(); - } else { // T_LONG || T_DOUBLE - return vlen < 8 && !VM_Version::supports_avx512vl(); + switch (elem_bt) { + case T_BYTE: return false; + case T_SHORT: return !VM_Version::supports_avx512bw(); + case T_INT: return !VM_Version::supports_avx(); + case T_LONG: return vlen < 8 && !VM_Version::supports_avx512vl(); + default: + ShouldNotReachHere(); + return false; } } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java index 71e6710ea670d..da20207e16820 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractShuffle.java @@ -26,6 +26,7 @@ import java.util.function.IntUnaryOperator; import jdk.internal.vm.annotation.ForceInline; +import jdk.internal.vm.vector.VectorSupport; abstract class AbstractShuffle extends VectorShuffle { static final IntUnaryOperator IDENTITY = i -> i; @@ -47,7 +48,20 @@ public final VectorSpecies vectorSpecies() { } /*package-private*/ - abstract Vector toBitsVector(); + abstract AbstractVector toBitsVector(); + + final AbstractVector toBitsVectorTemplate() { + AbstractSpecies dsp = vspecies().asIntegral(); + Class etype = dsp.elementType(); + Class rvtype = dsp.dummyVector().getClass(); + return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, + getClass(), etype, length(), + rvtype, etype, length(), + this, dsp, + (v, s) -> v.toBitsVector0()); + } + + abstract AbstractVector toBitsVector0(); @Override @ForceInline @@ -65,24 +79,11 @@ public final int[] toArray() { @Override @ForceInline - @SuppressWarnings("unchecked") public final VectorShuffle cast(VectorSpecies s) { - AbstractSpecies species = (AbstractSpecies) s; - if (length() != species.laneCount()) { + if (length() != s.length()) { throw new IllegalArgumentException("VectorShuffle length and species length differ"); } - Vector v = toBitsVector(); - if (species.asIntegral() == species) { - return v.castShape(species, 0).toShuffle(); - } else if (species.elementType() == float.class) { - return (VectorShuffle) v.castShape(species.asIntegral(), 0) - .reinterpretAsInts() - .toFPShuffle(); - } else { - return (VectorShuffle) v.castShape(species.asIntegral(), 0) - .reinterpretAsLongs() - .toFPShuffle(); - } + return toBitsVector().toShuffle((AbstractSpecies) s); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java index fac0012e646b4..d26e5814c4f96 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java @@ -48,6 +48,8 @@ abstract class AbstractSpecies extends jdk.internal.vm.vector.VectorSupport.V @Stable final Class> maskType; @Stable + final Class> shuffleType; + @Stable final Function> vectorFactory; @Stable @@ -61,11 +63,13 @@ abstract class AbstractSpecies extends jdk.internal.vm.vector.VectorSupport.V LaneType laneType, Class> vectorType, Class> maskType, + Class> shuffleType, Function> vectorFactory) { this.vectorShape = vectorShape; this.laneType = laneType; this.vectorType = vectorType; this.maskType = maskType; + this.shuffleType = shuffleType; this.vectorFactory = vectorFactory; // derived values: @@ -162,6 +166,10 @@ public final Class> maskType() { return maskType; } + @ForceInline Class> shuffleType() { + return shuffleType; + } + @Override @ForceInline public final int elementSize() { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java index 5a91bb5aa9930..5705a2614ad4c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java @@ -188,11 +188,35 @@ ByteVector asByteVectorRawTemplate() { abstract AbstractMask maskFromArray(boolean[] bits); - abstract AbstractShuffle iotaShuffle(); + abstract VectorShuffle toShuffle(AbstractSpecies dsp); + + /*package-private*/ + @ForceInline + final VectorShuffle toShuffleTemplate(AbstractSpecies dsp) { + Class etype = vspecies().elementType(); + Class dvtype = dsp.shuffleType(); + Class dtype = dsp.asIntegral().elementType(); + int dlength = dsp.dummyVector().length(); + return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, + getClass(), etype, length(), + dvtype, dtype, dlength, + this, dsp, + AbstractVector::toShuffle0); + } + + abstract VectorShuffle toShuffle0(AbstractSpecies dsp); + + @ForceInline + public final + VectorShuffle toShuffle() { + return toShuffle(vspecies()); + } + + abstract VectorShuffle iotaShuffle(); @ForceInline @SuppressWarnings({"rawtypes", "unchecked"}) - final AbstractShuffle iotaShuffle(int start, int step, boolean wrap) { + final VectorShuffle iotaShuffle(int start, int step, boolean wrap) { if ((length() & (length() - 1)) != 0) { return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, length()))) : shuffleFromOp(i -> i * step + start); @@ -209,7 +233,7 @@ final AbstractShuffle iotaShuffle(int start, int step, boolean wrap) { VectorMask mask = wrapped.compare(VectorOperators.EQ, iota); wrapped = wrappedEx.blend(wrapped, mask); } - return (AbstractShuffle) wrapped.toShuffle().cast(vspecies()); + return ((AbstractVector) wrapped).toShuffle(vspecies()); } abstract AbstractShuffle shuffleFromArray(int[] indexes, int i); diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java index 792ecf19f2c2b..fdf1f55dc69b6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java @@ -344,8 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Byte128Shuffle toShuffle() { - return (Byte128Shuffle) super.toShuffleTemplate(Byte128Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -833,16 +834,13 @@ public ByteSpecies vspecies() { @Override @ForceInline Byte128Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Byte128Shuffle.class, byte.class, VLENGTH, - Byte128Vector.class, byte.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Byte128Vector) super.toBitsVectorTemplate(); } - private static Byte128Vector toBitsVectorHelper(Byte128Shuffle s) { - return (Byte128Vector) Byte128Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + ByteVector toBitsVector0() { + return Byte128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java index 733e5cc97b75a..c9496d36b1392 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java @@ -344,8 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Byte256Shuffle toShuffle() { - return (Byte256Shuffle) super.toShuffleTemplate(Byte256Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -865,16 +866,13 @@ public ByteSpecies vspecies() { @Override @ForceInline Byte256Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Byte256Shuffle.class, byte.class, VLENGTH, - Byte256Vector.class, byte.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Byte256Vector) super.toBitsVectorTemplate(); } - private static Byte256Vector toBitsVectorHelper(Byte256Shuffle s) { - return (Byte256Vector) Byte256Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + ByteVector toBitsVector0() { + return Byte256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java index 615cdc324ae6e..10cbb3386c74e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java @@ -344,8 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Byte512Shuffle toShuffle() { - return (Byte512Shuffle) super.toShuffleTemplate(Byte512Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -929,16 +930,13 @@ public ByteSpecies vspecies() { @Override @ForceInline Byte512Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Byte512Shuffle.class, byte.class, VLENGTH, - Byte512Vector.class, byte.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Byte512Vector) super.toBitsVectorTemplate(); } - private static Byte512Vector toBitsVectorHelper(Byte512Shuffle s) { - return (Byte512Vector) Byte512Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + ByteVector toBitsVector0() { + return Byte512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java index 43069537615d2..6ddd22c3627b2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java @@ -344,8 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Byte64Shuffle toShuffle() { - return (Byte64Shuffle) super.toShuffleTemplate(Byte64Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -817,16 +818,13 @@ public ByteSpecies vspecies() { @Override @ForceInline Byte64Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Byte64Shuffle.class, byte.class, VLENGTH, - Byte64Vector.class, byte.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Byte64Vector) super.toBitsVectorTemplate(); } - private static Byte64Vector toBitsVectorHelper(Byte64Shuffle s) { - return (Byte64Vector) Byte64Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + ByteVector toBitsVector0() { + return Byte64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java index 52ecc0dbbabd3..83404b1634a09 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java @@ -344,8 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public ByteMaxShuffle toShuffle() { - return (ByteMaxShuffle) super.toShuffleTemplate(ByteMaxShuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -803,16 +804,13 @@ public ByteSpecies vspecies() { @Override @ForceInline ByteMaxVector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - ByteMaxShuffle.class, byte.class, VLENGTH, - ByteMaxVector.class, byte.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (ByteMaxVector) super.toBitsVectorTemplate(); } - private static ByteMaxVector toBitsVectorHelper(ByteMaxShuffle s) { - return (ByteMaxVector) ByteMaxVector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + ByteVector toBitsVector0() { + return ByteMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java index aaad8ac3391f3..87a70e8c28ea6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java @@ -2480,8 +2480,8 @@ ByteVector rearrangeTemplate(Class shuffletype, } @ForceInline - private final - VectorShuffle toShuffle0(ByteSpecies dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { byte[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2490,18 +2490,6 @@ VectorShuffle toShuffle0(ByteSpecies dsp) { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final VectorShuffle toShuffleTemplate(Class shuffleType) { - ByteSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - getClass(), byte.class, length(), - shuffleType, byte.class, length(), - this, vsp, - ByteVector::toShuffle0); - } - - /** * {@inheritDoc} * @since 19 @@ -4096,9 +4084,10 @@ static final class ByteSpecies extends AbstractSpecies { private ByteSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(byte.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Byte.SIZE); } @@ -4374,6 +4363,7 @@ static ByteSpecies species(VectorShape s) { = new ByteSpecies(VectorShape.S_64_BIT, Byte64Vector.class, Byte64Vector.Byte64Mask.class, + Byte64Vector.Byte64Shuffle.class, Byte64Vector::new); /** Species representing {@link ByteVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -4381,6 +4371,7 @@ static ByteSpecies species(VectorShape s) { = new ByteSpecies(VectorShape.S_128_BIT, Byte128Vector.class, Byte128Vector.Byte128Mask.class, + Byte128Vector.Byte128Shuffle.class, Byte128Vector::new); /** Species representing {@link ByteVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -4388,6 +4379,7 @@ static ByteSpecies species(VectorShape s) { = new ByteSpecies(VectorShape.S_256_BIT, Byte256Vector.class, Byte256Vector.Byte256Mask.class, + Byte256Vector.Byte256Shuffle.class, Byte256Vector::new); /** Species representing {@link ByteVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -4395,6 +4387,7 @@ static ByteSpecies species(VectorShape s) { = new ByteSpecies(VectorShape.S_512_BIT, Byte512Vector.class, Byte512Vector.Byte512Mask.class, + Byte512Vector.Byte512Shuffle.class, Byte512Vector::new); /** Species representing {@link ByteVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -4402,6 +4395,7 @@ static ByteSpecies species(VectorShape s) { = new ByteSpecies(VectorShape.S_Max_BIT, ByteMaxVector.class, ByteMaxVector.ByteMaxMask.class, + ByteMaxVector.ByteMaxShuffle.class, ByteMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java index 27338df235ed8..d747cb5136d2e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java @@ -331,10 +331,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Double128Shuffle toShuffle() { - return (Double128Shuffle) castShape(Long128Vector.VSPECIES, 0) - .reinterpretAsLongs() - .toFPShuffle(); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -796,16 +795,13 @@ public DoubleSpecies vspecies() { @Override @ForceInline Long128Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Double128Shuffle.class, long.class, VLENGTH, - Long128Vector.class, long.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Long128Vector) super.toBitsVectorTemplate(); } - private static Long128Vector toBitsVectorHelper(Double128Shuffle s) { - return (Long128Vector) Long128Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + LongVector toBitsVector0() { + return Long128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java index 5bc912f05a0dd..18325249d2ead 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java @@ -331,10 +331,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Double256Shuffle toShuffle() { - return (Double256Shuffle) castShape(Long256Vector.VSPECIES, 0) - .reinterpretAsLongs() - .toFPShuffle(); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -800,16 +799,13 @@ public DoubleSpecies vspecies() { @Override @ForceInline Long256Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Double256Shuffle.class, long.class, VLENGTH, - Long256Vector.class, long.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Long256Vector) super.toBitsVectorTemplate(); } - private static Long256Vector toBitsVectorHelper(Double256Shuffle s) { - return (Long256Vector) Long256Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + LongVector toBitsVector0() { + return Long256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java index adb57ebbb117d..3140fde3747b5 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java @@ -331,10 +331,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Double512Shuffle toShuffle() { - return (Double512Shuffle) castShape(Long512Vector.VSPECIES, 0) - .reinterpretAsLongs() - .toFPShuffle(); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -808,16 +807,13 @@ public DoubleSpecies vspecies() { @Override @ForceInline Long512Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Double512Shuffle.class, long.class, VLENGTH, - Long512Vector.class, long.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Long512Vector) super.toBitsVectorTemplate(); } - private static Long512Vector toBitsVectorHelper(Double512Shuffle s) { - return (Long512Vector) Long512Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + LongVector toBitsVector0() { + return Long512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java index b80beacf8f641..bce5b31a9b35f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java @@ -331,10 +331,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Double64Shuffle toShuffle() { - return (Double64Shuffle) castShape(Long64Vector.VSPECIES, 0) - .reinterpretAsLongs() - .toFPShuffle(); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -794,16 +793,13 @@ public DoubleSpecies vspecies() { @Override @ForceInline Long64Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Double64Shuffle.class, long.class, VLENGTH, - Long64Vector.class, long.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Long64Vector) super.toBitsVectorTemplate(); } - private static Long64Vector toBitsVectorHelper(Double64Shuffle s) { - return (Long64Vector) Long64Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + LongVector toBitsVector0() { + return Long64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java index f35a238ac300a..e1b24a64b8b8a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java @@ -331,10 +331,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public DoubleMaxShuffle toShuffle() { - return (DoubleMaxShuffle) castShape(LongMaxVector.VSPECIES, 0) - .reinterpretAsLongs() - .toFPShuffle(); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -793,16 +792,13 @@ public DoubleSpecies vspecies() { @Override @ForceInline LongMaxVector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - DoubleMaxShuffle.class, long.class, VLENGTH, - LongMaxVector.class, long.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (LongMaxVector) super.toBitsVectorTemplate(); } - private static LongMaxVector toBitsVectorHelper(DoubleMaxShuffle s) { - return (LongMaxVector) LongMaxVector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + LongVector toBitsVector0() { + return LongMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index a1f72a0e6264a..1f4c0a49d45c2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -2321,7 +2321,16 @@ DoubleVector rearrangeTemplate(Class shuffletype, return r1.blend(r0, valid); } - + @ForceInline + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { + double[] a = toArray(); + int[] sa = new int[a.length]; + for (int i = 0; i < a.length; i++) { + sa[i] = (int) a[i]; + } + return VectorShuffle.fromArray(dsp, sa, 0); + } /** * {@inheritDoc} @@ -3678,9 +3687,10 @@ static final class DoubleSpecies extends AbstractSpecies { private DoubleSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(double.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Double.SIZE); } @@ -3956,6 +3966,7 @@ static DoubleSpecies species(VectorShape s) { = new DoubleSpecies(VectorShape.S_64_BIT, Double64Vector.class, Double64Vector.Double64Mask.class, + Double64Vector.Double64Shuffle.class, Double64Vector::new); /** Species representing {@link DoubleVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -3963,6 +3974,7 @@ static DoubleSpecies species(VectorShape s) { = new DoubleSpecies(VectorShape.S_128_BIT, Double128Vector.class, Double128Vector.Double128Mask.class, + Double128Vector.Double128Shuffle.class, Double128Vector::new); /** Species representing {@link DoubleVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -3970,6 +3982,7 @@ static DoubleSpecies species(VectorShape s) { = new DoubleSpecies(VectorShape.S_256_BIT, Double256Vector.class, Double256Vector.Double256Mask.class, + Double256Vector.Double256Shuffle.class, Double256Vector::new); /** Species representing {@link DoubleVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -3977,6 +3990,7 @@ static DoubleSpecies species(VectorShape s) { = new DoubleSpecies(VectorShape.S_512_BIT, Double512Vector.class, Double512Vector.Double512Mask.class, + Double512Vector.Double512Shuffle.class, Double512Vector::new); /** Species representing {@link DoubleVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -3984,6 +3998,7 @@ static DoubleSpecies species(VectorShape s) { = new DoubleSpecies(VectorShape.S_Max_BIT, DoubleMaxVector.class, DoubleMaxVector.DoubleMaxMask.class, + DoubleMaxVector.DoubleMaxShuffle.class, DoubleMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java index 8aa95de599ac6..7ccf752540a83 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java @@ -331,10 +331,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Float128Shuffle toShuffle() { - return (Float128Shuffle) castShape(Int128Vector.VSPECIES, 0) - .reinterpretAsInts() - .toFPShuffle(); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -800,16 +799,13 @@ public FloatSpecies vspecies() { @Override @ForceInline Int128Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Float128Shuffle.class, int.class, VLENGTH, - Int128Vector.class, int.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Int128Vector) super.toBitsVectorTemplate(); } - private static Int128Vector toBitsVectorHelper(Float128Shuffle s) { - return (Int128Vector) Int128Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + IntVector toBitsVector0() { + return Int128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java index 7f90068ebee93..fd1763b0c40e2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java @@ -331,10 +331,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Float256Shuffle toShuffle() { - return (Float256Shuffle) castShape(Int256Vector.VSPECIES, 0) - .reinterpretAsInts() - .toFPShuffle(); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -808,16 +807,13 @@ public FloatSpecies vspecies() { @Override @ForceInline Int256Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Float256Shuffle.class, int.class, VLENGTH, - Int256Vector.class, int.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Int256Vector) super.toBitsVectorTemplate(); } - private static Int256Vector toBitsVectorHelper(Float256Shuffle s) { - return (Int256Vector) Int256Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + IntVector toBitsVector0() { + return Int256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java index 2978f48691de7..03dfb1e5049f9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java @@ -331,10 +331,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Float512Shuffle toShuffle() { - return (Float512Shuffle) castShape(Int512Vector.VSPECIES, 0) - .reinterpretAsInts() - .toFPShuffle(); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -824,16 +823,13 @@ public FloatSpecies vspecies() { @Override @ForceInline Int512Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Float512Shuffle.class, int.class, VLENGTH, - Int512Vector.class, int.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Int512Vector) super.toBitsVectorTemplate(); } - private static Int512Vector toBitsVectorHelper(Float512Shuffle s) { - return (Int512Vector) Int512Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + IntVector toBitsVector0() { + return Int512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java index 3d016c116c6c4..48d76adb3ebb1 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java @@ -331,10 +331,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Float64Shuffle toShuffle() { - return (Float64Shuffle) castShape(Int64Vector.VSPECIES, 0) - .reinterpretAsInts() - .toFPShuffle(); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -796,16 +795,13 @@ public FloatSpecies vspecies() { @Override @ForceInline Int64Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Float64Shuffle.class, int.class, VLENGTH, - Int64Vector.class, int.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Int64Vector) super.toBitsVectorTemplate(); } - private static Int64Vector toBitsVectorHelper(Float64Shuffle s) { - return (Int64Vector) Int64Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + IntVector toBitsVector0() { + return Int64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java index 785b5e32a08b0..ffea6b909e135 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java @@ -331,10 +331,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public FloatMaxShuffle toShuffle() { - return (FloatMaxShuffle) castShape(IntMaxVector.VSPECIES, 0) - .reinterpretAsInts() - .toFPShuffle(); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -793,16 +792,13 @@ public FloatSpecies vspecies() { @Override @ForceInline IntMaxVector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - FloatMaxShuffle.class, int.class, VLENGTH, - IntMaxVector.class, int.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (IntMaxVector) super.toBitsVectorTemplate(); } - private static IntMaxVector toBitsVectorHelper(FloatMaxShuffle s) { - return (IntMaxVector) IntMaxVector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + IntVector toBitsVector0() { + return IntMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index 7beca7842c980..e86523870ad72 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -2333,7 +2333,16 @@ FloatVector rearrangeTemplate(Class shuffletype, return r1.blend(r0, valid); } - + @ForceInline + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { + float[] a = toArray(); + int[] sa = new int[a.length]; + for (int i = 0; i < a.length; i++) { + sa[i] = (int) a[i]; + } + return VectorShuffle.fromArray(dsp, sa, 0); + } /** * {@inheritDoc} @@ -3628,9 +3637,10 @@ static final class FloatSpecies extends AbstractSpecies { private FloatSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(float.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Float.SIZE); } @@ -3906,6 +3916,7 @@ static FloatSpecies species(VectorShape s) { = new FloatSpecies(VectorShape.S_64_BIT, Float64Vector.class, Float64Vector.Float64Mask.class, + Float64Vector.Float64Shuffle.class, Float64Vector::new); /** Species representing {@link FloatVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -3913,6 +3924,7 @@ static FloatSpecies species(VectorShape s) { = new FloatSpecies(VectorShape.S_128_BIT, Float128Vector.class, Float128Vector.Float128Mask.class, + Float128Vector.Float128Shuffle.class, Float128Vector::new); /** Species representing {@link FloatVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -3920,6 +3932,7 @@ static FloatSpecies species(VectorShape s) { = new FloatSpecies(VectorShape.S_256_BIT, Float256Vector.class, Float256Vector.Float256Mask.class, + Float256Vector.Float256Shuffle.class, Float256Vector::new); /** Species representing {@link FloatVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -3927,6 +3940,7 @@ static FloatSpecies species(VectorShape s) { = new FloatSpecies(VectorShape.S_512_BIT, Float512Vector.class, Float512Vector.Float512Mask.class, + Float512Vector.Float512Shuffle.class, Float512Vector::new); /** Species representing {@link FloatVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -3934,6 +3948,7 @@ static FloatSpecies species(VectorShape s) { = new FloatSpecies(VectorShape.S_Max_BIT, FloatMaxVector.class, FloatMaxVector.FloatMaxMask.class, + FloatMaxVector.FloatMaxShuffle.class, FloatMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java index afebe84f779db..d0d8b1817b808 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java @@ -344,13 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Int128Shuffle toShuffle() { - return (Int128Shuffle) super.toShuffleTemplate(Int128Shuffle.class); // specialize - } - @Override - @ForceInline - Float128Vector.Float128Shuffle toFPShuffle() { - return (Float128Vector.Float128Shuffle) super.toFPShuffleTemplate(Float128Vector.Float128Shuffle.class, Float128Vector.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -814,16 +810,13 @@ public IntSpecies vspecies() { @Override @ForceInline Int128Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Int128Shuffle.class, int.class, VLENGTH, - Int128Vector.class, int.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Int128Vector) super.toBitsVectorTemplate(); } - private static Int128Vector toBitsVectorHelper(Int128Shuffle s) { - return (Int128Vector) Int128Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + IntVector toBitsVector0() { + return Int128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java index 7e5e2b8171c11..96d4d8569da16 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java @@ -344,13 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Int256Shuffle toShuffle() { - return (Int256Shuffle) super.toShuffleTemplate(Int256Shuffle.class); // specialize - } - @Override - @ForceInline - Float256Vector.Float256Shuffle toFPShuffle() { - return (Float256Vector.Float256Shuffle) super.toFPShuffleTemplate(Float256Vector.Float256Shuffle.class, Float256Vector.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -822,16 +818,13 @@ public IntSpecies vspecies() { @Override @ForceInline Int256Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Int256Shuffle.class, int.class, VLENGTH, - Int256Vector.class, int.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Int256Vector) super.toBitsVectorTemplate(); } - private static Int256Vector toBitsVectorHelper(Int256Shuffle s) { - return (Int256Vector) Int256Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + IntVector toBitsVector0() { + return Int256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java index 715ce7e2ab746..7a5b7a63c5289 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java @@ -344,13 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Int512Shuffle toShuffle() { - return (Int512Shuffle) super.toShuffleTemplate(Int512Shuffle.class); // specialize - } - @Override - @ForceInline - Float512Vector.Float512Shuffle toFPShuffle() { - return (Float512Vector.Float512Shuffle) super.toFPShuffleTemplate(Float512Vector.Float512Shuffle.class, Float512Vector.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -838,16 +834,13 @@ public IntSpecies vspecies() { @Override @ForceInline Int512Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Int512Shuffle.class, int.class, VLENGTH, - Int512Vector.class, int.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Int512Vector) super.toBitsVectorTemplate(); } - private static Int512Vector toBitsVectorHelper(Int512Shuffle s) { - return (Int512Vector) Int512Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + IntVector toBitsVector0() { + return Int512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java index 67e538fdea6bf..d582b0fe0b1c4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java @@ -344,13 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Int64Shuffle toShuffle() { - return (Int64Shuffle) super.toShuffleTemplate(Int64Shuffle.class); // specialize - } - @Override - @ForceInline - Float64Vector.Float64Shuffle toFPShuffle() { - return (Float64Vector.Float64Shuffle) super.toFPShuffleTemplate(Float64Vector.Float64Shuffle.class, Float64Vector.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -810,16 +806,13 @@ public IntSpecies vspecies() { @Override @ForceInline Int64Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Int64Shuffle.class, int.class, VLENGTH, - Int64Vector.class, int.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Int64Vector) super.toBitsVectorTemplate(); } - private static Int64Vector toBitsVectorHelper(Int64Shuffle s) { - return (Int64Vector) Int64Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + IntVector toBitsVector0() { + return Int64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java index 0f15fc0a58589..77c0d07fadb5a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java @@ -344,13 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public IntMaxShuffle toShuffle() { - return (IntMaxShuffle) super.toShuffleTemplate(IntMaxShuffle.class); // specialize - } - @Override - @ForceInline - FloatMaxVector.FloatMaxShuffle toFPShuffle() { - return (FloatMaxVector.FloatMaxShuffle) super.toFPShuffleTemplate(FloatMaxVector.FloatMaxShuffle.class, FloatMaxVector.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -819,16 +815,13 @@ public IntSpecies vspecies() { @Override @ForceInline IntMaxVector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - IntMaxShuffle.class, int.class, VLENGTH, - IntMaxVector.class, int.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (IntMaxVector) super.toBitsVectorTemplate(); } - private static IntMaxVector toBitsVectorHelper(IntMaxShuffle s) { - return (IntMaxVector) IntMaxVector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + IntVector toBitsVector0() { + return IntMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java index 5840d7e198e27..a6faa907387d0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java @@ -2465,8 +2465,8 @@ IntVector rearrangeTemplate(Class shuffletype, } @ForceInline - private final - VectorShuffle toShuffle0(IntSpecies dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { int[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2475,40 +2475,6 @@ VectorShuffle toShuffle0(IntSpecies dsp) { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final VectorShuffle toShuffleTemplate(Class shuffleType) { - IntSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - getClass(), int.class, length(), - shuffleType, int.class, length(), - this, vsp, - IntVector::toShuffle0); - } - - abstract AbstractShuffle toFPShuffle(); - - @ForceInline - private final - AbstractShuffle toFPShuffle0(VectorSpecies dsp) { - int[] a = toArray(); - int[] sa = new int[a.length]; - for (int i = 0; i < a.length; i++) { - sa[i] = (int) a[i]; - } - return (AbstractShuffle) VectorShuffle.fromArray(dsp, sa, 0); - } - - @ForceInline - final - AbstractShuffle toFPShuffleTemplate(Class shuffleType, FloatVector.FloatSpecies dsp) { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - getClass(), int.class, length(), - shuffleType, int.class, length(), - this, dsp, - IntVector::toFPShuffle0); - } - /** * {@inheritDoc} * @since 19 @@ -3827,9 +3793,10 @@ static final class IntSpecies extends AbstractSpecies { private IntSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(int.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Integer.SIZE); } @@ -4105,6 +4072,7 @@ static IntSpecies species(VectorShape s) { = new IntSpecies(VectorShape.S_64_BIT, Int64Vector.class, Int64Vector.Int64Mask.class, + Int64Vector.Int64Shuffle.class, Int64Vector::new); /** Species representing {@link IntVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -4112,6 +4080,7 @@ static IntSpecies species(VectorShape s) { = new IntSpecies(VectorShape.S_128_BIT, Int128Vector.class, Int128Vector.Int128Mask.class, + Int128Vector.Int128Shuffle.class, Int128Vector::new); /** Species representing {@link IntVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -4119,6 +4088,7 @@ static IntSpecies species(VectorShape s) { = new IntSpecies(VectorShape.S_256_BIT, Int256Vector.class, Int256Vector.Int256Mask.class, + Int256Vector.Int256Shuffle.class, Int256Vector::new); /** Species representing {@link IntVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -4126,6 +4096,7 @@ static IntSpecies species(VectorShape s) { = new IntSpecies(VectorShape.S_512_BIT, Int512Vector.class, Int512Vector.Int512Mask.class, + Int512Vector.Int512Shuffle.class, Int512Vector::new); /** Species representing {@link IntVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -4133,6 +4104,7 @@ static IntSpecies species(VectorShape s) { = new IntSpecies(VectorShape.S_Max_BIT, IntMaxVector.class, IntMaxVector.IntMaxMask.class, + IntMaxVector.IntMaxShuffle.class, IntMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java index 7424dc6702a28..f9fa51b220cd4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java @@ -339,13 +339,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Long128Shuffle toShuffle() { - return (Long128Shuffle) super.toShuffleTemplate(Long128Shuffle.class); // specialize - } - @Override - @ForceInline - Double128Vector.Double128Shuffle toFPShuffle() { - return (Double128Vector.Double128Shuffle) super.toFPShuffleTemplate(Double128Vector.Double128Shuffle.class, Double128Vector.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -800,16 +796,13 @@ public LongSpecies vspecies() { @Override @ForceInline Long128Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Long128Shuffle.class, long.class, VLENGTH, - Long128Vector.class, long.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Long128Vector) super.toBitsVectorTemplate(); } - private static Long128Vector toBitsVectorHelper(Long128Shuffle s) { - return (Long128Vector) Long128Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + LongVector toBitsVector0() { + return Long128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java index 8c16913b8e21c..9c666de8bd88e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java @@ -339,13 +339,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Long256Shuffle toShuffle() { - return (Long256Shuffle) super.toShuffleTemplate(Long256Shuffle.class); // specialize - } - @Override - @ForceInline - Double256Vector.Double256Shuffle toFPShuffle() { - return (Double256Vector.Double256Shuffle) super.toFPShuffleTemplate(Double256Vector.Double256Shuffle.class, Double256Vector.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -804,16 +800,13 @@ public LongSpecies vspecies() { @Override @ForceInline Long256Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Long256Shuffle.class, long.class, VLENGTH, - Long256Vector.class, long.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Long256Vector) super.toBitsVectorTemplate(); } - private static Long256Vector toBitsVectorHelper(Long256Shuffle s) { - return (Long256Vector) Long256Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + LongVector toBitsVector0() { + return Long256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java index 141617cc40f3e..edf796bbd2009 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java @@ -339,13 +339,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Long512Shuffle toShuffle() { - return (Long512Shuffle) super.toShuffleTemplate(Long512Shuffle.class); // specialize - } - @Override - @ForceInline - Double512Vector.Double512Shuffle toFPShuffle() { - return (Double512Vector.Double512Shuffle) super.toFPShuffleTemplate(Double512Vector.Double512Shuffle.class, Double512Vector.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -812,16 +808,13 @@ public LongSpecies vspecies() { @Override @ForceInline Long512Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Long512Shuffle.class, long.class, VLENGTH, - Long512Vector.class, long.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Long512Vector) super.toBitsVectorTemplate(); } - private static Long512Vector toBitsVectorHelper(Long512Shuffle s) { - return (Long512Vector) Long512Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + LongVector toBitsVector0() { + return Long512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java index d201e73cb6af0..a1739f1b1ca88 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java @@ -339,13 +339,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Long64Shuffle toShuffle() { - return (Long64Shuffle) super.toShuffleTemplate(Long64Shuffle.class); // specialize - } - @Override - @ForceInline - Double64Vector.Double64Shuffle toFPShuffle() { - return (Double64Vector.Double64Shuffle) super.toFPShuffleTemplate(Double64Vector.Double64Shuffle.class, Double64Vector.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -798,16 +794,13 @@ public LongSpecies vspecies() { @Override @ForceInline Long64Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Long64Shuffle.class, long.class, VLENGTH, - Long64Vector.class, long.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Long64Vector) super.toBitsVectorTemplate(); } - private static Long64Vector toBitsVectorHelper(Long64Shuffle s) { - return (Long64Vector) Long64Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + LongVector toBitsVector0() { + return Long64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java index 1731c38e7d745..d82895278f92f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java @@ -339,13 +339,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public LongMaxShuffle toShuffle() { - return (LongMaxShuffle) super.toShuffleTemplate(LongMaxShuffle.class); // specialize - } - @Override - @ForceInline - DoubleMaxVector.DoubleMaxShuffle toFPShuffle() { - return (DoubleMaxVector.DoubleMaxShuffle) super.toFPShuffleTemplate(DoubleMaxVector.DoubleMaxShuffle.class, DoubleMaxVector.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -798,16 +794,13 @@ public LongSpecies vspecies() { @Override @ForceInline LongMaxVector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - LongMaxShuffle.class, long.class, VLENGTH, - LongMaxVector.class, long.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (LongMaxVector) super.toBitsVectorTemplate(); } - private static LongMaxVector toBitsVectorHelper(LongMaxShuffle s) { - return (LongMaxVector) LongMaxVector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + LongVector toBitsVector0() { + return LongMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java index b7c5e2a4fe62e..bbaa8fdb79923 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java @@ -2331,8 +2331,8 @@ LongVector rearrangeTemplate(Class shuffletype, } @ForceInline - private final - VectorShuffle toShuffle0(LongSpecies dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { long[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2341,40 +2341,6 @@ VectorShuffle toShuffle0(LongSpecies dsp) { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final VectorShuffle toShuffleTemplate(Class shuffleType) { - LongSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - getClass(), long.class, length(), - shuffleType, long.class, length(), - this, vsp, - LongVector::toShuffle0); - } - - abstract AbstractShuffle toFPShuffle(); - - @ForceInline - private final - AbstractShuffle toFPShuffle0(VectorSpecies dsp) { - long[] a = toArray(); - int[] sa = new int[a.length]; - for (int i = 0; i < a.length; i++) { - sa[i] = (int) a[i]; - } - return (AbstractShuffle) VectorShuffle.fromArray(dsp, sa, 0); - } - - @ForceInline - final - AbstractShuffle toFPShuffleTemplate(Class shuffleType, DoubleVector.DoubleSpecies dsp) { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - getClass(), long.class, length(), - shuffleType, long.class, length(), - this, dsp, - LongVector::toFPShuffle0); - } - /** * {@inheritDoc} * @since 19 @@ -3762,9 +3728,10 @@ static final class LongSpecies extends AbstractSpecies { private LongSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(long.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Long.SIZE); } @@ -4031,6 +3998,7 @@ static LongSpecies species(VectorShape s) { = new LongSpecies(VectorShape.S_64_BIT, Long64Vector.class, Long64Vector.Long64Mask.class, + Long64Vector.Long64Shuffle.class, Long64Vector::new); /** Species representing {@link LongVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -4038,6 +4006,7 @@ static LongSpecies species(VectorShape s) { = new LongSpecies(VectorShape.S_128_BIT, Long128Vector.class, Long128Vector.Long128Mask.class, + Long128Vector.Long128Shuffle.class, Long128Vector::new); /** Species representing {@link LongVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -4045,6 +4014,7 @@ static LongSpecies species(VectorShape s) { = new LongSpecies(VectorShape.S_256_BIT, Long256Vector.class, Long256Vector.Long256Mask.class, + Long256Vector.Long256Shuffle.class, Long256Vector::new); /** Species representing {@link LongVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -4052,6 +4022,7 @@ static LongSpecies species(VectorShape s) { = new LongSpecies(VectorShape.S_512_BIT, Long512Vector.class, Long512Vector.Long512Mask.class, + Long512Vector.Long512Shuffle.class, Long512Vector::new); /** Species representing {@link LongVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -4059,6 +4030,7 @@ static LongSpecies species(VectorShape s) { = new LongSpecies(VectorShape.S_Max_BIT, LongMaxVector.class, LongMaxVector.LongMaxMask.class, + LongMaxVector.LongMaxShuffle.class, LongMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java index 085275bfaf7db..8796205b84d90 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java @@ -344,8 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Short128Shuffle toShuffle() { - return (Short128Shuffle) super.toShuffleTemplate(Short128Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -817,16 +818,13 @@ public ShortSpecies vspecies() { @Override @ForceInline Short128Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Short128Shuffle.class, short.class, VLENGTH, - Short128Vector.class, short.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Short128Vector) super.toBitsVectorTemplate(); } - private static Short128Vector toBitsVectorHelper(Short128Shuffle s) { - return (Short128Vector) Short128Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + ShortVector toBitsVector0() { + return Short128Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java index ed380d4db7bc1..a9d5f3c47bdfa 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java @@ -344,8 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Short256Shuffle toShuffle() { - return (Short256Shuffle) super.toShuffleTemplate(Short256Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -833,16 +834,13 @@ public ShortSpecies vspecies() { @Override @ForceInline Short256Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Short256Shuffle.class, short.class, VLENGTH, - Short256Vector.class, short.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Short256Vector) super.toBitsVectorTemplate(); } - private static Short256Vector toBitsVectorHelper(Short256Shuffle s) { - return (Short256Vector) Short256Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + ShortVector toBitsVector0() { + return Short256Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java index 2b88778403381..e0dbb928316bf 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java @@ -344,8 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Short512Shuffle toShuffle() { - return (Short512Shuffle) super.toShuffleTemplate(Short512Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -865,16 +866,13 @@ public ShortSpecies vspecies() { @Override @ForceInline Short512Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Short512Shuffle.class, short.class, VLENGTH, - Short512Vector.class, short.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Short512Vector) super.toBitsVectorTemplate(); } - private static Short512Vector toBitsVectorHelper(Short512Shuffle s) { - return (Short512Vector) Short512Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + ShortVector toBitsVector0() { + return Short512Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java index cdc5a500c5f74..8a0a1f616d68c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java @@ -344,8 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public Short64Shuffle toShuffle() { - return (Short64Shuffle) super.toShuffleTemplate(Short64Shuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -809,16 +810,13 @@ public ShortSpecies vspecies() { @Override @ForceInline Short64Vector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - Short64Shuffle.class, short.class, VLENGTH, - Short64Vector.class, short.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (Short64Vector) super.toBitsVectorTemplate(); } - private static Short64Vector toBitsVectorHelper(Short64Shuffle s) { - return (Short64Vector) Short64Vector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + ShortVector toBitsVector0() { + return Short64Vector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java index 2446e386e7951..110c792b85d2a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java @@ -344,8 +344,9 @@ public final long reduceLanesToLong(VectorOperators.Associative op, @Override @ForceInline - public ShortMaxShuffle toShuffle() { - return (ShortMaxShuffle) super.toShuffleTemplate(ShortMaxShuffle.class); // specialize + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } // Specialized unary testing @@ -803,16 +804,13 @@ public ShortSpecies vspecies() { @Override @ForceInline ShortMaxVector toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - ShortMaxShuffle.class, short.class, VLENGTH, - ShortMaxVector.class, short.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return (ShortMaxVector) super.toBitsVectorTemplate(); } - private static ShortMaxVector toBitsVectorHelper(ShortMaxShuffle s) { - return (ShortMaxVector) ShortMaxVector.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + ShortVector toBitsVector0() { + return ShortMaxVector.VSPECIES.dummyVector().vectorFactory(indices()); } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java index 51610789b18a9..8b08f766588df 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java @@ -2481,8 +2481,8 @@ ShortVector rearrangeTemplate(Class shuffletype, } @ForceInline - private final - VectorShuffle toShuffle0(ShortSpecies dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { short[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2491,18 +2491,6 @@ VectorShuffle toShuffle0(ShortSpecies dsp) { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final VectorShuffle toShuffleTemplate(Class shuffleType) { - ShortSpecies vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - getClass(), short.class, length(), - shuffleType, short.class, length(), - this, vsp, - ShortVector::toShuffle0); - } - - /** * {@inheritDoc} * @since 19 @@ -4090,9 +4078,10 @@ static final class ShortSpecies extends AbstractSpecies { private ShortSpecies(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of(short.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == Short.SIZE); } @@ -4368,6 +4357,7 @@ static ShortSpecies species(VectorShape s) { = new ShortSpecies(VectorShape.S_64_BIT, Short64Vector.class, Short64Vector.Short64Mask.class, + Short64Vector.Short64Shuffle.class, Short64Vector::new); /** Species representing {@link ShortVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -4375,6 +4365,7 @@ static ShortSpecies species(VectorShape s) { = new ShortSpecies(VectorShape.S_128_BIT, Short128Vector.class, Short128Vector.Short128Mask.class, + Short128Vector.Short128Shuffle.class, Short128Vector::new); /** Species representing {@link ShortVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -4382,6 +4373,7 @@ static ShortSpecies species(VectorShape s) { = new ShortSpecies(VectorShape.S_256_BIT, Short256Vector.class, Short256Vector.Short256Mask.class, + Short256Vector.Short256Shuffle.class, Short256Vector::new); /** Species representing {@link ShortVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -4389,6 +4381,7 @@ static ShortSpecies species(VectorShape s) { = new ShortSpecies(VectorShape.S_512_BIT, Short512Vector.class, Short512Vector.Short512Mask.class, + Short512Vector.Short512Shuffle.class, Short512Vector::new); /** Species representing {@link ShortVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -4396,6 +4389,7 @@ static ShortSpecies species(VectorShape s) { = new ShortSpecies(VectorShape.S_Max_BIT, ShortMaxVector.class, ShortMaxVector.ShortMaxMask.class, + ShortMaxVector.ShortMaxShuffle.class, ShortMaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java index 6cfe5f6d9d93e..3158bed0596e6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java @@ -24,6 +24,8 @@ */ package jdk.incubator.vector; +import jdk.internal.vm.annotation.ForceInline; + import java.lang.foreign.MemorySegment; import java.nio.ByteOrder; @@ -342,6 +344,7 @@ public interface VectorSpecies { * @see #withLanes(Class) * @see #withShape(VectorShape) */ + @ForceInline static VectorSpecies of(Class elementType, VectorShape shape) { LaneType laneType = LaneType.of(elementType); return AbstractSpecies.findSpecies(elementType, laneType, shape); @@ -367,6 +370,7 @@ static VectorSpecies of(Class elementType, VectorShape shape) { * or if the given type is not a valid {@code ETYPE} * @see VectorSpecies#ofPreferred(Class) */ + @ForceInline static VectorSpecies ofLargestShape(Class etype) { return VectorSpecies.of(etype, VectorShape.largestShapeFor(etype)); } @@ -410,6 +414,7 @@ static VectorSpecies ofLargestShape(Class etype) { * @see VectorShape#preferredShape() * @see VectorSpecies#ofLargestShape(Class) */ + @ForceInline public static VectorSpecies ofPreferred(Class etype) { return of(etype, VectorShape.preferredShape()); } @@ -432,6 +437,7 @@ public static VectorSpecies ofPreferred(Class etype) { * if the given {@code elementType} argument is not * a valid vector {@code ETYPE} */ + @ForceInline static int elementSize(Class elementType) { return LaneType.of(elementType).elementSize; } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template index aa2f708fc37b3..af7ff88f48986 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template @@ -2856,10 +2856,9 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { return r1.blend(r0, valid); } -#if[!FP] @ForceInline - private final - VectorShuffle<$Boxtype$> toShuffle0($Type$Species dsp) { + final + VectorShuffle toShuffle0(AbstractSpecies dsp) { $type$[] a = toArray(); int[] sa = new int[a.length]; for (int i = 0; i < a.length; i++) { @@ -2868,43 +2867,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { return VectorShuffle.fromArray(dsp, sa, 0); } - /*package-private*/ - @ForceInline - final VectorShuffle<$Boxtype$> toShuffleTemplate(Class shuffleType) { - $Type$Species vsp = vspecies(); - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - getClass(), $type$.class, length(), - shuffleType, $type$.class, length(), - this, vsp, - $Type$Vector::toShuffle0); - } -#end[!FP] - -#if[intOrLong] - abstract AbstractShuffle<$Boxfptype$> toFPShuffle(); - - @ForceInline - private final - AbstractShuffle<$Boxfptype$> toFPShuffle0(VectorSpecies<$Boxfptype$> dsp) { - $type$[] a = toArray(); - int[] sa = new int[a.length]; - for (int i = 0; i < a.length; i++) { - sa[i] = (int) a[i]; - } - return (AbstractShuffle<$Boxfptype$>) VectorShuffle.fromArray(dsp, sa, 0); - } - - @ForceInline - final - AbstractShuffle<$Boxfptype$> toFPShuffleTemplate(Class shuffleType, $Fptype$Vector.$Fptype$Species dsp) { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - getClass(), $type$.class, length(), - shuffleType, $type$.class, length(), - this, dsp, - $Type$Vector::toFPShuffle0); - } -#end[intOrLong] - /** * {@inheritDoc} * @since 19 @@ -5374,9 +5336,10 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { private $Type$Species(VectorShape shape, Class vectorType, Class> maskType, + Class> shuffleType, Function vectorFactory) { super(shape, LaneType.of($type$.class), - vectorType, maskType, + vectorType, maskType, shuffleType, vectorFactory); assert(this.elementSize() == $Boxtype$.SIZE); } @@ -5659,6 +5622,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = new $Type$Species(VectorShape.S_64_BIT, $Type$64Vector.class, $Type$64Vector.$Type$64Mask.class, + $Type$64Vector.$Type$64Shuffle.class, $Type$64Vector::new); /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */ @@ -5666,6 +5630,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = new $Type$Species(VectorShape.S_128_BIT, $Type$128Vector.class, $Type$128Vector.$Type$128Mask.class, + $Type$128Vector.$Type$128Shuffle.class, $Type$128Vector::new); /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */ @@ -5673,6 +5638,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = new $Type$Species(VectorShape.S_256_BIT, $Type$256Vector.class, $Type$256Vector.$Type$256Mask.class, + $Type$256Vector.$Type$256Shuffle.class, $Type$256Vector::new); /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */ @@ -5680,6 +5646,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = new $Type$Species(VectorShape.S_512_BIT, $Type$512Vector.class, $Type$512Vector.$Type$512Mask.class, + $Type$512Vector.$Type$512Shuffle.class, $Type$512Vector::new); /** Species representing {@link $Type$Vector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */ @@ -5687,6 +5654,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { = new $Type$Species(VectorShape.S_Max_BIT, $Type$MaxVector.class, $Type$MaxVector.$Type$MaxMask.class, + $Type$MaxVector.$Type$MaxShuffle.class, $Type$MaxVector::new); /** diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template index 231f05c9faf27..406f94f059467 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template @@ -346,29 +346,12 @@ final class $vectortype$ extends $abstractvectortype$ { return (long) super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m); // specialized } -#if[!FP] @Override @ForceInline - public $shuffletype$ toShuffle() { - return ($shuffletype$) super.toShuffleTemplate($shuffletype$.class); // specialize - } -#end[!FP] -#if[FP] - @Override - @ForceInline - public $shuffletype$ toShuffle() { - return ($shuffletype$) castShape($bitsvectortype$.VSPECIES, 0) - .reinterpretAs$Bitstype$s() - .toFPShuffle(); - } -#end[FP] -#if[intOrLong] - @Override - @ForceInline - $fpvectortype$.$Fptype$$bits$Shuffle toFPShuffle() { - return ($fpvectortype$.$Fptype$$bits$Shuffle) super.toFPShuffleTemplate($fpvectortype$.$Fptype$$bits$Shuffle.class, $fpvectortype$.VSPECIES); + public final + VectorShuffle toShuffle(AbstractSpecies dsp) { + return super.toShuffleTemplate(dsp); } -#end[intOrLong] // Specialized unary testing @@ -1109,16 +1092,13 @@ final class $vectortype$ extends $abstractvectortype$ { @Override @ForceInline $bitsvectortype$ toBitsVector() { - return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET, - $shuffletype$.class, $bitstype$.class, VLENGTH, - $bitsvectortype$.class, $bitstype$.class, VLENGTH, - this, vspecies().asIntegral(), - (v, s) -> toBitsVectorHelper(v)); + return ($bitsvectortype$) super.toBitsVectorTemplate(); } - private static $bitsvectortype$ toBitsVectorHelper($shuffletype$ s) { - return ($bitsvectortype$) $bitsvectortype$.VSPECIES.dummyVector() - .vectorFactory(s.indices()); + @Override + @ForceInline + $Bitstype$Vector toBitsVector0() { + return $bitsvectortype$.VSPECIES.dummyVector().vectorFactory(indices()); } @Override From 97c8fabffb9468c75ab670ea9c6c1bda116e8892 Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Fri, 31 Mar 2023 20:16:36 +0800 Subject: [PATCH 12/13] small cosmetics --- src/hotspot/cpu/x86/x86.ad | 2 +- .../share/classes/jdk/incubator/vector/AbstractSpecies.java | 5 +++-- .../share/classes/jdk/incubator/vector/AbstractVector.java | 2 +- .../share/classes/jdk/incubator/vector/VectorSpecies.java | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 67a540a0ba33e..4fe99953efef1 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2171,7 +2171,7 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* return false; } -// Do Vector::rearrange needs preparation of the shuffle argument +// Return true if Vector::rearrange needs preparation of the shuffle argument const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) { switch (elem_bt) { case T_BYTE: return false; diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java index d26e5814c4f96..e6f8a1b8d9661 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractSpecies.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -166,7 +166,8 @@ public final Class> maskType() { return maskType; } - @ForceInline Class> shuffleType() { + @ForceInline + final Class> shuffleType() { return shuffleType; } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java index 5705a2614ad4c..83875e8594b2f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java @@ -200,7 +200,7 @@ final VectorShuffle toShuffleTemplate(AbstractSpecies dsp) { return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST, getClass(), etype, length(), dvtype, dtype, dlength, - this, dsp, + this, dsp, AbstractVector::toShuffle0); } diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java index 3158bed0596e6..03cb0ae559e70 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorSpecies.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 079a6b5f5c5cd9c85066ebfe3bf874576b1b96fb Mon Sep 17 00:00:00 2001 From: Quan Anh Mai Date: Sat, 8 Apr 2023 01:07:52 +0800 Subject: [PATCH 13/13] special case iotaShuffle --- .../share/classes/jdk/incubator/vector/AbstractVector.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java index 83875e8594b2f..8df84cdd89243 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractVector.java @@ -217,6 +217,10 @@ VectorShuffle toShuffle() { @ForceInline @SuppressWarnings({"rawtypes", "unchecked"}) final VectorShuffle iotaShuffle(int start, int step, boolean wrap) { + if (start == 0 && step == 1) { + return iotaShuffle(); + } + if ((length() & (length() - 1)) != 0) { return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, length()))) : shuffleFromOp(i -> i * step + start);